From 902d9fd63a7df6f8714e35313b47e686f02bc205 Mon Sep 17 00:00:00 2001 From: khashab2 Date: Sat, 30 Apr 2016 20:16:08 -0500 Subject: [PATCH 1/9] merging joint training --- .../{JointTrain.scala => JointTrain2.scala} | 9 +- .../cogcomp/saul/classifier/JointTrain3.scala | 94 +++++++++++++++++++ .../classifier/JointTrainSparseNetwork.scala | 94 ------------------- 3 files changed, 97 insertions(+), 100 deletions(-) rename saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/{JointTrain.scala => JointTrain2.scala} (97%) create mode 100644 saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain3.scala delete mode 100644 saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain2.scala similarity index 97% rename from saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala rename to saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain2.scala index edce9cee..b087b6ea 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain2.scala @@ -6,11 +6,10 @@ import edu.illinois.cs.cogcomp.saul.datamodel.node.Node import scala.reflect.ClassTag -/** Created by parisakordjamshidi on 29/01/15. - */ -object JointTrain { - def testClassifiers(cls: Classifier, oracle: Classifier, ds: List[AnyRef]): Unit = { +object JointTrain2 { + def testClassifiers(cls: Classifier, oracle: Classifier, ds: List[AnyRef]): Unit = { + println("total instances = " + ds.size) val results = ds.map({ x => val pri = cls.discreteValue(x) @@ -85,7 +84,5 @@ object JointTrain { } train(node, cls, it - 1) } - } - } diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain3.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain3.scala new file mode 100644 index 00000000..50d9f048 --- /dev/null +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain3.scala @@ -0,0 +1,94 @@ +package edu.illinois.cs.cogcomp.saul.classifier + +import edu.illinois.cs.cogcomp.lbjava.classify.Classifier +import edu.illinois.cs.cogcomp.lbjava.learn.{ Learner, LinearThresholdUnit } +import edu.illinois.cs.cogcomp.saul.datamodel.node.Node + +import scala.reflect.ClassTag + +object JointTrain3 { + def apply[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]], iter: Int = 1)(implicit headTag: ClassTag[HEAD]): Unit = { + jointTrain(node, classifiers, iter) + } + + @scala.annotation.tailrec + def jointTrain[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]], iter: Int)(implicit headTag: ClassTag[HEAD]): Unit = { + println("Joint training iterations: " + iter) + if (iter > 0) { + val allHeads = node.getTrainingInstances + allHeads foreach { head => + classifiers.foreach { + case typedClassifier: ConstrainedClassifier[_, HEAD] => + val oracle = typedClassifier.onClassifier.getLabeler + typedClassifier.getCandidates(head) foreach { candidate => + typedClassifier.onClassifier.classifier match { + case _: LinearThresholdUnit => trainLinearThresholdUnitOnce[HEAD](typedClassifier, oracle, candidate) + case _: SparseNetworkLBP => trainSparseNetworkLearnerOnce[HEAD](typedClassifier, oracle, candidate) + } + } + } + } + jointTrain(node, classifiers, iter - 1) + } + } + + def trainLinearThresholdUnitOnce[HEAD <: AnyRef](typedClassifier: ConstrainedClassifier[_, HEAD], oracle: Classifier, candidate: Any): Unit = { + val result = typedClassifier.classifier.discreteValue(candidate) + val trueLabel = oracle.discreteValue(candidate) + if (result.equals("true") && trueLabel.equals("false")) { + val a = typedClassifier.onClassifier.getExampleArray(candidate) + val a0 = a(0).asInstanceOf[Array[Int]] + val a1 = a(1).asInstanceOf[Array[Double]] + typedClassifier.onClassifier.classifier.asInstanceOf[LinearThresholdUnit].promote(a0, a1, 0.1) + } else if (result.equals("false") && trueLabel.equals("true")) { + val a = typedClassifier.onClassifier.getExampleArray(candidate) + val a0 = a(0).asInstanceOf[Array[Int]] + val a1 = a(1).asInstanceOf[Array[Double]] + typedClassifier.onClassifier.classifier.asInstanceOf[LinearThresholdUnit].demote(a0, a1, 0.1) + } + } + + def trainSparseNetworkLearnerOnce[HEAD <: AnyRef](typedClassifier: ConstrainedClassifier[_, HEAD], oracle: Classifier, candidate: Any): Unit = { + val result = typedClassifier.classifier.discreteValue(candidate) + val trueLabel = oracle.discreteValue(candidate) + val ilearner = typedClassifier.onClassifier.classifier.asInstanceOf[SparseNetworkLBP] + val lLexicon = typedClassifier.onClassifier.getLabelLexicon + var LTU_actual = 0 + var LTU_predicted = 0 + for (i <- 0 until lLexicon.size()) { + if (lLexicon.lookupKey(i).valueEquals(result)) + LTU_predicted = i + if (lLexicon.lookupKey(i).valueEquals(trueLabel)) + LTU_actual = i + } + + // The idea is that when the prediction is wrong the LTU of the actual class should be promoted + // and the LTU of the predicted class should be demoted. + if (!result.equals(trueLabel)) { + val a = typedClassifier.onClassifier.getExampleArray(candidate) + val a0 = a(0).asInstanceOf[Array[Int]] //exampleFeatures + val a1 = a(1).asInstanceOf[Array[Double]] // exampleValues + val exampleLabels = a(2).asInstanceOf[Array[Int]] + val label = exampleLabels(0) + var N = ilearner.net.size() + + if (label >= N || ilearner.net.get(label) == null) { + ilearner.iConjuctiveLables = ilearner.iConjuctiveLables | ilearner.getLabelLexicon.lookupKey(label).isConjunctive + + val ltu: LinearThresholdUnit = ilearner.getbaseLTU + ltu.initialize(ilearner.getnumExamples, ilearner.getnumFeatures) + ilearner.net.set(label, ltu) + N = label + 1 + } + + // test push + val ltu_actual: LinearThresholdUnit = ilearner.getLTU(LTU_actual) + val ltu_predicted: LinearThresholdUnit = ilearner.getLTU(LTU_predicted) + + if (ltu_actual != null) + ltu_actual.promote(a0, a1, 0.1) + if (ltu_predicted != null) + ltu_predicted.demote(a0, a1, 0.1) + } + } +} diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala deleted file mode 100644 index 79c42e6b..00000000 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala +++ /dev/null @@ -1,94 +0,0 @@ -package edu.illinois.cs.cogcomp.saul.classifier - -import edu.illinois.cs.cogcomp.lbjava.learn.{ Learner, LinearThresholdUnit } -import edu.illinois.cs.cogcomp.saul.datamodel.node.Node - -import scala.reflect.ClassTag - -/** Created by Parisa on 5/22/15. - */ -object JointTrainSparseNetwork { - - def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]])(implicit headTag: ClassTag[HEAD]) = { - train[HEAD](node, cls, 1) - } - - def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int)(implicit headTag: ClassTag[HEAD]) = { - train[HEAD](node, cls, it) - } - - @scala.annotation.tailrec - def train[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int)(implicit headTag: ClassTag[HEAD]): Unit = { - // forall members in collection of the head (dm.t) do - println("Training iteration: " + it) - - if (it == 0) { - // Done - } else { - val allHeads = node.getTrainingInstances - allHeads foreach { - head => - { - cls.foreach { - case classifier: ConstrainedClassifier[_, HEAD] => - val typedClassifier = classifier.asInstanceOf[ConstrainedClassifier[_, HEAD]] - val oracle = typedClassifier.onClassifier.getLabeler - - typedClassifier.getCandidates(head) foreach { - candidate => - { - def trainOnce() = { - val result = typedClassifier.classifier.discreteValue(candidate) - val trueLabel = oracle.discreteValue(candidate) - val ilearner = typedClassifier.onClassifier.classifier.asInstanceOf[Learner].asInstanceOf[SparseNetworkLBP] - val lLexicon = typedClassifier.onClassifier.getLabelLexicon - var LTU_actual: Int = 0 - var LTU_predicted: Int = 0 - for (i <- 0 until lLexicon.size()) { - if (lLexicon.lookupKey(i).valueEquals(result)) - LTU_predicted = i - if (lLexicon.lookupKey(i).valueEquals(trueLabel)) - LTU_actual = i - } - - // The idea is that when the prediction is wrong the LTU of the actual class should be promoted - // and the LTU of the predicted class should be demoted. - if (!result.equals(trueLabel)) //equals("true") && trueLabel.equals("false") ) - { - val a = typedClassifier.onClassifier.getExampleArray(candidate) - val a0 = a(0).asInstanceOf[Array[Int]] //exampleFeatures - val a1 = a(1).asInstanceOf[Array[Double]] // exampleValues - val exampleLabels = a(2).asInstanceOf[Array[Int]] - val label = exampleLabels(0) - var N = ilearner.net.size() - - if (label >= N || ilearner.net.get(label) == null) { - ilearner.iConjuctiveLables = ilearner.iConjuctiveLables | ilearner.getLabelLexicon.lookupKey(label).isConjunctive - - val ltu: LinearThresholdUnit = ilearner.getbaseLTU - ltu.initialize(ilearner.getnumExamples, ilearner.getnumFeatures) - ilearner.net.set(label, ltu) - N = label + 1 - } - - // test push - val ltu_actual: LinearThresholdUnit = ilearner.getLTU(LTU_actual) //.net.get(i).asInstanceOf[LinearThresholdUnit] - val ltu_predicted: LinearThresholdUnit = ilearner.getLTU(LTU_predicted) - - if (ltu_actual != null) - ltu_actual.promote(a0, a1, 0.1) - if (ltu_predicted != null) - ltu_predicted.demote(a0, a1, 0.1) - } - } - - trainOnce() - } - } - } - } - } - train(node, cls, it - 1) - } - } -} From 614048ae7eb2b29ba999ca8bb42e96e4f1acd7c1 Mon Sep 17 00:00:00 2001 From: khashab2 Date: Sat, 30 Apr 2016 20:47:25 -0500 Subject: [PATCH 2/9] joint training in ER test. --- .../EntityRelation/EntityRelationTests.scala | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala index 98865367..493ab827 100644 --- a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala +++ b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala @@ -1,15 +1,17 @@ package edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation -import edu.illinois.cs.cogcomp.saul.classifier.ClassifierUtils +import edu.illinois.cs.cogcomp.saul.classifier.{ JointTrain, ClassifierUtils } +import edu.illinois.cs.cogcomp.saulexamples.EntityMentionRelation.datastruct.ConllRelation import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationClassifiers._ -import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationConstrainedClassifiers.{ WorksFor_PerOrg_ConstrainedClassifier, OrgConstrainedClassifier, PerConstrainedClassifier } +import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationConstrainedClassifiers._ +import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationDataModel._ import org.scalatest._ class EntityRelationTests extends FlatSpec with Matchers { val minScore = 0.3 EntityRelationDataModel.populateWithConllSmallSet() - "entity classifier " should " should work. " in { + "entity classifier " should " work. " in { ClassifierUtils.LoadClassifier( EntityRelationApp.jarModelPath, PersonClassifier, OrganizationClassifier, LocationClassifier @@ -18,7 +20,7 @@ class EntityRelationTests extends FlatSpec with Matchers { scores.foreach { case (label, score) => (score._1 > minScore) should be(true) } } - "independent relation classifier " should " should work. " in { + "independent relation classifier " should " work. " in { ClassifierUtils.LoadClassifier( EntityRelationApp.jarModelPath, WorksForClassifier, LivesInClassifier, LocatedInClassifier, OrgBasedInClassifier @@ -28,7 +30,7 @@ class EntityRelationTests extends FlatSpec with Matchers { scores.foreach { case (label, score) => (score._1 > minScore) should be(true) } } - "pipeline relation classifiers " should " should work. " in { + "pipeline relation classifiers " should " work. " in { ClassifierUtils.LoadClassifier( EntityRelationApp.jarModelPath, PersonClassifier, OrganizationClassifier, LocationClassifier, @@ -38,7 +40,7 @@ class EntityRelationTests extends FlatSpec with Matchers { scores.foreach { case (label, score) => (score._1 > minScore) should be(true) } } - "L+I entity-relation classifiers " should " should work. " in { + "L+I entity-relation classifiers " should " work. " in { ClassifierUtils.LoadClassifier( EntityRelationApp.jarModelPath, PersonClassifier, OrganizationClassifier, LocationClassifier, @@ -47,4 +49,27 @@ class EntityRelationTests extends FlatSpec with Matchers { val scores = PerConstrainedClassifier.test() ++ WorksFor_PerOrg_ConstrainedClassifier.test() scores.foreach { case (label, score) => (score._1 > minScore) should be(true) } } + + "Joint training for ER " should "work" in { + populateWithConll() + val testRels = pairs.getTestingInstances.toList + val testTokens = tokens.getTestingInstances.toList + + // load pre-trained independent models + ClassifierUtils.LoadClassifier(EntityRelationApp.jarModelPath, PersonClassifier, OrganizationClassifier, LocationClassifier, + WorksForClassifier, LivesInClassifier, LocatedInClassifier, OrgBasedInClassifier) + + // joint training + val jointTrainIteration = 1 + println(s"Joint training $jointTrainIteration iterations. ") + JointTrain.train[ConllRelation]( + pairs, + PerConstrainedClassifier :: OrgConstrainedClassifier :: LocConstrainedClassifier :: + WorksFor_PerOrg_ConstrainedClassifier :: LivesIn_PerOrg_relationConstrainedClassifier :: Nil, + jointTrainIteration + ) + + val scores = PerConstrainedClassifier.test(testTokens) ++ WorksFor_PerOrg_ConstrainedClassifier.test(testRels) + scores.foreach { case (label, score) => (score._1 > minScore) should be(true) } + } } \ No newline at end of file From f13284e09dbe697b31d62e6223c06182689d9726 Mon Sep 17 00:00:00 2001 From: khashab2 Date: Sat, 30 Apr 2016 20:47:48 -0500 Subject: [PATCH 3/9] get rid of the old joint training. --- .../{JointTrain3.scala => JointTrain.scala} | 8 +- .../cogcomp/saul/classifier/JointTrain2.scala | 88 ------------------- .../EntityRelation/EntityRelationApp.scala | 3 + 3 files changed, 7 insertions(+), 92 deletions(-) rename saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/{JointTrain3.scala => JointTrain.scala} (93%) delete mode 100644 saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain2.scala diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain3.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala similarity index 93% rename from saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain3.scala rename to saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala index 50d9f048..50538024 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain3.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala @@ -6,13 +6,13 @@ import edu.illinois.cs.cogcomp.saul.datamodel.node.Node import scala.reflect.ClassTag -object JointTrain3 { +object JointTrain { def apply[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]], iter: Int = 1)(implicit headTag: ClassTag[HEAD]): Unit = { - jointTrain(node, classifiers, iter) + train(node, classifiers, iter) } @scala.annotation.tailrec - def jointTrain[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]], iter: Int)(implicit headTag: ClassTag[HEAD]): Unit = { + def train[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]], iter: Int)(implicit headTag: ClassTag[HEAD]): Unit = { println("Joint training iterations: " + iter) if (iter > 0) { val allHeads = node.getTrainingInstances @@ -28,7 +28,7 @@ object JointTrain3 { } } } - jointTrain(node, classifiers, iter - 1) + train(node, classifiers, iter - 1) } } diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain2.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain2.scala deleted file mode 100644 index b087b6ea..00000000 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain2.scala +++ /dev/null @@ -1,88 +0,0 @@ -package edu.illinois.cs.cogcomp.saul.classifier - -import edu.illinois.cs.cogcomp.lbjava.classify.Classifier -import edu.illinois.cs.cogcomp.lbjava.learn.LinearThresholdUnit -import edu.illinois.cs.cogcomp.saul.datamodel.node.Node - -import scala.reflect.ClassTag - -object JointTrain2 { - - def testClassifiers(cls: Classifier, oracle: Classifier, ds: List[AnyRef]): Unit = { - println("total instances = " + ds.size) - val results = ds.map({ - x => - val pri = cls.discreteValue(x) - val truth = oracle.discreteValue(x) - (pri, truth) - }) - - val tp = results.count({ case (x, y) => x == y && (x == "true") }) * 1.0 - val fp = results.count({ case (x, y) => x != y && (x == "true") }) * 1.0 - - val tn = results.count({ case (x, y) => x == y && (x == "false") }) * 1.0 - val fn = results.count({ case (x, y) => x != y && (x == "false") }) * 1.0 - - println(s"tp: $tp fp: $fp tn: $tn fn: $fn ") - println(s" accuracy ${(tp + tn) / results.size} ") - println(s" precision ${tp / (tp + fp)} ") - println(s" recall ${tp / (tp + fn)} ") - println(s" f1 ${(2.0 * tp) / (2 * tp + fp + fn)} ") - - } - - def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]])(implicit headTag: ClassTag[HEAD]) = { - train[HEAD](node, cls, 1) - } - - def apply[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int)(implicit headTag: ClassTag[HEAD]) = { - train[HEAD](node, cls, it) - } - - @scala.annotation.tailrec - def train[HEAD <: AnyRef](node: Node[HEAD], cls: List[ConstrainedClassifier[_, HEAD]], it: Int)(implicit headTag: ClassTag[HEAD]): Unit = { - // forall members in collection of the head (dm.t) do - - println("Training iteration: " + it) - if (it == 0) { - // Done - } else { - val allHeads = node.getTrainingInstances - - allHeads foreach { - h => - { - cls.foreach { - case classifier: ConstrainedClassifier[_, HEAD] => - val typedC = classifier.asInstanceOf[ConstrainedClassifier[_, HEAD]] - val oracle = typedC.onClassifier.getLabeler - - typedC.getCandidates(h) foreach { - x => - { - def trainOnce() = { - val result = typedC.classifier.discreteValue(x) - val trueLabel = oracle.discreteValue(x) - - if (result.equals("true") && trueLabel.equals("false")) { - val a = typedC.onClassifier.getExampleArray(x) - val a0 = a(0).asInstanceOf[Array[Int]] - val a1 = a(1).asInstanceOf[Array[Double]] - typedC.onClassifier.classifier.asInstanceOf[LinearThresholdUnit].promote(a0, a1, 0.1) - } else if (result.equals("false") && trueLabel.equals("true")) { - val a = typedC.onClassifier.getExampleArray(x) - val a0 = a(0).asInstanceOf[Array[Int]] - val a1 = a(1).asInstanceOf[Array[Double]] - typedC.onClassifier.classifier.asInstanceOf[LinearThresholdUnit].demote(a0, a1, 0.1) - } - } - trainOnce() - } - } - } - } - } - train(node, cls, it - 1) - } - } -} diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala index 588cc7bc..2603b946 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala @@ -118,6 +118,8 @@ object EntityRelationApp { jointTrainIteration ) + // TODO: fix these after addressing the evaluation of joint + /* println(Console.BLUE + "Peop") JointTrain.testClassifiers(PersonClassifier.classifier, PersonClassifier.label.classifier, testTokens) @@ -147,5 +149,6 @@ object EntityRelationApp { println(Console.RED + "Live_In") JointTrain.testClassifiers(LivesIn_PerOrg_relationConstrainedClassifier.classifier, (relationType is "Live_In").classifier, testRels) + */ } } From 924d2ac85adde2e31c0722cb5f765ec7b040a3bb Mon Sep 17 00:00:00 2001 From: khashab2 Date: Sat, 30 Apr 2016 21:01:12 -0500 Subject: [PATCH 4/9] small fix to the ER test. --- .../saulexamples/nlp/EntityRelation/EntityRelationTests.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala index 493ab827..9e5a99a6 100644 --- a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala +++ b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala @@ -51,7 +51,6 @@ class EntityRelationTests extends FlatSpec with Matchers { } "Joint training for ER " should "work" in { - populateWithConll() val testRels = pairs.getTestingInstances.toList val testTokens = tokens.getTestingInstances.toList From 1d21eaa7e0272985e5acc2a2cc57c1496625dcc5 Mon Sep 17 00:00:00 2001 From: khashab2 Date: Sun, 1 May 2016 16:20:58 -0500 Subject: [PATCH 5/9] adding a comment for the case when joint training is not available for the given classifier type. --- .../edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala index 50538024..6c8bb4f4 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala @@ -24,6 +24,8 @@ object JointTrain { typedClassifier.onClassifier.classifier match { case _: LinearThresholdUnit => trainLinearThresholdUnitOnce[HEAD](typedClassifier, oracle, candidate) case _: SparseNetworkLBP => trainSparseNetworkLearnerOnce[HEAD](typedClassifier, oracle, candidate) + case _ => println("ERROR: The joint training is not available for the classifier you have: " + + typedClassifier.onClassifier.classifier.getClass.getTypeName) } } } From b246622b9c241b968d8d72ca1913fc6760920589 Mon Sep 17 00:00:00 2001 From: khashab2 Date: Thu, 5 May 2016 15:41:40 -0500 Subject: [PATCH 6/9] making the joint training a little efficient by creating partial calls. --- .../cogcomp/saul/classifier/JointTrain.scala | 61 +++++++++++++------ .../EntityRelation/EntityRelationApp.scala | 8 +-- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala index 6c8bb4f4..d890961f 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala @@ -1,40 +1,61 @@ package edu.illinois.cs.cogcomp.saul.classifier import edu.illinois.cs.cogcomp.lbjava.classify.Classifier -import edu.illinois.cs.cogcomp.lbjava.learn.{ Learner, LinearThresholdUnit } +import edu.illinois.cs.cogcomp.lbjava.learn.LinearThresholdUnit import edu.illinois.cs.cogcomp.saul.datamodel.node.Node import scala.reflect.ClassTag +/** Joint training for a list of classifiers with the same HEAD type, but potentially different classifiers with + * different input types */ object JointTrain { - def apply[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]], iter: Int = 1)(implicit headTag: ClassTag[HEAD]): Unit = { - train(node, classifiers, iter) + def train[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]])(implicit headTag: ClassTag[HEAD]): Unit = { + train[HEAD](node, classifiers, 1) } - @scala.annotation.tailrec def train[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]], iter: Int)(implicit headTag: ClassTag[HEAD]): Unit = { + /** this creates partial calls to the promote/demote function of the classifiers. Later these partial calls are + * called given the instances and their predictions. Note that the goal of creating this list of partial calls + * is to increase efficiency, by doing it once and at the beginning, for all the iterations. + */ + val partialUpdateCalls: List[Any => Unit] = classifiers.map { + case typedClassifier: ConstrainedClassifier[_, HEAD] => + val oracle = typedClassifier.onClassifier.getLabeler + typedClassifier.onClassifier.classifier match { + case _: LinearThresholdUnit => trainLinearThresholdUnitOnce[HEAD](typedClassifier, oracle)(_) + case _: SparseNetworkLBP => trainSparseNetworkLearnerOnce[HEAD](typedClassifier, oracle)(_) + case _ => throw new Exception("ERROR: The joint training is not available for the classifier you have: " + + typedClassifier.onClassifier.classifier.getClass.getTypeName) + } + } + recurstiveTrain(node, classifiers, partialUpdateCalls, iter) + } + + @scala.annotation.tailrec + private def recurstiveTrain[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]], partialUpdateCalls: List[Any => Unit], iter: Int)(implicit headTag: ClassTag[HEAD]): Unit = { println("Joint training iterations: " + iter) if (iter > 0) { val allHeads = node.getTrainingInstances - allHeads foreach { head => - classifiers.foreach { - case typedClassifier: ConstrainedClassifier[_, HEAD] => - val oracle = typedClassifier.onClassifier.getLabeler - typedClassifier.getCandidates(head) foreach { candidate => - typedClassifier.onClassifier.classifier match { - case _: LinearThresholdUnit => trainLinearThresholdUnitOnce[HEAD](typedClassifier, oracle, candidate) - case _: SparseNetworkLBP => trainSparseNetworkLearnerOnce[HEAD](typedClassifier, oracle, candidate) - case _ => println("ERROR: The joint training is not available for the classifier you have: " + - typedClassifier.onClassifier.classifier.getClass.getTypeName) - } - } - } + for { + head <- allHeads + partialUpdateCall <- partialUpdateCalls + classifier <- classifiers + } { +// println("head = ") +// println(head) +// println("partialUpdateCall = ") +// println(partialUpdateCall) +// println("classifier = ") +// println(classifier) +// println(headTag) +// println(classifier.getCandidates(head)) + classifier.getCandidates(head) foreach { candidate => partialUpdateCall(candidate) } } - train(node, classifiers, iter - 1) + recurstiveTrain(node, classifiers, partialUpdateCalls, iter - 1) } } - def trainLinearThresholdUnitOnce[HEAD <: AnyRef](typedClassifier: ConstrainedClassifier[_, HEAD], oracle: Classifier, candidate: Any): Unit = { + private def trainLinearThresholdUnitOnce[HEAD <: AnyRef](typedClassifier: ConstrainedClassifier[_, HEAD], oracle: Classifier)(candidate: Any): Unit = { val result = typedClassifier.classifier.discreteValue(candidate) val trueLabel = oracle.discreteValue(candidate) if (result.equals("true") && trueLabel.equals("false")) { @@ -50,7 +71,7 @@ object JointTrain { } } - def trainSparseNetworkLearnerOnce[HEAD <: AnyRef](typedClassifier: ConstrainedClassifier[_, HEAD], oracle: Classifier, candidate: Any): Unit = { + private def trainSparseNetworkLearnerOnce[HEAD <: AnyRef](typedClassifier: ConstrainedClassifier[_, HEAD], oracle: Classifier)(candidate: Any): Unit = { val result = typedClassifier.classifier.discreteValue(candidate) val trueLabel = oracle.discreteValue(candidate) val ilearner = typedClassifier.onClassifier.classifier.asInstanceOf[SparseNetworkLBP] diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala index 2603b946..d8126399 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala @@ -1,7 +1,6 @@ package edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation import edu.illinois.cs.cogcomp.saul.classifier.{ ClassifierUtils, JointTrain } -import edu.illinois.cs.cogcomp.saulexamples.EntityMentionRelation.datastruct.ConllRelation import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationClassifiers._ import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationDataModel._ import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationConstrainedClassifiers._ @@ -12,7 +11,7 @@ object EntityRelationApp { def main(args: Array[String]): Unit = { /** Choose the experiment you're interested in by changing the following line */ - val testType = ERExperimentType.LPlusI + val testType = ERExperimentType.JointTraining testType match { case ERExperimentType.IndependentClassifiers => trainIndependentClassifiers() @@ -111,10 +110,9 @@ object EntityRelationApp { // joint training val jointTrainIteration = 5 println(s"Joint training $jointTrainIteration iterations. ") - JointTrain.train[ConllRelation]( + JointTrain.train( pairs, - PerConstrainedClassifier :: OrgConstrainedClassifier :: LocConstrainedClassifier :: - WorksFor_PerOrg_ConstrainedClassifier :: LivesIn_PerOrg_relationConstrainedClassifier :: Nil, + PerConstrainedClassifier :: OrgConstrainedClassifier :: LocConstrainedClassifier :: Nil, jointTrainIteration ) From ae4ddc2f2176b159f39786c6c13b17eeb9f9ea09 Mon Sep 17 00:00:00 2001 From: khashab2 Date: Thu, 5 May 2016 16:48:17 -0500 Subject: [PATCH 7/9] joint training: fixed type issues in partial calls. --- .../cogcomp/saul/classifier/JointTrain.scala | 36 +++++++++---------- .../EntityRelation/EntityRelationApp.scala | 5 ++- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala index d890961f..370372eb 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrain.scala @@ -7,18 +7,23 @@ import edu.illinois.cs.cogcomp.saul.datamodel.node.Node import scala.reflect.ClassTag /** Joint training for a list of classifiers with the same HEAD type, but potentially different classifiers with - * different input types */ + * different input types + */ object JointTrain { - def train[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]])(implicit headTag: ClassTag[HEAD]): Unit = { - train[HEAD](node, classifiers, 1) + def train[HEAD <: AnyRef](node: Node[HEAD], classifiers: ConstrainedClassifier[_, HEAD]*)(implicit headTag: ClassTag[HEAD]): Unit = { + train[HEAD](node, 1, classifiers) } - def train[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]], iter: Int)(implicit headTag: ClassTag[HEAD]): Unit = { + def train[HEAD <: AnyRef](node: Node[HEAD], iter: Int, classifiers: ConstrainedClassifier[_, HEAD]*)(implicit headTag: ClassTag[HEAD]): Unit = { + train[HEAD](node, iter, classifiers) + } + + def train[HEAD <: AnyRef](node: Node[HEAD], iter: Int, classifiers: Seq[ConstrainedClassifier[_, HEAD]])(implicit headTag: ClassTag[HEAD], d1: DummyImplicit): Unit = { /** this creates partial calls to the promote/demote function of the classifiers. Later these partial calls are * called given the instances and their predictions. Note that the goal of creating this list of partial calls * is to increase efficiency, by doing it once and at the beginning, for all the iterations. */ - val partialUpdateCalls: List[Any => Unit] = classifiers.map { + val partialUpdateCallsPerClassifier = classifiers.map { case typedClassifier: ConstrainedClassifier[_, HEAD] => val oracle = typedClassifier.onClassifier.getLabeler typedClassifier.onClassifier.classifier match { @@ -28,30 +33,25 @@ object JointTrain { typedClassifier.onClassifier.classifier.getClass.getTypeName) } } - recurstiveTrain(node, classifiers, partialUpdateCalls, iter) + + val callsZipWithClassifiers = partialUpdateCallsPerClassifier.zip(classifiers) + + // now do training with the partial calls + recurstiveTrain[HEAD](node, callsZipWithClassifiers, iter) } @scala.annotation.tailrec - private def recurstiveTrain[HEAD <: AnyRef](node: Node[HEAD], classifiers: List[ConstrainedClassifier[_, HEAD]], partialUpdateCalls: List[Any => Unit], iter: Int)(implicit headTag: ClassTag[HEAD]): Unit = { + private def recurstiveTrain[HEAD <: AnyRef](node: Node[HEAD], callsZipWithClassifiers: Seq[(Any => Unit, ConstrainedClassifier[_, HEAD])], iter: Int)(implicit headTag: ClassTag[HEAD]): Unit = { println("Joint training iterations: " + iter) if (iter > 0) { val allHeads = node.getTrainingInstances for { head <- allHeads - partialUpdateCall <- partialUpdateCalls - classifier <- classifiers + (partialUpdateCall, classifier) <- callsZipWithClassifiers } { -// println("head = ") -// println(head) -// println("partialUpdateCall = ") -// println(partialUpdateCall) -// println("classifier = ") -// println(classifier) -// println(headTag) -// println(classifier.getCandidates(head)) classifier.getCandidates(head) foreach { candidate => partialUpdateCall(candidate) } } - recurstiveTrain(node, classifiers, partialUpdateCalls, iter - 1) + recurstiveTrain(node, callsZipWithClassifiers, iter - 1) } } diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala index fde50766..ee88f7f1 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala @@ -111,9 +111,8 @@ object EntityRelationApp { val jointTrainIteration = 5 println(s"Joint training $jointTrainIteration iterations. ") JointTrain.train( - pairs, - PerConstrainedClassifier :: OrgConstrainedClassifier :: LocConstrainedClassifier :: Nil, - jointTrainIteration + pairs, jointTrainIteration, + PerConstrainedClassifier, OrgConstrainedClassifier, LocConstrainedClassifier, LivesIn_PerOrg_relationConstrainedClassifier, WorksFor_PerOrg_ConstrainedClassifier ) // TODO: merge the following two tests From 3ab151371f91922acfa83829fda39bc8dfdd339a Mon Sep 17 00:00:00 2001 From: khashab2 Date: Thu, 5 May 2016 17:11:23 -0500 Subject: [PATCH 8/9] fix ER unit test. --- .../nlp/EntityRelation/EntityRelationTests.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala index 9e5a99a6..0d1de74e 100644 --- a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala +++ b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala @@ -62,10 +62,9 @@ class EntityRelationTests extends FlatSpec with Matchers { val jointTrainIteration = 1 println(s"Joint training $jointTrainIteration iterations. ") JointTrain.train[ConllRelation]( - pairs, - PerConstrainedClassifier :: OrgConstrainedClassifier :: LocConstrainedClassifier :: - WorksFor_PerOrg_ConstrainedClassifier :: LivesIn_PerOrg_relationConstrainedClassifier :: Nil, - jointTrainIteration + pairs, jointTrainIteration, + PerConstrainedClassifier, OrgConstrainedClassifier, LocConstrainedClassifier, + WorksFor_PerOrg_ConstrainedClassifier, LivesIn_PerOrg_relationConstrainedClassifier ) val scores = PerConstrainedClassifier.test(testTokens) ++ WorksFor_PerOrg_ConstrainedClassifier.test(testRels) From 627f4ca906fe5419052ccb4854079d1b563de67f Mon Sep 17 00:00:00 2001 From: khashab2 Date: Thu, 5 May 2016 17:11:59 -0500 Subject: [PATCH 9/9] version minor bump. --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index aa892183..cec97aa5 100644 --- a/build.sbt +++ b/build.sbt @@ -7,7 +7,7 @@ lazy val root = (project in file(".")). lazy val commonSettings = Seq( organization := "edu.illinois.cs.cogcomp", name := "saul-project", - version := "0.1", + version := "0.2", scalaVersion := "2.11.7", resolvers ++= Seq( Resolver.mavenLocal,