diff --git a/build.sbt b/build.sbt index 4746a096..1b6bb5b8 100644 --- a/build.sbt +++ b/build.sbt @@ -63,17 +63,20 @@ lazy val commonSettings = Seq( libraryDependencies ++= Seq( ccgGroupId % "LBJava" % "1.2.25" withSources, ccgGroupId % "illinois-core-utilities" % cogcompNLPVersion withSources, + ccgGroupId % "illinois-inference" % "0.9.0" withSources, "com.gurobi" % "gurobi" % "6.0", "org.apache.commons" % "commons-math3" % "3.0", "org.scalatest" % "scalatest_2.11" % "2.2.4", "ch.qos.logback" % "logback-classic" % "1.1.7" ), + scalacOptions ++= Seq("-unchecked", "-feature", "-language:postfixOps"), fork := true, connectInput in run := true, headers := Map( "scala" -> (HeaderPattern.cStyleBlockComment, headerMsg), "java" -> (HeaderPattern.cStyleBlockComment, headerMsg) - ) + ), + testOptions in Test += Tests.Argument("-oF") // shows the complete stack-trace, if things break in the test ) ++ publishSettings lazy val root = (project in file(".")) diff --git a/saul-core/doc/SAULLANGUAGE.md b/saul-core/doc/SAULLANGUAGE.md index f4cab89a..8e4daa10 100644 --- a/saul-core/doc/SAULLANGUAGE.md +++ b/saul-core/doc/SAULLANGUAGE.md @@ -20,14 +20,14 @@ object OrgClassifier extends Learnable[ConllRawToken](ErDataModelExample) { ``` ### Training and Testing classifiers --Call `train()` method to train your classifier using the populated data in the data model's training instances: + - Call `train()` method to train your classifier using the populated data in the data model's training instances: ```scala OrgClassifier.learn(numberOfIterations) ``` where number of iteration determines how many times the training algorithm should iterate over training data. --Call `test()` method to test your classifier using the populated data model's test instance: + - Call `test()` method to test your classifier using the populated data model's test instance: ```scala OrgClassifier.test() @@ -73,33 +73,179 @@ OrgClassifier.save() This would add the suffix "20-iterations" to the files of the classifier at the time of saving them. Note that at the time of calling `load()` method it will look for model files with suffix "20-iterations". -## Constraints +## Constrained Classifiers +A constrained classifiers is a classifier that predicts the class labels subject to a specified constraints. +Here is the general form: + + +```scala +object CONSTRAINED_CLASSIFIER extends ConstraintClassifier[INPUT_TYPE, HEAD_TYPE] { + override lazy val onClassifier = CLASSIFIER + override def subjectTo = Some(CONSTRAINT) // optional + override def pathToHead = Some(PATH-TO-HEAD-EDGE) // optional + override def filter: (t: INPUT_TYPE, h:HEAD_TYPE) => Boolean // optional + override def solverType = SOLVER // optional +} +``` + +Here we describe each of the parameters in the above snippet: + + - `CONSTRAINED_CLASSIFIER`: the name of your desired constrained classifier + - `INPUT_TYPE`: the input type of the desired constrained classifier + - `HEAD_TYPE`: the inference starts from the head object. This type often subsumes `INPUT_TYPE`. For example if we define + constraints over sentences while making predictions for each word, `INPUT_TYPE` would be a consituent type in the + sentence, while `HEAD_TYPE` would be a sentential type. + - `CLASSIFIER`: The base classifier based on which the confidence scores of the constrained inference problem is set. + - `CONSTRAINT`: The constraint definition. For more details see the section below. + - `SOLVER`: The ILP solver machine used for the inference. Here are the possible values for `solverType`: + - `OJAlgo`: The [OjAlgo solver](http://ojalgo.org/), an opensource solver. + - `Gurobi`: Gurobi, a powerful industrial solver. + - `Balas`: Egon Balas' zero-one ILP solving algorithm. It is a branch and bound algorithm that can return the best + solution found so far if stopped early. + More details can be found in the [`illinois-inference` package](https://gitlab-beta.engr.illinois.edu/cogcomp/inference/). + - `PATH-TO-HEAD-EDGE`: Returns only one object of type `HEAD_TYPE` given an instance of `INPUT_TYPE`; if there are many + of them i.e. `Iterable[HEAD]` then it simply returns the head object. + - `filter`: The function is used to filter the generated candidates from the head object; remember that + the inference starts from the head object. This function finds the objects of type `INPUT_TYPE` which are + connected to the target object of type `HEAD_TYPE`. If we don't define `filter`, by default it returns all + objects connected to `HEAD_TYPE`. The filter is useful for the `JointTraining` when we go over all + global objects and generate all contained objects that serve as examples for the basic classifiers involved in + the `JoinTraining`. It is possible that we do not want to use all possible candidates but some of them, for + example when we have a way to filter the negative candidates, this can come in the filter. + +Here is an example usage of this definition: + +```scala +object OrgConstrainedClassifier extends ConstrainedClassifier[ConllRawToken, ConllRelation] { + override lazy val onClassifier = EntityRelationClassifiers.OrganizationClassifier + override def pathToHead = Some(-EntityRelationDataModel.pairTo2ndArg) + override def subjectTo = Some(EntityRelationConstraints.relationArgumentConstraints) + override def filter(t: ConllRawToken, h: ConllRelation): Boolean = t.wordId == h.wordId2 + override def solverType = OJAlgo +} +``` + +In this example, the base (non-constrained) classifier is `OrganizationClassifier` which predicts whether the given instance +(of type `ConllRawToken`) is an organization or not. Since the constraints `relationArgumentConstraints` are defined over +triples (i.e two entities and the relation between them), the head type is defined as `ConllRelation` (which is +relatively more general than `ConllRawToken`). The filter function ensures that the head relation corresponds to the given +input entity token. + +**Tip:** The constrained classifier is using in-memory caching to make the inference faster. If you want to turn off caching + just include `override def useCaching = false` in body of the classifier definition. + +### Constraints A "constraint" is a logical restriction over possible values that can be assigned to a number of variables; For example, a binary constraint could be `{if {A} then NOT {B}}`. -In Saul, the constraints are defined for the assignments to class labels. -A constraint classifiers is a classifier that predicts the class labels with regard to the specified constraints. +In Saul, the constraints are defined for the assignments to class labels. In what follows we outine the details of operators +which help us define the constraints. Before jumping into the details, note that you have to have the following import +in order to have the following operators work: + +```scala +import edu.illinois.cs.cogcomp.saul.infer.Constraint._ +``` -This is done with the following construct +#### `Node` as starting point -```scala -val PersonWorkFor=ConstraintClassifier.constraintOf[ConllRelation] - { - x:ConllRelation => - { - ((workForClassifier on x) isTrue) ==> ((PersonClassifier on x.e1) isTrue) - } - } +In Saul the starting point for writing programs are nodes. The `ForEach` operator, connects a node (and its instances) to constraints. For example: + +```scala +someNode.ForEach { x: HEAD_TYPE => Some-Constraint-On-X } +``` + +where `Some-Constraint-On-X` is a constraint (which we will define in the following sections). +An important point here is that, if you are defining using a constrained classifier with head type `HEAD_TYPE`, the definition of the constraint have to start with the node corresponding to this type. + +#### Propositional constraints + This defines constraint on the prediction of a classifier on a given instance. Here is the basic form. Consider an + imaginary classifier `SomeClassifier` which returns `A` or `B`. Here is how we create propositional constraint + to force the prediction on instance `x` to have label `A`: +``` + SomeClassifier on x is "A" +``` + +In the above definition, `on` and `is` are keywords. + +Here different variations and extensions to this basic usage: + + - If the label were `true` and `false`, one can use `isTrue` instead of `is "true"` (and similarily `isFalse` instead of `is "false"`). + - If instead of equality you want to use inequality, you can use the keyword `isNot`, instead of `is`. + - If you want to use the equality on multiple label values, you can use the `isOneOf(.)` keywors, instead of `is`. + - If you want a classifier have the same label on two different instances, you can do: + +```scala + SomeClassifier on x1 equalsTo x2 ``` + Similarly if you want a classifier have the different label on two different instances, you can do: -## Constrained Classifiers +```scala + SomeClassifier on x1 differentFrom x2 +``` -A constrained classifier can be defined in the following form: + - If you want two different classifiers have the same labels on a instances, you can do: -```scala -object LocConstraintClassifier extends ConstraintClassifier[ConllRawToken, ConllRelation](ErDataModelExample, LocClassifier) - { - def subjectTo = Per_Org - override val pathToHead = Some('containE2) - //override def filter(t: ConllRawToken,h:ConllRelation): Boolean = t.wordId==h.wordId2 - } - ``` \ No newline at end of file +```scala + SomeClassifier1 on x equalsTo SomeClassifier2 +``` + + And similarly if you want two different classifiers have different labels on a instances, you can do: + +```scala + SomeClassifier1 on x differentFrom SomeClassifier2 +``` + + +#### Binary and Unary binary operators + +One way of combining base logical rules is applying binary operations (for example conjunction, etc). + +| Operator | Name | Definition | Example | +|----------|---------------|---------|------| +| `and` | conjunction | A binary operator to create a conjunction of the two constraints before and after it | `(SomeClassifier1 on x is "A1") and (SomeClassifier2 on y is "A2")` | +| `or` | disjunction | A binary operator to create a disjunction of the two constraints before and after it | `(SomeClassifier1 on x is "A1") or (SomeClassifier2 on y is "A2")` | +| `==>` | implication | The implication operator, meaning that if the contraint before it is true, the constraint following it must be true as well | `(SomeClassifier1 on x is "A1") ==> (SomeClassifier2 on y is "A2")` | +| `<==>` | double implication | The double-implication operator (aka "if and only if"), meaning that if the contraint before it is true, if and only if the constraint following it is true as well | `(SomeClassifier1 on x is "A1") <==> (SomeClassifier2 on y is "A2")` | +| `!` | negation | A prefix unary operator to negate the effect of the constraint following it. | `!(SomeClassifier1 on x is "A1")` | + +#### Collection operators + +This operators distribute the definitions of the constraints over collections. Here are the definition and examples: + +| Operator | Definition | Example | +|----------|------------|---------|---| +| `ForAll` | For **all** the elements in the collection it applies the constraints. In other words, the constrain should hold for **all** elements of the collection. | `textAnnotationNode.ForAll { x: TextAnnotation => Some-Constraint-On-x }` | +| `Exists` | The constrain should hold for **at least one** element of the collection. | `textAnnotationNode.Exists { x: TextAnnotation => Some-Constraint-On-x }` | +| `AtLeast(k: Int)` | The constrain should hold for **at least `k`** elements of the collection. | `textAnnotationNode.AtLeast(2) { x: TextAnnotation => Some-Constraint-On-x }` | +| `AtMost(k: Int)` | The constrain should hold for **at most `k`** elements of the collection. | `textAnnotationNode.AtMost(3) { x: TextAnnotation => Some-Constraint-On-x }` | +| `Exactly(k: Int)` | The constrain should hold for **exactly `k`** elements of the collection. | `textAnnotationNode.Exactly(3){ x: TextAnnotation => Some-Constraint-On-x }` | + +**Tip:** Except `ForEach` which is used only on nodes, all the above operators can be used as postfix operator on the list of constraints. For example: + +```scala +val constraintCollection = for { + // some complicated loop variables +} + yield someConstraint + +constraintCollection.ForAll +``` + + +There are just the definitions of the operations. If you want to see real examples of the operators in actions see [the definitions of constraints for ER-example](https://github.com/IllinoisCogComp/saul/blob/master/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationConstraints.scala). + +**Tip:** Note whenever the constrained inference is infeasible (i.e. the constraints are overly tight), we use the default +prediction of the base classifier. Hence if you see the performance of the constrained classifier is very close to the performance +of the base classifier it's probably most of your inference problems are becoming infeasible. In such cases it is worth verifying +the correctness of your constraint definitions. + +#### Common mistakes in using constrained classifiers + - Not defining the constraints with `def` keyword (instead defining them with the `val` keyword). The `def` keyword + makes the propositionalization of the constraints lazy, i.e. it waits until you call them and then they get + evaluated. + - If you face the following error: + ``` + requirement failed: The target value Some(SomeLabel) is not a valid value for classifier ClassifierName with the tag-set: Set(SomeTags) + ``` + it often means that it is constraiend to have a label which it does not contain in output label lexicon. Another reason + for this can be not loading the base classifier model properly. + diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/ClassifierUtils.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/ClassifierUtils.scala index f211d057..5b1a0135 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/ClassifierUtils.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/ClassifierUtils.scala @@ -6,7 +6,7 @@ */ package edu.illinois.cs.cogcomp.saul.classifier -import edu.illinois.cs.cogcomp.saul.classifier.infer.InitSparseNetwork +import edu.illinois.cs.cogcomp.saul.classifier.infer._ import edu.illinois.cs.cogcomp.saul.datamodel.node.Node import edu.illinois.cs.cogcomp.saul.util.Logging @@ -69,72 +69,72 @@ object ClassifierUtils extends Logging { def apply[T <: AnyRef](c: (Learnable[T], Iterable[T])*): Seq[Results] = { val testResults = c.map { case (learner, testInstances) => - logger.info(evalSeparator) - logger.info("Evaluating " + learner.getClassSimpleNameForClassifier) + println(evalSeparator) + println("Evaluating " + learner.getClassSimpleNameForClassifier) learner.test(testInstances) } - logger.info(evalSeparator) + println(evalSeparator) testResults } def apply[T <: AnyRef](testInstances: Iterable[T], c: Learnable[T]*): Seq[Results] = { val testResults = c.map { learner => - logger.info(evalSeparator) - logger.info("Evaluating " + learner.getClassSimpleNameForClassifier) + println(evalSeparator) + println("Evaluating " + learner.getClassSimpleNameForClassifier) learner.test(testInstances) } - logger.info(evalSeparator) + println(evalSeparator) testResults } def apply(c: Learnable[_]*)(implicit d1: DummyImplicit, d2: DummyImplicit): Seq[Results] = { val testResults = c.map { learner => - logger.info(evalSeparator) - logger.info("Evaluating " + learner.getClassSimpleNameForClassifier) + println(evalSeparator) + println("Evaluating " + learner.getClassSimpleNameForClassifier) learner.test() } - logger.info(evalSeparator) + println(evalSeparator) testResults } def apply(c: List[Learnable[_]])(implicit d1: DummyImplicit, d2: DummyImplicit, d3: DummyImplicit): Seq[Results] = { val testResults = c.map { learner => - logger.info(evalSeparator) - logger.info("Evaluating " + learner.getClassSimpleNameForClassifier) + println(evalSeparator) + println("Evaluating " + learner.getClassSimpleNameForClassifier) learner.test() } - logger.info(evalSeparator) + println(evalSeparator) testResults } def apply(c: ConstrainedClassifier[_, _]*)(implicit d1: DummyImplicit, d2: DummyImplicit, d3: DummyImplicit): Seq[Results] = { val testResults = c.map { learner => - logger.info(evalSeparator) - logger.info("Evaluating " + learner.getClassSimpleNameForClassifier) + println(evalSeparator) + println("Evaluating " + learner.getClassSimpleNameForClassifier) learner.test() } - logger.info(evalSeparator) + println(evalSeparator) testResults } def apply[T <: AnyRef](testInstances: Iterable[T], c: ConstrainedClassifier[T, _]*)(implicit d1: DummyImplicit, d2: DummyImplicit, d3: DummyImplicit): Seq[Results] = { val testResults = c.map { learner => - logger.info(evalSeparator) - logger.info("Evaluating " + learner.getClassSimpleNameForClassifier) + println(evalSeparator) + println("Evaluating " + learner.getClassSimpleNameForClassifier) learner.test(testInstances) } - logger.info(evalSeparator) + println(evalSeparator) testResults } def apply[T <: AnyRef](instanceClassifierPairs: (Iterable[T], ConstrainedClassifier[T, _])*)(implicit d1: DummyImplicit, d2: DummyImplicit, d3: DummyImplicit, d4: DummyImplicit): Seq[Results] = { val testResults = instanceClassifierPairs.map { case (testInstances, learner) => - logger.info(evalSeparator) - logger.info("Evaluating " + learner.getClassSimpleNameForClassifier) + println(evalSeparator) + println("Evaluating " + learner.getClassSimpleNameForClassifier) learner.test(testInstances) } - logger.info(evalSeparator) + println(evalSeparator) testResults } } @@ -169,7 +169,7 @@ object ClassifierUtils extends Logging { object InitializeClassifiers { def apply[HEAD <: AnyRef](node: Node[HEAD], cl: ConstrainedClassifier[_, HEAD]*) = { - cl.map { + cl.foreach { constrainedLearner => InitSparseNetwork(node, constrainedLearner) } diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/ConstrainedClassifier.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/ConstrainedClassifier.scala deleted file mode 100644 index 337956f6..00000000 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/ConstrainedClassifier.scala +++ /dev/null @@ -1,193 +0,0 @@ -/** This software is released under the University of Illinois/Research and Academic Use License. See - * the LICENSE file in the root folder for details. Copyright (c) 2016 - * - * Developed by: The Cognitive Computations Group, University of Illinois at Urbana-Champaign - * http://cogcomp.cs.illinois.edu/ - */ -package edu.illinois.cs.cogcomp.saul.classifier - -import edu.illinois.cs.cogcomp.lbjava.classify.{ Classifier, FeatureVector, TestDiscrete } -import edu.illinois.cs.cogcomp.infer.ilp.{ GurobiHook, ILPSolver, OJalgoHook } -import edu.illinois.cs.cogcomp.lbjava.infer.{ BalasHook, FirstOrderConstraint, InferenceManager } -import edu.illinois.cs.cogcomp.lbjava.learn.Learner -import edu.illinois.cs.cogcomp.saul.classifier.infer.InferenceCondition -import edu.illinois.cs.cogcomp.saul.constraint.LfsConstraint -import edu.illinois.cs.cogcomp.saul.datamodel.edge.Edge -import edu.illinois.cs.cogcomp.saul.lbjrelated.{ LBJClassifierEquivalent, LBJLearnerEquivalent } -import edu.illinois.cs.cogcomp.saul.parser.IterableToLBJavaParser -import edu.illinois.cs.cogcomp.saul.test.TestWithStorage -import edu.illinois.cs.cogcomp.saul.util.Logging -import scala.reflect.ClassTag - -/** The input to a ConstrainedClassifier is of type `T`. However given an input, the inference is based upon the - * head object of type `HEAD` corresponding to the input (of type `T`). - * - * @tparam T the object type given to the classifier as input - * @tparam HEAD the object type inference is based upon - */ -abstract class ConstrainedClassifier[T <: AnyRef, HEAD <: AnyRef](val onClassifier: LBJLearnerEquivalent)( - implicit - val tType: ClassTag[T], - implicit val headType: ClassTag[HEAD] -) extends LBJClassifierEquivalent with Logging { - - type LEFT = T - type RIGHT = HEAD - - def className: String = this.getClass.getName - - def getClassSimpleNameForClassifier = this.getClass.getSimpleName - - def __allowableValues: List[String] = "*" :: "*" :: Nil - - def subjectTo: LfsConstraint[HEAD] - - def solver: ILPSolver = new GurobiHook() - - /** The function is used to filter the generated candidates from the head object; remember that the inference starts - * from the head object. This function finds the objects of type `T` which are connected to the target object of - * type `HEAD`. If we don't define `filter`, by default it returns all objects connected to `HEAD`. - * The filter is useful for the `JointTraining` when we go over all global objects and generate all contained object - * that serve as examples for the basic classifiers involved in the `JoinTraining`. It is possible that we do not - * want to use all possible candidates but some of them, for example when we have a way to filter the negative - * candidates, this can come in the filter. - */ - def filter(t: T, head: HEAD): Boolean = true - - /** The `pathToHead` returns only one object of type HEAD, if there are many of them i.e. `Iterable[HEAD]` then it - * simply returns the `head` of the `Iterable` - */ - val pathToHead: Option[Edge[T, HEAD]] = None - - /** syntactic sugar to create simple calls to the function */ - def apply(example: AnyRef): String = classifier.discreteValue(example: AnyRef) - - def findHead(x: T): Option[HEAD] = { - if (tType.equals(headType) || pathToHead.isEmpty) { - Some(x.asInstanceOf[HEAD]) - } else { - val l = pathToHead.get.forward.neighborsOf(x).toSet.toList - - if (l.isEmpty) { - logger.error("Warning: Failed to find head") - None - } else if (l.size != 1) { - logger.warn("Find too many heads") - Some(l.head) - } else { - logger.info(s"Found head ${l.head} for child $x") - Some(l.head) - } - } - } - - def getCandidates(head: HEAD): Seq[T] = { - if (tType.equals(headType) || pathToHead.isEmpty) { - head.asInstanceOf[T] :: Nil - } else { - val l = pathToHead.get.backward.neighborsOf(head) - - if (l.isEmpty) { - logger.error("Failed to find part") - Seq.empty[T] - } else { - l.filter(filter(_, head)).toSeq - } - } - } - - def buildWithConstraint(infer: InferenceCondition[T, HEAD], cls: Learner)(t: T): String = { - findHead(t) match { - case Some(head) => - val name = String.valueOf(infer.subjectTo.hashCode()) - var inference = InferenceManager.get(name, head) - if (inference == null) { - inference = infer(head) - logger.warn(s"Inference ${name} has not been cached; running inference . . . ") - InferenceManager.put(name, inference) - } - inference.valueOf(cls, t) - - case None => - cls.discreteValue(t) - } - } - - def buildWithConstraint(inferenceCondition: InferenceCondition[T, HEAD])(t: T): String = { - buildWithConstraint(inferenceCondition, onClassifier.classifier)(t) - } - - private def getSolverInstance = solver match { - case _: OJalgoHook => () => new OJalgoHook() - case _: GurobiHook => () => new GurobiHook() - case _: BalasHook => () => new BalasHook() - } - - override val classifier = new Classifier() { - override def classify(o: scala.Any) = new FeatureVector(featureValue(discreteValue(o))) - override def discreteValue(o: scala.Any): String = - buildWithConstraint( - subjectTo.createInferenceCondition[T](getSolverInstance()).convertToType[T], - onClassifier.classifier - )(o.asInstanceOf[T]) - } - - /** Derives test instances from the data model - * - * @return Iterable of test instances for this classifier - */ - private def deriveTestInstances: Iterable[T] = { - pathToHead.map(edge => edge.from) - .orElse({ - onClassifier match { - case clf: Learnable[T] => Some(clf.node) - case _ => logger.error("pathToHead is not provided and the onClassifier is not a Learnable!"); None - } - }) - .map(node => node.getTestingInstances) - .getOrElse(Iterable.empty) - } - - /** Test Constrained Classifier with automatically derived test instances. - * - * @return List of (label, (f1,precision,recall)) - */ - def test(): Results = { - test(deriveTestInstances) - } - - /** Test with given data, use internally - * - * @param testData if the collection of data (which is and Iterable of type T) is not given it is derived from the data model based on its type - * @param exclude it is the label that we want to exclude for evaluation, this is useful for evaluating the multi-class classifiers when we need to measure overall F1 instead of accuracy and we need to exclude the negative class - * @param outFile The file to write the predictions (can be `null`) - * @return List of (label, (f1,precision,recall)) - */ - def test(testData: Iterable[T] = null, outFile: String = null, outputGranularity: Int = 0, exclude: String = ""): Results = { - println() - - val testReader = new IterableToLBJavaParser[T](if (testData == null) deriveTestInstances else testData) - testReader.reset() - - val tester: TestDiscrete = new TestDiscrete() - TestWithStorage.test(tester, classifier, onClassifier.getLabeler, testReader, outFile, outputGranularity, exclude) - val perLabelResults = tester.getLabels.map { - label => - ResultPerLabel(label, tester.getF1(label), tester.getPrecision(label), tester.getRecall(label), - tester.getAllClasses, tester.getLabeled(label), tester.getPredicted(label), tester.getCorrect(label)) - } - val overalResultArray = tester.getOverallStats() - val overalResult = OverallResult(overalResultArray(0), overalResultArray(1), overalResultArray(2)) - Results(perLabelResults, ClassifierUtils.getAverageResults(perLabelResults), overalResult) - } -} - -object ConstrainedClassifier { - val ConstraintManager = scala.collection.mutable.HashMap[Int, LfsConstraint[_]]() - def constraint[HEAD <: AnyRef](f: HEAD => FirstOrderConstraint)(implicit headTag: ClassTag[HEAD]): LfsConstraint[HEAD] = { - val hash = f.hashCode() - ConstraintManager.getOrElseUpdate(hash, new LfsConstraint[HEAD] { - override def makeConstrainDef(x: HEAD): FirstOrderConstraint = f(x) - }).asInstanceOf[LfsConstraint[HEAD]] - } -} diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala index e114fadf..cb31411d 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparseNetwork.scala @@ -7,13 +7,12 @@ package edu.illinois.cs.cogcomp.saul.classifier import edu.illinois.cs.cogcomp.lbjava.learn.{ LinearThresholdUnit, SparseNetworkLearner } +import edu.illinois.cs.cogcomp.saul.classifier.infer.ConstrainedClassifier import edu.illinois.cs.cogcomp.saul.datamodel.node.Node import org.slf4j.{ Logger, LoggerFactory } import Predef._ import scala.reflect.ClassTag -/** Created by Parisa on 5/22/15. - */ object JointTrainSparseNetwork { val logger: Logger = LoggerFactory.getLogger(this.getClass) @@ -58,8 +57,7 @@ object JointTrainSparseNetwork { candidate => { def trainOnce() = { - - val result = currentClassifier.classifier.discreteValue(candidate) + val result = currentClassifier.onClassifier.classifier.discreteValue(candidate) val trueLabel = oracle.discreteValue(candidate) val lLexicon = currentClassifier.onClassifier.getLabelLexicon var LTU_actual: Int = 0 diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparsePerceptron.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparsePerceptron.scala index 4b70d89e..547bd5c6 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparsePerceptron.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/JointTrainSparsePerceptron.scala @@ -8,6 +8,7 @@ package edu.illinois.cs.cogcomp.saul.classifier import edu.illinois.cs.cogcomp.lbjava.classify.Classifier import edu.illinois.cs.cogcomp.lbjava.learn.LinearThresholdUnit +import edu.illinois.cs.cogcomp.saul.classifier.infer.ConstrainedClassifier import edu.illinois.cs.cogcomp.saul.datamodel.node.Node import scala.reflect.ClassTag @@ -68,7 +69,7 @@ object JointTrainSparsePerceptron { x => { def trainOnce() = { - val result = typedC.classifier.discreteValue(x) + val result = typedC.onClassifier.classifier.discreteValue(x) val trueLabel = oracle.discreteValue(x) if (result.equals("true") && trueLabel.equals("false")) { diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/ConstrainedClassifier.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/ConstrainedClassifier.scala new file mode 100644 index 00000000..2b7ed7f7 --- /dev/null +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/ConstrainedClassifier.scala @@ -0,0 +1,336 @@ +/** This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computations Group, University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.saul.classifier.infer + +import java.util.Date + +import edu.illinois.cs.cogcomp.core.io.LineIO +import edu.illinois.cs.cogcomp.infer.ilp.{ GurobiHook, ILPSolver, OJalgoHook } +import edu.illinois.cs.cogcomp.lbjava.classify.TestDiscrete +import edu.illinois.cs.cogcomp.lbjava.infer.BalasHook +import edu.illinois.cs.cogcomp.saul.classifier._ +import edu.illinois.cs.cogcomp.saul.datamodel.edge.Edge +import edu.illinois.cs.cogcomp.saul.lbjrelated.LBJLearnerEquivalent +import edu.illinois.cs.cogcomp.saul.util.Logging + +import scala.collection.{ Iterable, Seq, mutable } +import scala.reflect.ClassTag + +/** possible solvers to use */ +sealed trait SolverType +case object Gurobi extends SolverType +case object OJAlgo extends SolverType +case object Balas extends SolverType + +abstract class ConstrainedClassifier[T <: AnyRef, HEAD <: AnyRef]( + implicit + val tType: ClassTag[T], + implicit val headType: ClassTag[HEAD] +) extends Logging { + + def onClassifier: LBJLearnerEquivalent + protected def subjectTo: Option[Constraint[HEAD]] = None + protected def solverType: SolverType = OJAlgo + protected def useCaching: Boolean = true + + protected sealed trait OptimizationType + protected case object Max extends OptimizationType + protected case object Min extends OptimizationType + protected def optimizationType: OptimizationType = Max + + private val inferenceManager = new InferenceManager() + + def getClassSimpleNameForClassifier = this.getClass.getSimpleName + + /** The function is used to filter the generated candidates from the head object; remember that the inference starts + * from the head object. This function finds the objects of type [[T]] which are connected to the target object of + * type [[HEAD]]. If we don't define [[filter]], by default it returns all objects connected to [[HEAD]]. + * The filter is useful for the `JointTraining` when we go over all global objects and generate all contained object + * that serve as examples for the basic classifiers involved in the `JoinTraining`. It is possible that we do not + * want to use all possible candidates but some of them, for example when we have a way to filter the negative + * candidates, this can come in the filter. + */ + protected def filter(t: T, head: HEAD): Boolean = true + + /** The [[pathToHead]] returns only one object of type HEAD, if there are many of them i.e. `Iterable[HEAD]` then it + * simply returns the head of the [[Iterable]] + */ + protected def pathToHead: Option[Edge[T, HEAD]] = None + + private def deriveTestInstances: Iterable[T] = { + pathToHead.map(edge => edge.from) + .orElse({ + onClassifier match { + case clf: Learnable[T] => Some(clf.node) + case _ => logger.error("pathToHead is not provided and the onClassifier is not a Learnable!"); None + } + }) + .map(node => node.getTestingInstances) + .getOrElse(Iterable.empty) + } + + def getCandidates(head: HEAD): Seq[T] = { + if (tType.equals(headType) || pathToHead.isEmpty) { + Seq(head.asInstanceOf[T]) + } else { + val l = pathToHead.get.backward.neighborsOf(head) + l.size match { + case 0 => + logger.error("Failed to find part") + Seq.empty[T] + case _ => l.filter(filter(_, head)).toSeq + } + } + } + + private def findHead(x: T): Option[HEAD] = { + if (tType.equals(headType) || pathToHead.isEmpty) { + Some(x.asInstanceOf[HEAD]) + } else { + val l = pathToHead.get.forward.neighborsOf(x).toSet.toSeq + l.length match { + case 0 => + logger.trace("Failed to find head") + None + case 1 => + logger.trace(s"Found head ${l.head} for child $x") + Some(l.head) + case _ => + logger.trace("Found too many heads; this is usually because some instances belong to multiple 'head's") + Some(l.head) + } + } + } + + /** given a solver type, instantiates a solver, uppon calling it */ + private def getSolverInstance: ILPSolver = solverType match { + case OJAlgo => new OJalgoHook() + case Gurobi => new GurobiHook() + case Balas => new BalasHook() + case _ => throw new Exception("Hook not found! ") + } + + /** given an instance */ + def apply(t: T): String = { + findHead(t) match { + case Some(head) => build(head, t) + case None => onClassifier.classifier.discreteValue(t) + } + } + + private def getInstancesInvolvedInProblem(constraintsOpt: Option[Constraint[_]]): Option[Set[_]] = { + constraintsOpt.map { constraint => getInstancesInvolved(constraint) } + } + + private def getClassifiersInvolvedInProblem(constraintsOpt: Option[Constraint[_]]): Option[Set[LBJLearnerEquivalent]] = { + constraintsOpt.map { constraint => getClassifiersInvolved(constraint) } + } + + /** given a head instance, produces a constraint based off of it */ + private def instantiateConstraintGivenInstance(head: HEAD): Option[Constraint[_]] = { + // look at only the first level; if it is PerInstanceConstraint, replace it. + subjectTo.map { + case constraint: PerInstanceConstraint[HEAD] => constraint.sensor(head) + case constraint: Constraint[_] => constraint + } + } + + /** find all the instances used in the definiton of the constraint. This is used in caching the results of inference */ + private def getInstancesInvolved(constraint: Constraint[_]): Set[_] = { + constraint match { + case c: PropositionalEqualityConstraint[_] => + Set(c.instanceOpt.get) + case c: PairConjunction[_, _] => + getInstancesInvolved(c.c1) ++ getInstancesInvolved(c.c2) + case c: PairDisjunction[_, _] => + getInstancesInvolved(c.c1) ++ getInstancesInvolved(c.c2) + case c: Negation[_] => + getInstancesInvolved(c.p) + case c: ConstraintCollections[_, _] => + c.constraints.foldRight(Set[Any]()) { + case (singleConstraint, ins) => + ins union getInstancesInvolved(singleConstraint).asInstanceOf[Set[Any]] + } + case c: EstimatorPairEqualityConstraint[_] => + Set(c.instance) + case c: InstancePairEqualityConstraint[_] => + Set(c.instance1, c.instance2Opt.get) + case _ => + throw new Exception("Unknown constraint exception! This constraint should have been rewritten in terms of other constraints. ") + } + } + + /** find all the classifiers involved in the definition of the constraint. This is used for caching of inference */ + private def getClassifiersInvolved(constraint: Constraint[_]): Set[LBJLearnerEquivalent] = { + constraint match { + case c: PropositionalEqualityConstraint[_] => + Set(c.estimator) + case c: PairConjunction[_, _] => + getClassifiersInvolved(c.c1) ++ getClassifiersInvolved(c.c2) + case c: PairDisjunction[_, _] => + getClassifiersInvolved(c.c1) ++ getClassifiersInvolved(c.c2) + case c: Negation[_] => + getClassifiersInvolved(c.p) + case c: ConstraintCollections[_, _] => + c.constraints.foldRight(Set[LBJLearnerEquivalent]()) { + case (singleConstraint, ins) => + ins union getClassifiersInvolved(singleConstraint) + } + case c: EstimatorPairEqualityConstraint[_] => + Set(c.estimator1, c.estimator2Opt.get) + case c: InstancePairEqualityConstraint[_] => + Set(c.estimator) + case _ => + throw new Exception("Unknown constraint exception! This constraint should have been rewritten in terms of other constraints. ") + } + } + + private def build(head: HEAD, t: T)(implicit d: DummyImplicit): String = { + val constraintsOpt = instantiateConstraintGivenInstance(head) + val instancesInvolved = getInstancesInvolvedInProblem(constraintsOpt) + val classifiersInvolved = getClassifiersInvolvedInProblem(constraintsOpt) + if (constraintsOpt.isDefined && instancesInvolved.get.isEmpty) { + logger.warn("there are no instances associated with the constraints. It might be because you have defined " + + "the constraints with 'val' modifier, instead of 'def'.") + } + val instanceIsInvolvedInConstraint = instancesInvolved.exists { set => + set.exists { + case x: T => x == t + case _ => false + } + } + val classifierIsInvolvedInProblem = classifiersInvolved.exists { classifierSet => + classifierSet.exists { + case c: LBJLearnerEquivalent => onClassifier == c + case _ => false + } + } + if (instanceIsInvolvedInConstraint & classifierIsInvolvedInProblem) { + /** The following cache-key is very important, as it defines what to and when to cache the results of the inference. + * The first term encodes the instances involved in the constraint, after propositionalization, and the second term + * contains pure definition of the constraint before any propositionalization. + */ + val cacheKey = instancesInvolved.map(_.toString).toSeq.sorted.mkString("*") + constraintsOpt + val resultOpt = if (useCaching) InferenceManager.cachedResults.get(cacheKey) else None + resultOpt match { + case Some((cachedSolver, cachedClassifier, cachedEstimatorToSolverLabelMap)) => + getInstanceLabel(t, cachedSolver, onClassifier, cachedEstimatorToSolverLabelMap) + case None => + // create a new solver instance + val solver = getSolverInstance + solver.setMaximize(optimizationType == Max) + + // populate the instances connected to head + val candidates = getCandidates(head) + inferenceManager.addVariablesToInferenceProblem(candidates, onClassifier, solver) + + constraintsOpt.foreach { constraints => + val inequalities = inferenceManager.processConstraints(constraints, solver) + inequalities.foreach { ineq => + solver.addGreaterThanConstraint(ineq.x, ineq.a, ineq.b) + } + } + + solver.solve() + + if (!solver.isSolved) { + logger.warn("Instance not solved . . . ") + } + + if (useCaching) { + InferenceManager.cachedResults.put(cacheKey, (solver, onClassifier, inferenceManager.estimatorToSolverLabelMap)) + } + + getInstanceLabel(t, solver, onClassifier, inferenceManager.estimatorToSolverLabelMap) + } + } else { + // if the instance doesn't involve in any constraints, it means that it's a simple non-constrained problem. + logger.trace("getting the label with the highest score . . . ") + onClassifier.classifier.scores(t).highScoreValue() + } + } + + /** given an instance, the result of the inference insidde an [[ILPSolver]], and a hashmap which connects + * classifier labels to solver's internal variables, returns a label for a given instance + */ + private def getInstanceLabel(t: T, solver: ILPSolver, + classifier: LBJLearnerEquivalent, + estimatorToSolverLabelMap: mutable.Map[LBJLearnerEquivalent, mutable.Map[_, Seq[(Int, String)]]]): String = { + val estimatorSpecificMap = estimatorToSolverLabelMap(classifier).asInstanceOf[mutable.Map[T, Seq[(Int, String)]]] + estimatorSpecificMap.get(t) match { + case Some(indexLabelPairs) => + val values = indexLabelPairs.map { + case (ind, _) => solver.getIntegerValue(ind) + } + // exactly one label should be active; if not, [probably] the inference has been infeasible and + // it is not usable, in which case we make direct calls to the non-constrained classifier. + if (values.sum == 1) { + indexLabelPairs.collectFirst { + case (ind, label) if solver.getIntegerValue(ind) == 1.0 => label + }.get + } else { + classifier.classifier.scores(t).highScoreValue() + } + case None => throw new Exception("instance is not cached ... weird! :-/ ") + } + } + + /** Test Constrained Classifier with automatically derived test instances. + * + * @return A [[Results]] object + */ + def test(): Results = { + test(deriveTestInstances) + } + + /** Test with given data, use internally + * + * @param testData if the collection of data (which is and Iterable of type T) is not given it is derived from the + * data model based on its type + * @param exclude it is the label that we want to exclude for evaluation, this is useful for evaluating the multi-class + * classifiers when we need to measure overall F1 instead of accuracy and we need to exclude the negative class + * @param outFile The file to write the predictions (can be `null`) + * @return A [[Results]] object + */ + def test(testData: Iterable[T] = null, outFile: String = null, outputGranularity: Int = 0, exclude: String = ""): Results = { + val testReader = if (testData == null) deriveTestInstances else testData + val tester = new TestDiscrete() + if (exclude.nonEmpty) tester.addNull(exclude) + testReader.zipWithIndex.foreach { + case (instance, idx) => + val gold = onClassifier.getLabeler.discreteValue(instance) + val prediction = apply(instance) + tester.reportPrediction(prediction, gold) + + if (outputGranularity > 0 && idx % outputGranularity == 0) { + println(idx + " examples tested at " + new Date()) + } + + // Append the predictions to a file (if the outFile parameter is given) + if (outFile != null) { + try { + val line = "Example " + idx + "\tprediction:\t" + prediction + "\t gold:\t" + gold + "\t" + (if (gold.equals(prediction)) "correct" else "incorrect") + LineIO.append(outFile, line); + } catch { + case e: Exception => e.printStackTrace() + } + } + } + + println() // for an extra empty line, for visual convenience :) + tester.printPerformance(System.out) + + val perLabelResults = tester.getLabels.map { + label => + ResultPerLabel(label, tester.getF1(label), tester.getPrecision(label), tester.getRecall(label), + tester.getAllClasses, tester.getLabeled(label), tester.getPredicted(label), tester.getCorrect(label)) + } + val overallResultArray = tester.getOverallStats() + val overallResult = OverallResult(overallResultArray(0), overallResultArray(1), overallResultArray(2)) + Results(perLabelResults, ClassifierUtils.getAverageResults(perLabelResults), overallResult) + } +} diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/Constraints.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/Constraints.scala new file mode 100644 index 00000000..3182fe80 --- /dev/null +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/Constraints.scala @@ -0,0 +1,265 @@ +/** This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computations Group, University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.saul.classifier.infer + +import edu.illinois.cs.cogcomp.saul.datamodel.node.Node +import edu.illinois.cs.cogcomp.saul.lbjrelated.LBJLearnerEquivalent + +import scala.collection.mutable +import scala.reflect.ClassTag + +import scala.collection.JavaConverters._ +import scala.language.implicitConversions + +object Constraint { + implicit class LearnerToFirstOrderConstraint1(estimator: LBJLearnerEquivalent) { + // connecting a classifier to a specific instance + def on[T](newInstance: T)(implicit tag: ClassTag[T]): InstanceWrapper[T] = InstanceWrapper(newInstance, estimator) + } + + implicit def toPropositionalEqualityConstraint[T](wrapper: InstanceWrapper[T])( + implicit + tag: ClassTag[T] + ): PropositionalEqualityConstraint[T] = { + new PropositionalEqualityConstraint[T](wrapper.estimator, Some(wrapper.instance), None, None) + } + implicit def toInstancePairEqualityConstraint[T](wrapper: InstanceWrapper[T])(implicit + tag: ClassTag[T], + d1: DummyImplicit): InstancePairEqualityConstraint[T] = { + new InstancePairEqualityConstraint[T](wrapper.estimator, wrapper.instance, None, None) + } + implicit def toEstimatorPairEqualityConstraint[T]( + wrapper: InstanceWrapper[T] + )(implicit tag: ClassTag[T]): EstimatorPairEqualityConstraint[T] = { + new EstimatorPairEqualityConstraint[T](wrapper.estimator, None, wrapper.instance, None) + } + + // collection of target object types + implicit def firstOrderConstraint[T <: AnyRef]( + coll: Traversable[T] + ): ConstraintObjWrapper[T] = new ConstraintObjWrapper[T](coll.toSeq) + implicit def firstOrderConstraint[T <: AnyRef]( + coll: Set[T] + ): ConstraintObjWrapper[T] = new ConstraintObjWrapper[T](coll.toSeq) + implicit def firstOrderConstraint[T <: AnyRef]( + coll: java.util.Collection[T] + ): ConstraintObjWrapper[T] = new ConstraintObjWrapper[T](coll.asScala.toSeq) + implicit def firstOrderConstraint[T <: AnyRef]( + coll: mutable.LinkedHashSet[T] + ): ConstraintObjWrapper[T] = new ConstraintObjWrapper[T](coll.toSeq) + implicit def firstOrderConstraint[T <: AnyRef]( + node: Node[T] + ): ConstraintObjWrapper[T] = { + new ConstraintObjWrapper[T](node.getAllInstances.toSeq) + } + + // node object + implicit def nodeObjectConstraint[T <: AnyRef](node: Node[T]): NodeWrapper[T] = { + new NodeWrapper[T](node) + } + + // collection of constraints + implicit def createConstraintCollection[T <: AnyRef]( + coll: Traversable[Constraint[T]] + ): ConstraintCollection[T, T] = new ConstraintCollection[T, T](coll.toSet) + implicit def createConstraintCollection[T <: AnyRef]( + coll: Set[Constraint[T]] + ): ConstraintCollection[T, T] = new ConstraintCollection[T, T](coll) + implicit def createConstraintCollection[T <: AnyRef]( + coll: java.util.Collection[Constraint[T]] + ): ConstraintCollection[T, T] = new ConstraintCollection[T, T](coll.asScala.toSet) + implicit def createConstraintCollection[T <: AnyRef]( + coll: mutable.LinkedHashSet[Constraint[T]] + ): ConstraintCollection[T, T] = new ConstraintCollection[T, T](coll.toSet) +} + +class ConstraintCollection[T, U](coll: Set[Constraint[U]]) { + def ForAll = new ForAll[T, U](coll) + def Exists = new AtLeast[T, U](coll, 1) + def AtLeast(k: Int) = new AtLeast[T, U](coll, k) + def AtMost(k: Int) = new AtMost[T, U](coll, k) + def Exactly(k: Int) = new Exactly[T, U](coll, k) +} + +class NodeWrapper[T <: AnyRef](node: Node[T]) { + def ForEach(sensor: T => Constraint[_]) = PerInstanceConstraint(sensor) +} + +case class PerInstanceConstraint[T](sensor: T => Constraint[_]) extends Constraint[T] { + override def negate: Constraint[T] = ??? +} + +class ConstraintObjWrapper[T](coll: Seq[T]) { + def ForAll[U](sensors: T => Constraint[U])(implicit tag: ClassTag[T]): ForAll[T, U] = { + new ForAll[T, U](coll.map(sensors).toSet) + } + def Exists[U](sensors: T => Constraint[U])(implicit tag: ClassTag[T]): AtLeast[T, U] = { + new AtLeast[T, U](coll.map(sensors).toSet, 1) + } + def AtLeast[U](k: Int)(sensors: T => Constraint[U])(implicit tag: ClassTag[T]): AtLeast[T, U] = { + new AtLeast[T, U](coll.map(sensors).toSet, k) + } + def AtMost[U](k: Int)(sensors: T => Constraint[U])(implicit tag: ClassTag[T]): AtMost[T, U] = { + new AtMost[T, U](coll.map(sensors).toSet, k) + } + def Exactly[U](k: Int)(sensors: T => Constraint[U])(implicit tag: ClassTag[T]): Exactly[T, U] = { + new Exactly[T, U](coll.map(sensors).toSet, k) + } +} + +sealed trait Constraint[T] { + def and[U](cons: Constraint[U]) = { + new PairConjunction[T, U](this, cons) + } + + def or[U](cons: Constraint[U]) = { + new PairDisjunction[T, U](this, cons) + } + + def implies[U](q: Constraint[U]): PairDisjunction[T, U] = { + // p --> q can be modelled as not(p) or q + //PairConjunction[T, U](this.negate, q) + this.negate or (q and this) + } + def ==>[U](q: Constraint[U]): PairDisjunction[T, U] = implies(q) + + def ifAndOnlyIf[U](q: Constraint[U]): PairDisjunction[T, T] = { + // p <--> q can be modelled as (p and q) or (not(p) and not(q)) + PairConjunction[T, U](this, q) or PairConjunction[T, U](this.negate, q.negate) + } + def <==>[U](q: Constraint[U]): PairDisjunction[T, T] = ifAndOnlyIf(q) + + def negate: Constraint[T] + def unary_! = negate +} + +// zero-th order constraints +sealed trait PropositionalConstraint[T] extends Constraint[T] { + def estimator: LBJLearnerEquivalent +} + +case class InstanceWrapper[T](instance: T, estimator: LBJLearnerEquivalent) + +case class PropositionalEqualityConstraint[T]( + estimator: LBJLearnerEquivalent, + instanceOpt: Option[T], + equalityValOpt: Option[String], + inequalityValOpt: Option[String] +) extends PropositionalConstraint[T] { + def is(targetValue: String): PropositionalEqualityConstraint[T] = new PropositionalEqualityConstraint[T]( + estimator, instanceOpt, Some(targetValue), None + ) + def isTrue = is("true") + def isFalse = is("false") + def isNot(targetValue: String): PropositionalEqualityConstraint[T] = new PropositionalEqualityConstraint[T]( + estimator, instanceOpt, None, Some(targetValue) + ) + def negate: Constraint[T] = { + if (equalityValOpt.isDefined) { + new PropositionalEqualityConstraint[T](estimator, instanceOpt, None, equalityValOpt) + } else { + new PropositionalEqualityConstraint[T](estimator, instanceOpt, inequalityValOpt, None) + } + } + def isOneOf(values: Traversable[String]): AtLeast[T, T] = { + val equalityConst = values.map { v => + new PropositionalEqualityConstraint[T]( + estimator, instanceOpt, Some(v), None + ) + } + new AtLeast[T, T](equalityConst.toSet, 1) + } + + def isOneOf(values: String*): AtLeast[T, T] = { + isOneOf(values.toArray) + } +} + +// the two estimators should have the same prediction on the given instance +case class EstimatorPairEqualityConstraint[T]( + estimator1: LBJLearnerEquivalent, + estimator2Opt: Option[LBJLearnerEquivalent], + instance: T, + equalsOpt: Option[Boolean] +) extends PropositionalConstraint[T] { + def equalsTo(estimator2: LBJLearnerEquivalent) = new EstimatorPairEqualityConstraint[T]( + this.estimator1, + Some(estimator2), + this.instance, + Some(true) + ) + def differentFrom(estimator2: LBJLearnerEquivalent) = new EstimatorPairEqualityConstraint[T]( + this.estimator1, + Some(estimator2), + this.instance, + Some(false) + ) + override def estimator: LBJLearnerEquivalent = ??? + override def negate: Constraint[T] = EstimatorPairEqualityConstraint( + estimator1, estimator2Opt, instance, Some(!equalsOpt.get) + ) +} + +case class InstancePairEqualityConstraint[T]( + estimator: LBJLearnerEquivalent, + instance1: T, + instance2Opt: Option[T], + equalsOpt: Option[Boolean] +) extends PropositionalConstraint[T] { + def equalsTo(instance2: T) = new InstancePairEqualityConstraint[T]( + this.estimator, + this.instance1, + Some(instance2), + Some(true) + ) + def differentFrom(instance2: T) = new InstancePairEqualityConstraint[T]( + this.estimator, + this.instance1, + Some(instance2), + Some(false) + ) + override def negate: Constraint[T] = InstancePairEqualityConstraint( + estimator, instance1, instance2Opt, Some(!equalsOpt.get) + ) +} + +case class PairConjunction[T, U](c1: Constraint[T], c2: Constraint[U]) extends Constraint[T] { + def negate: Constraint[T] = new PairDisjunction[T, U](c1.negate, c2.negate) +} + +case class PairDisjunction[T, U](c1: Constraint[T], c2: Constraint[U]) extends Constraint[T] { + def negate: Constraint[T] = new PairConjunction[T, U](c1.negate, c2.negate) +} + +sealed trait ConstraintCollections[T, U] extends Constraint[T] { + val constraints: Set[Constraint[U]] +} + +case class ForAll[T, U](constraints: Set[Constraint[U]]) extends ConstraintCollections[T, U] { + def negate: Constraint[T] = new ForAll[T, U](constraints.map(_.negate)) +} + +case class AtLeast[T, U](constraints: Set[Constraint[U]], k: Int) extends ConstraintCollections[T, U] { + def negate: Constraint[T] = new AtMost[T, U](constraints, k) +} + +case class AtMost[T, U](constraints: Set[Constraint[U]], k: Int) extends ConstraintCollections[T, U] { + def negate: Constraint[T] = new AtLeast[T, U](constraints, k) +} + +case class Exactly[T, U](constraints: Set[Constraint[U]], k: Int) extends ConstraintCollections[T, U] { + def negate: Constraint[T] = new Exactly[T, U](constraints.map(_.negate), k) +} + +case class Implication[T, U](p: Constraint[T], q: Constraint[U]) extends Constraint[T] { + def negate: Constraint[T] = Implication[T, U](p, q.negate) +} + +case class Negation[T](p: Constraint[T]) extends Constraint[T] { + // negation of negation + def negate: Constraint[T] = p +} diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/InferenceCondition.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/InferenceCondition.scala deleted file mode 100644 index 26d7c7c5..00000000 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/InferenceCondition.scala +++ /dev/null @@ -1,34 +0,0 @@ -/** This software is released under the University of Illinois/Research and Academic Use License. See - * the LICENSE file in the root folder for details. Copyright (c) 2016 - * - * Developed by: The Cognitive Computations Group, University of Illinois at Urbana-Champaign - * http://cogcomp.cs.illinois.edu/ - */ -package edu.illinois.cs.cogcomp.saul.classifier.infer - -import edu.illinois.cs.cogcomp.infer.ilp.ILPSolver -import edu.illinois.cs.cogcomp.lbjava.infer.ParameterizedConstraint -import edu.illinois.cs.cogcomp.saul.constraint.LfsConstraint - -import scala.reflect.ClassTag - -abstract class InferenceCondition[INPUT <: AnyRef, HEAD <: AnyRef](solver: ILPSolver) { - def subjectTo: LfsConstraint[HEAD] - - def transfer(t: HEAD): JointTemplate[HEAD] = { - new JointTemplate[HEAD](t, solver) { - // TODO: Define this function - override def getSubjectToInstance: ParameterizedConstraint = { - subjectTo.transfer - } - // TODO: override other functions that needed here - } - } - - def apply(head: HEAD): JointTemplate[HEAD] = { - this.transfer(head) - } - - val outer = this - def convertToType[T <: AnyRef]: InferenceCondition[T, HEAD] = this.asInstanceOf[InferenceCondition[T, HEAD]] -} diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/InferenceManager.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/InferenceManager.scala new file mode 100644 index 00000000..c5aeafea --- /dev/null +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/InferenceManager.scala @@ -0,0 +1,382 @@ +/** This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computations Group, University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.saul.classifier.infer + +import edu.illinois.cs.cogcomp.infer.ilp.ILPSolver +import edu.illinois.cs.cogcomp.saul.lbjrelated.LBJLearnerEquivalent + +import scala.collection._ +import scala.reflect.ClassTag + +class InferenceManager { + // a small number used in creation of exclusive inequalities + private val epsilon = 0.01 + + // for each estimator, maps the label of the estimator, to the integer label of the solver + val estimatorToSolverLabelMap = mutable.Map[LBJLearnerEquivalent, mutable.Map[_, Seq[(Int, String)]]]() + + // greater or equal to: ax >= b + case class ILPInequalityGEQ(a: Array[Double], x: Array[Int], b: Double) + + def processConstraints[V <: Any](saulConstraint: Constraint[V], solver: ILPSolver): Set[ILPInequalityGEQ] = { + + saulConstraint match { + case c: PropositionalEqualityConstraint[V] => + assert(c.instanceOpt.isDefined, "the instance in the constraint should definitely be defined.") + + // add the missing variables to the map + addVariablesToInferenceProblem(Seq(c.instanceOpt.get), c.estimator, solver) + + // estimates per instance + val estimatorScoresMap = estimatorToSolverLabelMap(c.estimator).asInstanceOf[mutable.Map[V, Seq[(Int, String)]]] + + val (ilpIndices, labels) = estimatorScoresMap.get(c.instanceOpt.get) match { + case Some(ilpIndexLabelPairs) => ilpIndexLabelPairs.unzip + case None => + val confidenceScores = c.estimator.classifier.scores(c).toArray.map(_.score) + val labels = c.estimator.classifier.scores(c.instanceOpt.get).toArray.map(_.value) + val indicesPerLabels = solver.addDiscreteVariable(confidenceScores) + estimatorScoresMap += (c.instanceOpt.get -> indicesPerLabels.zip(labels)) + estimatorToSolverLabelMap.put(c.estimator, estimatorScoresMap) + (indicesPerLabels.toSeq, labels.toSeq) + } + + assert( + c.inequalityValOpt.isEmpty || c.equalityValOpt.isEmpty, + s"the equality constraint $c is not completely defined" + ) + assert( + c.inequalityValOpt.isDefined || c.equalityValOpt.isDefined, + s"the equality constraint $c has values for both equality and inequality" + ) + + val classifierTagSet = if (c.estimator.classifier.getLabeler != null) { + (0 until c.estimator.classifier.getLabelLexicon.size).map { i => + c.estimator.classifier.getLabelLexicon.lookupKey(i).getStringValue + }.toSet + } else { + c.estimator.classifier.allowableValues().toSet + } + + if (c.equalityValOpt.isDefined) { + // first make sure the target value is valid + require( + classifierTagSet.contains(c.equalityValOpt.get), + s"The target value ${c.equalityValOpt} is not a valid value for classifier ${c.estimator} - " + + s"the classifier tag-set is $classifierTagSet" + ) + val labelIndexOpt = labels.zipWithIndex.collectFirst { case (label, idx) if label == c.equalityValOpt.get => idx } + val x = ilpIndices(labelIndexOpt.getOrElse( + throw new Exception(s"the corresponding index to label ${c.equalityValOpt.get} not found") + )) + + // 1x == 1; which can be written as + // (a) +1x >= +1 + // (b) -1x >= -1 + // ILPInequalityGEQ(Array(-1.0), Array(x), -1.0) + Set(ILPInequalityGEQ(Array(1.0), Array(x), 1.0)) //, ILPInequalityGEQ(Array(-1.0), Array(x), -1.0)) + } else { + require( + classifierTagSet.contains(c.inequalityValOpt.get), + s"The target value ${c.inequalityValOpt} is not a valid value for classifier ${c.estimator} " + + s"with the tag-set: $classifierTagSet" + ) + val labelIndexOpt = labels.zipWithIndex.collectFirst { case (label, idx) if label == c.inequalityValOpt.get => idx } + val x = ilpIndices(labelIndexOpt.getOrElse( + throw new Exception(s"the corresponding index to label ${c.equalityValOpt.get} not found") + )) + // 1 x == 0 : possible only when x = 0, which can be written as + // (a) +1 x >= 0 + // (b) -1 x >= 0 + Set(ILPInequalityGEQ(Array(-1.0), Array(x), 0.0)) //, ILPInequalityGEQ(Array(1.0), Array(x), 0.0)) + } + case c: EstimatorPairEqualityConstraint[V] => + assert(c.estimator2Opt.isDefined, "the second estimator is not defined for estimator-pair constraint. Weird . . . ") + + // add the missing variables to the map + addVariablesToInferenceProblem(Seq(c.instance), c.estimator1, solver) + addVariablesToInferenceProblem(Seq(c.instance), c.estimator2Opt.get, solver) + + // estimates per instance + val estimatorScoresMap1 = estimatorToSolverLabelMap(c.estimator1).asInstanceOf[mutable.Map[V, Seq[(Int, String)]]] + val estimatorScoresMap2 = estimatorToSolverLabelMap(c.estimator2Opt.get).asInstanceOf[mutable.Map[V, Seq[(Int, String)]]] + + val labelToIndices1 = estimatorScoresMap1.get(c.instance) match { + case Some(ilpIndexLabelPairs) => ilpIndexLabelPairs.map(_.swap).toMap + case None => throw new Exception("The instance hasn't been seen??") + } + + val labelToIndices2 = estimatorScoresMap2.get(c.instance) match { + case Some(ilpIndexLabelPairs) => ilpIndexLabelPairs.map(_.swap).toMap + case None => throw new Exception("The instance hasn't been seen??") + } + + // this requirement might be an overkill, but keeping it for now. + require( + labelToIndices1.keySet == labelToIndices2.keySet, + "the label set for the two classifiers is not the same" + ) + + val yAll = solver.addDiscreteVariable(Array.fill(labelToIndices1.keySet.size) { 0 }) + + val labels = labelToIndices1.keySet.toSeq + labels.indices.flatMap { idx => + val label = labels(idx) + val y = yAll(idx) + val variable1 = labelToIndices1(label) + val variable2 = labelToIndices2(label) + + if (c.equalsOpt.get) { + // for each variable, if y is active, that variable should also be active: + // 1 variable >= 1 y + Set( + ILPInequalityGEQ(Array(1.0, -1.0), Array(variable1, y), 0.0), + ILPInequalityGEQ(Array(1.0, -1.0), Array(variable2, y), 0.0) + ) + } else { + // for each variable, if y is active, that variable should also be active: + // variable >= y which is, variable - y >= 0 + // 1 - variable >= 1 y which is, -variable -y >= -1 + Set( + ILPInequalityGEQ(Array(1.0, -1.0), Array(variable1, y), 0.0), + ILPInequalityGEQ(Array(-1.0, -1.0), Array(variable1, y), -1.0), + ILPInequalityGEQ(Array(1.0, -1.0), Array(variable2, y), 0.0), + ILPInequalityGEQ(Array(-1.0, -1.0), Array(variable2, y), -1.0) + ) + } + }.toSet + + case c: InstancePairEqualityConstraint[V] => + assert(c.instance2Opt.isDefined, "the second instance is not defined for estimator-pair constraint. Weird . . . ") + + // add the missing variables to the map + addVariablesToInferenceProblem(Seq(c.instance1), c.estimator, solver) + addVariablesToInferenceProblem(Seq(c.instance2Opt.get), c.estimator, solver) + + // estimates per instance + val estimatorScoresMap = estimatorToSolverLabelMap(c.estimator).asInstanceOf[mutable.Map[V, Seq[(Int, String)]]] + + val labelToIndices1 = estimatorScoresMap.get(c.instance1) match { + case Some(ilpIndexLabelPairs) => ilpIndexLabelPairs.map(_.swap).toMap + case None => throw new Exception("The instance hasn't been seen??") + } + + val labelToIndices2 = estimatorScoresMap.get(c.instance2Opt.get) match { + case Some(ilpIndexLabelPairs) => ilpIndexLabelPairs.map(_.swap).toMap + case None => throw new Exception("The instance hasn't been seen??") + } + + // this requirement might be an overkill, but keeping it for now. + require( + labelToIndices1.keySet == labelToIndices2.keySet, + "the label set for the two classifiers is not the same; " + + "although they belong to the same classifier; weird . . . " + ) + + val yAll = solver.addDiscreteVariable(Array.fill(labelToIndices1.keySet.size) { 0 }) + + val labels = labelToIndices1.keySet.toSeq + labels.indices.flatMap { idx => + val label = labels(idx) + val y = yAll(idx) + val variable1 = labelToIndices1(label) + val variable2 = labelToIndices2(label) + + if (c.equalsOpt.get) { + // for each variable, if y is active, that variable should also be active: + // 1 variable >= 1 y + Set( + ILPInequalityGEQ(Array(1.0, -1.0), Array(variable1, y), 0.0), + ILPInequalityGEQ(Array(1.0, -1.0), Array(variable2, y), 0.0) + ) + } else { + // for each variable, if y is active, that variable should also be active: + // variable >= y which is, variable - y >= 0 + // 1 - variable >= 1 y which is, -variable -y >= -1 + Set( + ILPInequalityGEQ(Array(1.0, -1.0), Array(variable1, y), 0.0), + ILPInequalityGEQ(Array(-1.0, -1.0), Array(variable1, y), -1.0), + ILPInequalityGEQ(Array(1.0, -1.0), Array(variable2, y), 0.0), + ILPInequalityGEQ(Array(-1.0, -1.0), Array(variable2, y), -1.0) + ) + } + }.toSet + + case c: PairConjunction[V, _] => + val InequalitySystem1 = processConstraints(c.c1, solver) + val InequalitySystem2 = processConstraints(c.c2, solver) + + // conjunction is simple; you just include all the inequalities + InequalitySystem1 union InequalitySystem2 + case c: PairDisjunction[V, _] => + val InequalitySystem1 = processConstraints(c.c1, solver) + val InequalitySystem2 = processConstraints(c.c2, solver) + val y1 = solver.addBooleanVariable(0.0) + val y2 = solver.addBooleanVariable(0.0) + // a1.x >= b1 or a2.x >= b2: + // should be converted to + // a1.x >= b1.y1 + min(a1.x).(1-y1) + // a2.x >= b2.(1-y2) + min(a2.x).y2 + // y1 + y2 >= 1 + // We can summarize the first one as: + // newA1 = [a1, min(a1.x)-b1] + // newX1 = [x, y] + // newB1 = min(a1.x) + val InequalitySystem1New = InequalitySystem1.map { ins => + val minValue = (ins.a.filter(_ < 0) :+ 0.0).sum + val a1New = ins.a :+ (minValue - ins.b) + val x1New = ins.x :+ y1 + val b1New = minValue + ILPInequalityGEQ(a1New, x1New, b1New) + } + val InequalitySystem2New = InequalitySystem2.map { ins => + val minValue = (ins.a.filter(_ < 0) :+ 0.0).sum + val a2New = ins.a :+ (minValue - ins.b) + val x2New = ins.x :+ y2 + val b2New = minValue + ILPInequalityGEQ(a2New, x2New, b2New) + } + val atLeastOne = ILPInequalityGEQ(Array(1, 1), Array(y1, y2), 1.0) + InequalitySystem1New union InequalitySystem2New + atLeastOne + case c: Negation[V] => + // change the signs of the coefficients + val InequalitySystemToBeNegated = processConstraints(c.p, solver) + InequalitySystemToBeNegated.map { in => + val minusA = in.a.map(-_) + val minusB = -in.b + epsilon + ILPInequalityGEQ(minusA, in.x, minusB) + } + case c: AtLeast[V, _] => + val InequalitySystems = c.constraints.map { processConstraints(_, solver) } + // for each inequality ax >= b we introduce a binary variable y + // and convert the constraint to ax >= by + (1-y).min(ax) and ax < (b-e)(1-y) + y.max(ax) + // which can be written as ax+y(min(ax)-b) >= min(ax) and ax + y(b - e - max(ax)) < b - e + // newA1 = [a, min(ax) - b] + // newX1 = [x, y] + // newB1 = min(ax) + // newA2 = [-a, max(ax) - b] + // newX2 = [x, y] + // newB2 = -b + epsilon + val (inequalities, newAuxillaryVariables) = InequalitySystems.map { inequalitySystem => + val y = solver.addBooleanVariable(0.0) + val newInequalities = inequalitySystem.flatMap { inequality => + val maxValue = (inequality.a.filter(_ > 0) :+ 0.0).sum + val minValue = (inequality.a.filter(_ < 0) :+ 0.0).sum + val newA1 = inequality.a :+ (minValue - inequality.b) + val newX1 = inequality.x :+ y + val newB1 = minValue + val newA2 = inequality.a.map(-_) :+ (maxValue + epsilon - inequality.b) + val newX2 = inequality.x :+ y + val newB2 = -inequality.b + epsilon + Set(ILPInequalityGEQ(newA1, newX1, newB1), ILPInequalityGEQ(newA2, newX2, newB2)) + } + (newInequalities, y) + }.unzip + // add a new constraint: at least k constraints should be active + inequalities.flatten + + ILPInequalityGEQ(newAuxillaryVariables.toArray.map(_ => 1.0), newAuxillaryVariables.toArray, c.k) + case c: AtMost[V, _] => + val InequalitySystems = c.constraints.map { processConstraints(_, solver) } + // for each inequality ax >= b we introduce a binary variable y + // and convert the constraint to ax >= by + (1-y).min(ax) and ax < (b-e)(1-y) + y.max(ax) + // which can be written as ax+y(min(ax)-b) >= min(ax) and ax + y(b - e - max(ax)) < b - e + // newA1 = [a, min(ax) - b] + // newX1 = [x, y] + // newB1 = min(ax) + // newA2 = [-a, max(ax) - b] + // newX2 = [x, y] + // newB2 = -b + epsilon + val (inequalities, newAuxillaryVariables) = InequalitySystems.map { inequalitySystem => + val y = solver.addBooleanVariable(0.0) + val newInequalities = inequalitySystem.flatMap { inequality => + val maxValue = (inequality.a.filter(_ > 0) :+ 0.0).sum + val minValue = (inequality.a.filter(_ < 0) :+ 0.0).sum + val newA1 = inequality.a :+ (minValue - inequality.b) + val newX1 = inequality.x :+ y + val newB1 = minValue + val newA2 = inequality.a.map(-_) :+ (maxValue + epsilon - inequality.b) + val newX2 = inequality.x :+ y + val newB2 = -inequality.b + epsilon + Set(ILPInequalityGEQ(newA1, newX1, newB1), ILPInequalityGEQ(newA2, newX2, newB2)) + } + (newInequalities, y) + }.unzip + // add a new constraint: at least k constraints should be active + inequalities.flatten + + ILPInequalityGEQ(newAuxillaryVariables.toArray.map(_ => -1.0), newAuxillaryVariables.toArray, -c.k) + case c: Exactly[V, _] => + val InequalitySystems = c.constraints.map { processConstraints(_, solver) } + // for each inequality ax >= b we introduce a binary variable y + // and convert the constraint to ax >= by + (1-y).min(ax) and ax < (b-e)(1-y) + y.max(ax) + // which can be written as ax+y(min(ax)-b) >= min(ax) and ax + y(b - e - max(ax)) < b - e + // newA1 = [a, min(ax) - b] + // newX1 = [x, y] + // newB1 = min(ax) + // newA2 = [-a, max(ax) - b] + // newX2 = [x, y] + // newB2 = -b + epsilon + val (inequalities, newAuxillaryVariables) = InequalitySystems.map { inequalitySystem => + val y = solver.addBooleanVariable(0.0) + val newInequalities = inequalitySystem.flatMap { inequality => + val maxValue = (inequality.a.filter(_ > 0) :+ 0.0).sum + val minValue = (inequality.a.filter(_ < 0) :+ 0.0).sum + val newA1 = inequality.a :+ (minValue - inequality.b) + val newX1 = inequality.x :+ y + val newB1 = minValue + val newA2 = inequality.a.map(-_) :+ (maxValue + epsilon - inequality.b) + val newX2 = inequality.x :+ y + val newB2 = -inequality.b + epsilon + Set(ILPInequalityGEQ(newA1, newX1, newB1), ILPInequalityGEQ(newA2, newX2, newB2)) + } + (newInequalities, y) + }.unzip + // add a new constraint: at least k constraints should be active + inequalities.flatten union Set( + ILPInequalityGEQ(newAuxillaryVariables.toArray.map(_ => 1.0), newAuxillaryVariables.toArray, c.k), + ILPInequalityGEQ(newAuxillaryVariables.toArray.map(_ => -1.0), newAuxillaryVariables.toArray, -c.k) + ) + case c: ForAll[V, _] => + c.constraints.flatMap { processConstraints(_, solver) } + case _ => + throw new Exception("Saul implication is converted to other operations. ") + } + } + + // if the estimator has never been seen before, add its labels to the map + def createEstimatorSpecificCache[V](estimator: LBJLearnerEquivalent): Unit = { + if (!estimatorToSolverLabelMap.keySet.contains(estimator)) { + estimatorToSolverLabelMap += (estimator -> mutable.Map[V, Seq[(Int, String)]]()) + } + } + + def addVariablesToInferenceProblem[V](instances: Seq[V], estimator: LBJLearnerEquivalent, solver: ILPSolver): Unit = { + createEstimatorSpecificCache(estimator) + + // estimates per instance + val estimatorScoresMap = estimatorToSolverLabelMap(estimator).asInstanceOf[mutable.Map[V, Seq[(Int, String)]]] + + // adding the estimates to the solver and to the map + instances.foreach { c => + val confidenceScores = estimator.classifier.scores(c).toArray.map(_.score) + val labels = estimator.classifier.scores(c).toArray.map(_.value) + val instanceIndexPerLabel = solver.addDiscreteVariable(confidenceScores) + if (!estimatorScoresMap.contains(c)) { + estimatorScoresMap += (c -> instanceIndexPerLabel.zip(labels).toSeq) + } + } + + // add the variables back into the map + estimatorToSolverLabelMap.put(estimator, estimatorScoresMap) + } +} + +object InferenceManager { + /** Contains cache of problems already solved. The key is the head object, which maps to instances and their + * predicted values in the output of inference + */ + val cachedResults = mutable.Map[String, (ILPSolver, LBJLearnerEquivalent, mutable.Map[LBJLearnerEquivalent, mutable.Map[_, Seq[(Int, String)]]])]() +} diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/InitSparseNetwork.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/InitSparseNetwork.scala index d1f738fe..ee3afa8c 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/InitSparseNetwork.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/infer/InitSparseNetwork.scala @@ -7,11 +7,8 @@ package edu.illinois.cs.cogcomp.saul.classifier.infer import edu.illinois.cs.cogcomp.lbjava.learn.{ LinearThresholdUnit, SparseNetworkLearner } -import edu.illinois.cs.cogcomp.saul.classifier.ConstrainedClassifier import edu.illinois.cs.cogcomp.saul.datamodel.node.Node -/** Created by Parisa on 9/18/16. - */ object InitSparseNetwork { def apply[HEAD <: AnyRef](node: Node[HEAD], cClassifier: ConstrainedClassifier[_, HEAD]) = { val allHeads = node.getTrainingInstances diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/constraint/Constraint.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/constraint/Constraint.scala deleted file mode 100755 index 9dbddb94..00000000 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/constraint/Constraint.scala +++ /dev/null @@ -1,137 +0,0 @@ -/** This software is released under the University of Illinois/Research and Academic Use License. See - * the LICENSE file in the root folder for details. Copyright (c) 2016 - * - * Developed by: The Cognitive Computations Group, University of Illinois at Urbana-Champaign - * http://cogcomp.cs.illinois.edu/ - */ -package edu.illinois.cs.cogcomp.saul.constraint - -import edu.illinois.cs.cogcomp.lbjava.infer._ -import edu.illinois.cs.cogcomp.lbjava.learn.Learner -import edu.illinois.cs.cogcomp.saul.lbjrelated.LBJLearnerEquivalent - -import scala.language.implicitConversions - -/** We need to define the language of constraints here to work with the first order constraints that are programmed in - * our main LBP script. The wrapper just gives us a java [[FirstOrderConstraint]] object in the shell of an scala - * object in this way our language works on scala objects. - */ -object ConstraintTypeConversion { - implicit def learnerToLFS(l: Learner): LBJLearnerEquivalent = { - new LBJLearnerEquivalent { - override val classifier = l - } - } - - implicit def constraintWrapper(p: FirstOrderConstraint): FirstOrderConstraints = { - new FirstOrderConstraints(p) - } - - implicit def javaCollToMyQuantifierWrapper[T](coll: java.util.Collection[T]): QuantifierWrapper[T] = { - import scala.collection.JavaConversions._ - new QuantifierWrapper[T](coll.toSeq) - } - - implicit def scalaCollToMyQuantifierWrapper[T](coll: Seq[T]): QuantifierWrapper[T] = { - new QuantifierWrapper[T](coll) - } -} - -class QuantifierWrapper[T](val coll: Seq[T]) { - def _exists(p: T => FirstOrderConstraint): FirstOrderConstraint = { - val alwaysFalse: FirstOrderConstraint = new FirstOrderConstant(false) - def makeDisjunction(c1: FirstOrderConstraint, c2: FirstOrderConstraint): FirstOrderConstraint = { - new FirstOrderDisjunction(c1, c2) - } - coll.map(p).foldLeft[FirstOrderConstraint](alwaysFalse)(makeDisjunction) - } - - def _forall(p: T => FirstOrderConstraint): FirstOrderConstraint = { - val alwaysTrue: FirstOrderConstraint = new FirstOrderConstant(true) - def makeConjunction(c1: FirstOrderConstraint, c2: FirstOrderConstraint): FirstOrderConstraint = { - new FirstOrderConjunction(c1, c2) - } - coll.map(p).foldLeft[FirstOrderConstraint](alwaysTrue)(makeConjunction) - } - - /** transfer the constraint to a constant - * These functions can be slow, if not used properly - * The best performance is when n is too big (close to the size of the collection) or too small - */ - def _atmost(n: Int)(p: T => FirstOrderConstraint): FirstOrderConstraint = { - val constraintCombinations = coll.map(p).combinations(n + 1) - val listOfConjunctions = for { - constraints <- constraintCombinations - dummyConstraint = new FirstOrderConstant(true) - } yield constraints.foldLeft[FirstOrderConstraint](dummyConstraint)(new FirstOrderConjunction(_, _)) - - val dummyConstraint = new FirstOrderConstant(false) - new FirstOrderNegation(listOfConjunctions.toList.foldLeft[FirstOrderConstraint](dummyConstraint)(new FirstOrderDisjunction(_, _))) - } - - /** transfer the constraint to a constant - * These functions can be slow, if not used properly - * The best performance is when n is too big (close to the size of the collection) or too small - */ - def _atleast(n: Int)(p: T => FirstOrderConstraint): FirstOrderConstraint = { - val constraintCombinations = coll.map(p).combinations(n) - val listOfConjunctions = for { - constraints <- constraintCombinations - dummyConstraint = new FirstOrderConstant(true) - } yield constraints.foldLeft[FirstOrderConstraint](dummyConstraint)(new FirstOrderConjunction(_, _)) - - val dummyConstraint = new FirstOrderConstant(false) - listOfConjunctions.toList.foldLeft[FirstOrderConstraint](dummyConstraint)(new FirstOrderDisjunction(_, _)) - } -} - -class FirstOrderConstraints(val r: FirstOrderConstraint) { - - def ==>(other: FirstOrderConstraint) = new FirstOrderImplication(this.r, other) - - def <==>(other: FirstOrderConstraint) = new FirstOrderDoubleImplication(this.r, other) - - def unary_! = new FirstOrderNegation(this.r) - - def and(other: FirstOrderConstraint) = new FirstOrderConjunction(this.r, other) - - def or(other: FirstOrderConstraint) = new FirstOrderDisjunction(this.r, other) - -} - -class LHSFirstOrderEqualityWithValueLBP(cls: Learner, t: AnyRef) { - - // probably we need to write here - // LHSFirstOrderEqualityWithValueLBP(cls : Learner, t : AnyRef) extends ConstraintTrait - - val lbjRepr = new FirstOrderVariable(cls, t) - - def is(v: String): FirstOrderConstraint = { - new FirstOrderEqualityWithValue(true, lbjRepr, v) - } - - //TODO: not sure if this works correctly. Make sure it works. - def is(v: LHSFirstOrderEqualityWithValueLBP): FirstOrderConstraint = { - new FirstOrderEqualityWithVariable(true, lbjRepr, v.lbjRepr) - } - - def isTrue: FirstOrderConstraint = is("true") - - def isNotTrue: FirstOrderConstraint = is("false") - - def isNot(v: String): FirstOrderConstraint = { - new FirstOrderNegation(new FirstOrderEqualityWithValue(true, lbjRepr, v)) - } - - def isNot(v: LHSFirstOrderEqualityWithValueLBP): FirstOrderConstraint = { - new FirstOrderNegation(new FirstOrderEqualityWithVariable(true, lbjRepr, v.lbjRepr)) - } - - def in(v: Array[String]): FirstOrderConstraint = { - val falseConstant = new FirstOrderDisjunction(new FirstOrderEqualityWithValue(true, lbjRepr, v(0)), new FirstOrderConstant(false)) - v.tail.foldRight(falseConstant) { - (value, newConstraint) => - new FirstOrderDisjunction(new FirstOrderEqualityWithValue(true, lbjRepr, value), newConstraint) - } - } -} diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/constraint/LfsConstraint.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/constraint/LfsConstraint.scala deleted file mode 100644 index 0a85942e..00000000 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/constraint/LfsConstraint.scala +++ /dev/null @@ -1,48 +0,0 @@ -/** This software is released under the University of Illinois/Research and Academic Use License. See - * the LICENSE file in the root folder for details. Copyright (c) 2016 - * - * Developed by: The Cognitive Computations Group, University of Illinois at Urbana-Champaign - * http://cogcomp.cs.illinois.edu/ - */ -package edu.illinois.cs.cogcomp.saul.constraint - -import edu.illinois.cs.cogcomp.infer.ilp.ILPSolver -import edu.illinois.cs.cogcomp.lbjava.infer.{ ParameterizedConstraint, FirstOrderConstraint } -import edu.illinois.cs.cogcomp.saul.classifier.infer.InferenceCondition - -import scala.reflect.ClassTag - -abstract class LfsConstraint[T <: AnyRef] { - - def makeConstrainDef(x: T): FirstOrderConstraint - - def evalDiscreteValue(t: T): String = { - this.makeConstrainDef(t).evaluate().toString - } - - def apply(t: T) = makeConstrainDef(t) - - def transfer: ParameterizedConstraint = { - new ParameterizedConstraint() { - override def makeConstraint(__example: AnyRef): FirstOrderConstraint = { - val t: T = __example.asInstanceOf[T] - makeConstrainDef(t) - } - - override def discreteValue(__example: AnyRef): String = - { - val t: T = __example.asInstanceOf[T] - evalDiscreteValue(t) - //Todo type check error catch - } - } - } - - val lc = this - - def createInferenceCondition[C <: AnyRef](solver: ILPSolver): InferenceCondition[C, T] = { - new InferenceCondition[C, T](solver) { - override def subjectTo: LfsConstraint[T] = lc - } - } -} diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/lbjrelated/LBJLearnerEquivalent.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/lbjrelated/LBJLearnerEquivalent.scala index 2ab048a6..b58a98ea 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/lbjrelated/LBJLearnerEquivalent.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/lbjrelated/LBJLearnerEquivalent.scala @@ -8,7 +8,6 @@ package edu.illinois.cs.cogcomp.saul.lbjrelated import edu.illinois.cs.cogcomp.lbjava.classify.Classifier import edu.illinois.cs.cogcomp.lbjava.learn.{ Lexicon, Learner } -import edu.illinois.cs.cogcomp.saul.constraint.LHSFirstOrderEqualityWithValueLBP /** Encapsulates an instance of LBJava's [[Learner]] class. */ @@ -16,8 +15,6 @@ trait LBJLearnerEquivalent extends LBJClassifierEquivalent { val classifier: Learner - def on(t: AnyRef): LHSFirstOrderEqualityWithValueLBP = new LHSFirstOrderEqualityWithValueLBP(this.classifier, t) - def getLabeler: Classifier = classifier.getLabeler def getExampleArray(example: Any): Array[AnyRef] = classifier.getExampleArray(example) diff --git a/saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/classifier/JoinTrainingTests/IntializeSparseNetwork.scala b/saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/classifier/JoinTrainingTests/InitializeSparseNetwork.scala similarity index 83% rename from saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/classifier/JoinTrainingTests/IntializeSparseNetwork.scala rename to saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/classifier/JoinTrainingTests/InitializeSparseNetwork.scala index cc284619..d8fce9b4 100644 --- a/saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/classifier/JoinTrainingTests/IntializeSparseNetwork.scala +++ b/saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/classifier/JoinTrainingTests/InitializeSparseNetwork.scala @@ -6,15 +6,12 @@ */ package edu.illinois.cs.cogcomp.saul.classifier.JoinTrainingTests -import edu.illinois.cs.cogcomp.infer.ilp.OJalgoHook -import edu.illinois.cs.cogcomp.lbjava.infer.FirstOrderConstant import edu.illinois.cs.cogcomp.lbjava.learn.{ LinearThresholdUnit, SparseNetworkLearner } -import edu.illinois.cs.cogcomp.saul.classifier.{ JointTrainSparseNetwork, ClassifierUtils, ConstrainedClassifier, Learnable } +import edu.illinois.cs.cogcomp.saul.classifier.infer.{ ConstrainedClassifier, OJAlgo } +import edu.illinois.cs.cogcomp.saul.classifier.{ ClassifierUtils, JointTrainSparseNetwork, Learnable } import edu.illinois.cs.cogcomp.saul.datamodel.DataModel import org.scalatest.{ FlatSpec, Matchers } -/** Created by Parisa on 9/18/16. - */ class InitializeSparseNetwork extends FlatSpec with Matchers { // Testing the original functions with real classifiers @@ -31,13 +28,16 @@ class InitializeSparseNetwork extends FlatSpec with Matchers { override def feature = using(word, biWord) override lazy val classifier = new SparseNetworkLearner() } - object TestConstraintClassifier extends ConstrainedClassifier[String, String](TestClassifier) { - def subjectTo = ConstrainedClassifier.constraint { _ => new FirstOrderConstant(true) } - override val solver = new OJalgoHook + object TestConstraintClassifier extends ConstrainedClassifier[String, String] { + override def subjectTo = None + override val solverType = OJAlgo + override lazy val onClassifier = TestClassifier } - object TestConstraintClassifierWithExtendedFeatures extends ConstrainedClassifier[String, String](TestClassifierWithExtendedFeatures) { - def subjectTo = ConstrainedClassifier.constraint { _ => new FirstOrderConstant(true) } - override val solver = new OJalgoHook + + object TestConstraintClassifierWithExtendedFeatures extends ConstrainedClassifier[String, String] { + override def subjectTo = None + override val solverType = OJAlgo + override lazy val onClassifier = TestClassifierWithExtendedFeatures } val words = List("this", "is", "a", "test", "candidate", ".") @@ -72,7 +72,7 @@ class InitializeSparseNetwork extends FlatSpec with Matchers { wv1.size() should be(0) wv2.size() should be(0) TestClassifierWithExtendedFeatures.learn(2) - JointTrainSparseNetwork.train(tokens, cls, 5, false) + JointTrainSparseNetwork.train(tokens, cls, 5, init = false) val wv1After = clNet1.getNetwork.get(0).asInstanceOf[LinearThresholdUnit].getWeightVector val wv2After = clNet2.getNetwork.get(0).asInstanceOf[LinearThresholdUnit].getWeightVector @@ -89,4 +89,3 @@ class InitializeSparseNetwork extends FlatSpec with Matchers { val biWord = property(tokens) { x: String => x + "-" + x } } } - diff --git a/saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/infer/InferenceTest.scala b/saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/infer/InferenceTest.scala new file mode 100644 index 00000000..4e465b62 --- /dev/null +++ b/saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/infer/InferenceTest.scala @@ -0,0 +1,439 @@ +/** This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computations Group, University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.saul.infer + +import java.io.PrintStream + +import edu.illinois.cs.cogcomp.lbjava.classify.{ FeatureVector, ScoreSet } +import edu.illinois.cs.cogcomp.lbjava.learn.Learner +import edu.illinois.cs.cogcomp.saul.classifier.infer.Constraint._ +import edu.illinois.cs.cogcomp.saul.classifier.infer.{ ConstrainedClassifier, Constraint, OJAlgo } +import edu.illinois.cs.cogcomp.saul.datamodel.DataModel +import edu.illinois.cs.cogcomp.saul.lbjrelated.LBJLearnerEquivalent +import org.scalatest.{ FlatSpec, Matchers } + +class StaticClassifier(trueLabelScore: Double) extends Learner("DummyClassifer") { + override def getInputType: String = { "DummyInstance" } + + override def allowableValues: Array[String] = { Array[String]("false", "true") } + + override def equals(o: Any): Boolean = { getClass == o.getClass } + + /** The reason for true to be -1 is because the internal optimization by default finds the maximizer, while in this + * problem we are looking for a minimizer + */ + override def scores(example: AnyRef): ScoreSet = { + val result: ScoreSet = new ScoreSet + result.put("false", 0) + result.put("true", trueLabelScore) + result + } + + override def write(printStream: PrintStream): Unit = ??? + + override def scores(ints: Array[Int], doubles: Array[Double]): ScoreSet = ??? + + override def classify(ints: Array[Int], doubles: Array[Double]): FeatureVector = ??? + + override def learn(ints: Array[Int], doubles: Array[Double], ints1: Array[Int], doubles1: Array[Double]): Unit = ??? +} + +case class Instance(value: Int) + +object DummyDataModel extends DataModel { + val instanceNode = node[Instance] + + // only used for implications + val instanceNode2 = node[Instance] + + /** definition of the constraints */ + val classifierPositiveScoreForTrue: LBJLearnerEquivalent = new LBJLearnerEquivalent { + override val classifier: Learner = new StaticClassifier(1.0) + } + val classifierNegativeScoreForTrue: LBJLearnerEquivalent = new LBJLearnerEquivalent { + override val classifier: Learner = new StaticClassifier(-1.0) + } + + def singleInstanceMustBeTrue(x: Instance) = { classifierNegativeScoreForTrue on x isTrue } + def singleInstanceMustBeFalse(x: Instance) = { classifierPositiveScoreForTrue on x isFalse } + def forAllTrue = instanceNode.ForAll { x: Instance => classifierPositiveScoreForTrue on x isTrue } + def forAllFalse = instanceNode.ForAll { x: Instance => classifierPositiveScoreForTrue on x isFalse } + def forAllOneOfTheLabelsPositiveClassifier = instanceNode.ForAll { x: Instance => classifierPositiveScoreForTrue on x isOneOf ("true", "true") } + def forAllOneOfTheLabelsNegativeClassifier = instanceNode.ForAll { x: Instance => classifierPositiveScoreForTrue on x isOneOf ("true", "true") } + def forAllNotFalse = instanceNode.ForAll { x: Instance => classifierPositiveScoreForTrue on x isNot "false" } + def forAllNotTrue = instanceNode.ForAll { x: Instance => classifierPositiveScoreForTrue on x isNot "true" } + def existsTrue = instanceNode.Exists { x: Instance => classifierNegativeScoreForTrue on x isTrue } + def existsFalse = instanceNode.Exists { x: Instance => classifierPositiveScoreForTrue on x isFalse } + def exatclyTrue(k: Int) = instanceNode.Exactly(k) { x: Instance => classifierPositiveScoreForTrue on x isTrue } + def exatclyFalse(k: Int) = instanceNode.Exactly(k) { x: Instance => classifierPositiveScoreForTrue on x isFalse } + def atLeastTrue(k: Int) = instanceNode.AtLeast(k) { x: Instance => classifierNegativeScoreForTrue on x isTrue } + def atLeastFalse(k: Int) = instanceNode.AtLeast(k) { x: Instance => classifierPositiveScoreForTrue on x isFalse } + def atMostTrue(k: Int) = instanceNode.AtMost(k) { x: Instance => classifierPositiveScoreForTrue on x isTrue } + def atMostFalse(k: Int) = instanceNode.AtMost(k) { x: Instance => classifierNegativeScoreForTrue on x isFalse } + def classifierHasSameValueOnTwoInstances(x: Instance, y: Instance) = classifierPositiveScoreForTrue on x equalsTo y + + // negation + def forAllFalseWithNegation = instanceNode.ForAll { x: Instance => !(classifierPositiveScoreForTrue on x isTrue) } + def forAllTrueNegated = !forAllTrue + def atLeastFalseNegated(k: Int) = !atLeastFalse(k) + + // conjunction + def allTrueAllTrueConjunction = forAllTrue and forAllTrue + def allTrueAllFalseConjunction = forAllTrue and forAllFalse + def allFalseAllTrueConjunction = forAllFalse and forAllTrue + def allFalseAllFalseConjunction = forAllFalse and forAllFalse + + // disjunction + def allTrueAllTrueDisjunction = forAllTrue or forAllTrue + def allTrueAllFalseDisjunction = forAllTrue or forAllFalse + def allFalseAllTrueDisjunction = forAllFalse or forAllTrue + def allFalseAllFalseDisjunction = forAllFalse or forAllFalse +} + +class DummyConstrainedInference(someConstraint: Some[Constraint[Instance]], classifier: LBJLearnerEquivalent) extends ConstrainedClassifier[Instance, Instance] { + override lazy val onClassifier = classifier + override def pathToHead = None + override def subjectTo = someConstraint + override def solverType = OJAlgo +} + +class InferenceTest extends FlatSpec with Matchers { + import DummyDataModel._ + + val instanceSet = (1 to 5).map(Instance) + DummyDataModel.instanceNode.populate(instanceSet) + + // extra constraints based on data + // all instances should have the same label + def classifierHasSameValueOnTwoInstancesInstantiated = { + classifierHasSameValueOnTwoInstances(instanceSet(0), instanceSet(1)) and + classifierHasSameValueOnTwoInstances(instanceSet(1), instanceSet(2)) and + classifierHasSameValueOnTwoInstances(instanceSet(2), instanceSet(3)) and + classifierHasSameValueOnTwoInstances(instanceSet(3), instanceSet(4)) + } + + def allInstancesShouldBeTrue = { + classifierHasSameValueOnTwoInstancesInstantiated and singleInstanceMustBeTrue(instanceSet(0)) + } + + def trueImpliesTrue = { + ((classifierNegativeScoreForTrue on instanceSet(0) isTrue) ==> + (classifierNegativeScoreForTrue on instanceSet(1) isTrue)) and (classifierNegativeScoreForTrue on instanceSet(0) isTrue) + } + + def trueImpliesFalse = { + ((classifierNegativeScoreForTrue on instanceSet(0) isTrue) ==> + (classifierNegativeScoreForTrue on instanceSet(1) isFalse)) and (classifierNegativeScoreForTrue on instanceSet(0) isTrue) + } + + def falseImpliesTrue = { + ((classifierNegativeScoreForTrue on instanceSet(0) isFalse) ==> + (classifierNegativeScoreForTrue on instanceSet(1) isTrue)) and (classifierNegativeScoreForTrue on instanceSet(0) isFalse) + } + + def falseImpliesFalse = { + ((classifierNegativeScoreForTrue on instanceSet(0) isFalse) ==> + (classifierNegativeScoreForTrue on instanceSet(1) isFalse)) and (classifierNegativeScoreForTrue on instanceSet(0) isFalse) + } + + def halfHalfConstraint(classifier: LBJLearnerEquivalent, firstHalfLabel: String, secondHalfLabel: String) = { + (0 to instanceSet.size / 2).map(i => classifier on instanceSet(i) is firstHalfLabel).ForAll and + ((instanceSet.size / 2 + 1) until instanceSet.size).map(i => classifier on instanceSet(i) is secondHalfLabel).ForAll + } + + def conjunctionOfDisjunction = { + (classifierPositiveScoreForTrue on instanceSet(0) isFalse) and ( + (classifierPositiveScoreForTrue on instanceSet(1) isFalse) or + (classifierPositiveScoreForTrue on instanceSet(2) isFalse) + ) + } + + def disjunctionOfConjunctions = { + (classifierPositiveScoreForTrue on instanceSet(0) isFalse) or ( + (classifierPositiveScoreForTrue on instanceSet(1) isFalse) and + (classifierPositiveScoreForTrue on instanceSet(2) isFalse) + ) + } + + def halfTrueHalfFalsePositiveClassifier = { + halfHalfConstraint(classifierPositiveScoreForTrue, "true", "false") or + halfHalfConstraint(classifierPositiveScoreForTrue, "false", "true") + } + + def halfTrueHalfFalseNegativeClassifier = { + halfHalfConstraint(classifierNegativeScoreForTrue, "true", "false") or + halfHalfConstraint(classifierNegativeScoreForTrue, "false", "true") + } + + // single instance constraint + "first instance " should "true and the rest should be false" in { + val singleInstanceMustBeTrueInference = new DummyConstrainedInference( + Some(singleInstanceMustBeTrue(instanceSet.head)), classifierNegativeScoreForTrue + ) + singleInstanceMustBeTrueInference(instanceSet.head) should be("true") + instanceSet.drop(1).foreach { ins => singleInstanceMustBeTrueInference(ins) should be("false") } + } + + // single instance constraint + "first instance " should "false and the rest should be true" in { + val singleInstanceMustBeFalseInference = new DummyConstrainedInference( + Some(singleInstanceMustBeFalse(instanceSet.head)), classifierPositiveScoreForTrue + ) + singleInstanceMustBeFalseInference(instanceSet.head) should be("false") + instanceSet.drop(1).foreach { ins => singleInstanceMustBeFalseInference(ins) should be("true") } + } + + // all true + "ForAllTrue " should " return all true instances" in { + val allTrueInference = new DummyConstrainedInference(Some(forAllTrue), classifierPositiveScoreForTrue) + instanceSet.foreach { ins => allTrueInference(ins) should be("true") } + } + + // all false + "ForAllFalse " should " return all false instances" in { + val allFalseInference = new DummyConstrainedInference(Some(forAllFalse), classifierPositiveScoreForTrue) + instanceSet.foreach { ins => allFalseInference(ins) should be("false") } + } + + // for all one of the labels + "OneOf(true, some label) with positive true weight " should " work properly " in { + val forAllOneOfTheLabelsPositiveClassifierInference = new DummyConstrainedInference( + Some(forAllOneOfTheLabelsPositiveClassifier), classifierPositiveScoreForTrue + ) + instanceSet.foreach { ins => forAllOneOfTheLabelsPositiveClassifierInference(ins) should be("true") } + } + + // for all one of the labels + "OneOf(true, some label) with negative true weight " should " work properly " in { + val forAllOneOfTheLabelsNegativeClassifierInference = new DummyConstrainedInference( + Some(forAllOneOfTheLabelsNegativeClassifier), classifierPositiveScoreForTrue + ) + instanceSet.foreach { ins => forAllOneOfTheLabelsNegativeClassifierInference(ins) should be("true") } + } + + // all not false, should always return true + "ForAllNotFalse " should " return all true instances" in { + val allNotFalseInference = new DummyConstrainedInference(Some(forAllNotFalse), classifierPositiveScoreForTrue) + instanceSet.foreach { ins => allNotFalseInference(ins) should be("true") } + } + + // all not true, should always return false + "ForAllNotTrue " should " return all false instances" in { + val allNotTrueInference = new DummyConstrainedInference(Some(forAllNotTrue), classifierPositiveScoreForTrue) + instanceSet.foreach { ins => allNotTrueInference(ins) should be("false") } + instanceSet.foreach { ins => info(allNotTrueInference(ins)) } + } + + // exists true + "ExistsTrue " should " return exactly one true when true weight is negative" in { + val existOneTrueInference = new DummyConstrainedInference(Some(existsTrue), classifierNegativeScoreForTrue) + instanceSet.count { ins => existOneTrueInference(ins) == "true" } should be(1) + } + + // exists false + "ExistsFalse " should " return exactly one false when true weight is positive" in { + val existOneFalseInference = new DummyConstrainedInference(Some(existsFalse), classifierPositiveScoreForTrue) + instanceSet.count { ins => existOneFalseInference(ins) == "false" } should be(1) + } + + // at least 2 true + "AtLeast2True " should " return at least two true instance" in { + val atLeastTwoTrueInference = new DummyConstrainedInference(Some(atLeastTrue(2)), classifierNegativeScoreForTrue) + instanceSet.count { ins => atLeastTwoTrueInference(ins) == "true" } should be(2) + } + + // at least 2 false + "AtLeast2False " should " return at least two false instance" in { + val atLeastTwoFalseInference = new DummyConstrainedInference(Some(atLeastFalse(2)), classifierPositiveScoreForTrue) + instanceSet.count { ins => atLeastTwoFalseInference(ins) == "false" } should be(2) + } + + // at least 3 true + "AtLeast3True " should " return at least three true instance" in { + val atLeastThreeTrueInference = new DummyConstrainedInference(Some(atLeastTrue(3)), classifierNegativeScoreForTrue) + instanceSet.count { ins => atLeastThreeTrueInference(ins) == "true" } should be(3) + } + + // at least 3 false + "AtLeast3False " should " return at least three false instance" in { + val atLeastThreeFalseInference = new DummyConstrainedInference(Some(atLeastFalse(3)), classifierPositiveScoreForTrue) + instanceSet.count { ins => atLeastThreeFalseInference(ins) == "false" } should be >= 3 + } + + // exactly 1 true + "ExactlyOneTrue " should " return exactly one true instance" in { + val exactlyOneTrue = new DummyConstrainedInference(Some(exatclyTrue(1)), classifierPositiveScoreForTrue) + instanceSet.count { ins => exactlyOneTrue(ins) == "true" } should be(1) + } + + // exactly 2 true + "ExactlyTwoTrue " should " return exactly two true instances" in { + val exactlyOneTrue = new DummyConstrainedInference(Some(exatclyTrue(2)), classifierPositiveScoreForTrue) + instanceSet.count { ins => exactlyOneTrue(ins) == "true" } should be(2) + } + + // exactly 3 true + "ExactlyTwoTrue " should " return exactly three true instances" in { + val exactlyOneTrue = new DummyConstrainedInference(Some(exatclyTrue(3)), classifierPositiveScoreForTrue) + instanceSet.count { ins => exactlyOneTrue(ins) == "true" } should be(3) + } + + // exactly 1 false + "ExactlyOneFalse " should " return exactly one true instances" in { + val exactlyOneFalse = new DummyConstrainedInference(Some(exatclyFalse(1)), classifierPositiveScoreForTrue) + instanceSet.count { ins => exactlyOneFalse(ins) == "false" } should be(1) + } + + // exactly 2 false + "ExactlyTwoFalse " should " return exactly two true instances" in { + val exactlyOneFalse = new DummyConstrainedInference(Some(exatclyFalse(2)), classifierPositiveScoreForTrue) + instanceSet.count { ins => exactlyOneFalse(ins) == "false" } should be(2) + } + + // exactly 3 false + "ExactlyTwoFalse " should " return exactly three true instances" in { + val exactlyOneFalse = new DummyConstrainedInference(Some(exatclyFalse(3)), classifierPositiveScoreForTrue) + instanceSet.count { ins => exactlyOneFalse(ins) == "false" } should be(3) + } + + // at most 2 true + "AtMost " should " return at most two true instances" in { + val atMostTwoTrueInference = new DummyConstrainedInference(Some(atMostTrue(1)), classifierPositiveScoreForTrue) + instanceSet.count { ins => atMostTwoTrueInference(ins) == "true" } should be(1) + } + + // at most 2 false + "AtMost " should " return at most two false instances" in { + val atMostTwoFalseInference = new DummyConstrainedInference(Some(atMostFalse(1)), classifierNegativeScoreForTrue) + instanceSet.count { ins => atMostTwoFalseInference(ins) == "false" } should be(1) + } + + // at most 3 true + "AtMost " should " return at most three true instances" in { + val atMostThreeTrueInference = new DummyConstrainedInference(Some(atMostTrue(3)), classifierPositiveScoreForTrue) + instanceSet.count { ins => atMostThreeTrueInference(ins) == "true" } should be(3) + } + + // at most 3 false + "AtMost " should " return at most three false instances" in { + val atMostThreeFalseInference = new DummyConstrainedInference(Some(atMostFalse(3)), classifierNegativeScoreForTrue) + instanceSet.count { ins => atMostThreeFalseInference(ins) == "false" } should be(3) + } + + // negation of ForAllTrue + "ForAllFalseWithNegation " should " all be false" in { + val forAllFalseWithNegationInference = new DummyConstrainedInference(Some(forAllFalseWithNegation), classifierPositiveScoreForTrue) + instanceSet.count { ins => forAllFalseWithNegationInference(ins) == "false" } should be(instanceSet.length) + } + + // negation of ForAllTrue + "ForAllTrueNegated " should " contain at least one false" in { + val forAllTrueNegatedInference = new DummyConstrainedInference(Some(forAllTrueNegated), classifierPositiveScoreForTrue) + instanceSet.count { ins => forAllTrueNegatedInference(ins) == "false" } should be >= 1 + } + + // conjunctions + "AllTrueAllTrueConjunction " should " always be true" in { + val allTrueAllTrueConjunctionInference = new DummyConstrainedInference(Some(allTrueAllTrueConjunction), classifierPositiveScoreForTrue) + instanceSet.forall { ins => allTrueAllTrueConjunctionInference(ins) == "true" } should be(true) + } + + "AllFalseAllTrueConjunction " should " always be false" in { + val allFalseAllFalseConjunctionInference = new DummyConstrainedInference(Some(allFalseAllFalseConjunction), classifierPositiveScoreForTrue) + instanceSet.forall { ins => allFalseAllFalseConjunctionInference(ins) == "false" } should be(true) + } + + // disjunctions + "AllTrueAllTrueDisjunction " should " always be true" in { + val allTrueAllTrueDisjunctionInference = new DummyConstrainedInference(Some(allTrueAllTrueDisjunction), classifierPositiveScoreForTrue) + instanceSet.forall { ins => allTrueAllTrueDisjunctionInference(ins) == "true" } should be(true) + } + + "AllFalseAllFalseDisjunction " should " always be false" in { + val allFalseAllFalseDisjunctionInference = new DummyConstrainedInference(Some(allFalseAllFalseDisjunction), classifierPositiveScoreForTrue) + instanceSet.count { ins => allFalseAllFalseDisjunctionInference(ins) == "false" } should be(instanceSet.size) + } + + "AllTrueAllFalseDisjunction " should " always all be false, or should all be true" in { + val allTrueAllFalseDisjunctionInference = new DummyConstrainedInference(Some(allTrueAllFalseDisjunction), classifierPositiveScoreForTrue) + (instanceSet.forall { ins => allTrueAllFalseDisjunctionInference(ins) == "false" } || + instanceSet.forall { ins => allTrueAllFalseDisjunctionInference(ins) == "true" }) should be(true) + } + + "AllFalseAllTrueDisjunction " should " always all be false, or should all be true" in { + val allFalseAllTrueDisjunctionInference = new DummyConstrainedInference(Some(allFalseAllTrueDisjunction), classifierPositiveScoreForTrue) + (instanceSet.forall { ins => allFalseAllTrueDisjunctionInference(ins) == "false" } || + instanceSet.forall { ins => allFalseAllTrueDisjunctionInference(ins) == "true" }) should be(true) + } + + "classifiers with instance pair label equality constraint " should " have the same value for all instances" in { + val classifierSameValueTwoInstancesInference = new DummyConstrainedInference( + Some(allInstancesShouldBeTrue), classifierPositiveScoreForTrue + ) + instanceSet.forall { ins => classifierSameValueTwoInstancesInference(ins) == "true" } + } + + "trueImpliesTrue " should "work" in { + val classifierSameValueTwoInstancesInference = new DummyConstrainedInference( + Some(trueImpliesTrue), classifierNegativeScoreForTrue + ) + classifierSameValueTwoInstancesInference(instanceSet(0)) == "true" && + classifierSameValueTwoInstancesInference(instanceSet(1)) == "true" + } + + "trueImpliesFalse " should "work" in { + val classifierSameValueTwoInstancesInference = new DummyConstrainedInference( + Some(trueImpliesFalse), classifierNegativeScoreForTrue + ) + classifierSameValueTwoInstancesInference(instanceSet(0)) == "true" && + classifierSameValueTwoInstancesInference(instanceSet(1)) == "false" + } + + "falseImpliesTrue " should "work" in { + val classifierSameValueTwoInstancesInference = new DummyConstrainedInference( + Some(falseImpliesTrue), classifierNegativeScoreForTrue + ) + classifierSameValueTwoInstancesInference(instanceSet(0)) == "false" && + classifierSameValueTwoInstancesInference(instanceSet(1)) == "true" + } + + "falseImpliesFalse " should "work" in { + val classifierSameValueTwoInstancesInference = new DummyConstrainedInference( + Some(falseImpliesFalse), classifierNegativeScoreForTrue + ) + classifierSameValueTwoInstancesInference(instanceSet(0)) == "false" && + classifierSameValueTwoInstancesInference(instanceSet(1)) == "false" + } + + "halfTrueHalfFalsePositiveClassifier" should " work properly" in { + val halfTrueHalfFalsePositiveClassifierInference = new DummyConstrainedInference( + Some(halfTrueHalfFalsePositiveClassifier), classifierPositiveScoreForTrue + ) + ((0 to instanceSet.size / 2).forall(i => halfTrueHalfFalsePositiveClassifierInference(instanceSet(i)) == "true") && + ((instanceSet.size / 2 + 1) until instanceSet.size).forall(i => halfTrueHalfFalsePositiveClassifierInference(instanceSet(i)) == "false")) || + ((0 to instanceSet.size / 2).forall(i => halfTrueHalfFalsePositiveClassifierInference(instanceSet(i)) == "false") && + ((instanceSet.size / 2 + 1) until instanceSet.size).forall(i => halfTrueHalfFalsePositiveClassifierInference(instanceSet(i)) == "true")) + } + + "conjunctionOfDisjunctions " should " work" in { + val conjunctionOfDisjunctionInference = new DummyConstrainedInference( + Some(conjunctionOfDisjunction), classifierPositiveScoreForTrue + ) + (0 to 2).count { i => + conjunctionOfDisjunctionInference(instanceSet(i)) == "false" + } should be(2) + } + + "disjunctionOfConjunction " should " work" in { + val disjunctionOfConjunctionsInference = new DummyConstrainedInference( + Some(disjunctionOfConjunctions), classifierPositiveScoreForTrue + ) + (0 to 2).count { i => + disjunctionOfConjunctionsInference(instanceSet(i)) == "false" + } should be(1) + } +} diff --git a/saul-examples/src/main/resources/SetCover/example.txt b/saul-examples/src/main/resources/SetCover/example.txt new file mode 100644 index 00000000..282029e9 --- /dev/null +++ b/saul-examples/src/main/resources/SetCover/example.txt @@ -0,0 +1,9 @@ +1 2 3 4 5 +2 1 +3 1 +4 1 +5 1 +6 7 8 9 +7 6 +8 6 +9 6 diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/Badge/BadgeConstrainedClassifiers.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/Badge/BadgeConstrainedClassifiers.scala index b5587d9d..6f9681df 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/Badge/BadgeConstrainedClassifiers.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/Badge/BadgeConstrainedClassifiers.scala @@ -7,41 +7,43 @@ package edu.illinois.cs.cogcomp.saulexamples.Badge import edu.illinois.cs.cogcomp.infer.ilp.OJalgoHook -import edu.illinois.cs.cogcomp.saul.classifier.ConstrainedClassifier -import edu.illinois.cs.cogcomp.saul.constraint.ConstraintTypeConversion._ +import edu.illinois.cs.cogcomp.saul.classifier.infer.{ ConstrainedClassifier, Gurobi } +import edu.illinois.cs.cogcomp.saul.classifier.infer.Constraint._ import edu.illinois.cs.cogcomp.saulexamples.Badge.BadgeClassifiers.{ BadgeOppositClassifierMulti, BadgeClassifierMulti, BadgeClassifier, BadgeOppositClassifier } /** Created by Parisa on 11/1/16. */ object BadgeConstrainedClassifiers { - val binaryConstraint = ConstrainedClassifier.constraint[String] { - x: String => - (BadgeClassifier on x is "negative") ==> (BadgeOppositClassifier on x is "positive") + def binaryConstraint = BadgeDataModel.badge.ForEach { x: String => + (BadgeClassifier on x is "negative") ==> (BadgeOppositClassifier on x is "positive") } - val binaryConstraintOverMultiClassifiers = ConstrainedClassifier.constraint[String] { - x: String => - (BadgeClassifierMulti on x is "negative") ==> (BadgeOppositClassifierMulti on x is "positive") - } - object badgeConstrainedClassifier extends ConstrainedClassifier[String, String](BadgeClassifier) { - def subjectTo = binaryConstraint - override val solver = new OJalgoHook + def binaryConstraintOverMultiClassifiers = BadgeDataModel.badge.ForEach { x: String => + (BadgeClassifierMulti on x is "negative") ==> (BadgeOppositClassifierMulti on x is "positive") } - object oppositBadgeConstrainedClassifier extends ConstrainedClassifier[String, String](BadgeOppositClassifier) { - def subjectTo = binaryConstraint - override val solver = new OJalgoHook + object badgeConstrainedClassifier extends ConstrainedClassifier[String, String] { + override def subjectTo = Some(binaryConstraint) + override def solverType = Gurobi + override lazy val onClassifier = BadgeClassifier } - object badgeConstrainedClassifierMulti extends ConstrainedClassifier[String, String](BadgeClassifierMulti) { - def subjectTo = binaryConstraintOverMultiClassifiers - override val solver = new OJalgoHook + object oppositBadgeConstrainedClassifier extends ConstrainedClassifier[String, String] { + override def subjectTo = Some(binaryConstraint) + override def solverType = Gurobi + override lazy val onClassifier = BadgeOppositClassifier } - object oppositBadgeConstrainedClassifierMulti extends ConstrainedClassifier[String, String](BadgeOppositClassifierMulti) { - def subjectTo = binaryConstraintOverMultiClassifiers - override val solver = new OJalgoHook + object badgeConstrainedClassifierMulti extends ConstrainedClassifier[String, String] { + override def subjectTo = Some(binaryConstraintOverMultiClassifiers) + override def solverType = Gurobi + override lazy val onClassifier = BadgeClassifierMulti } + object oppositBadgeConstrainedClassifierMulti extends ConstrainedClassifier[String, String] { + override def subjectTo = Some(binaryConstraintOverMultiClassifiers) + override def solverType = Gurobi + override lazy val onClassifier = BadgeOppositClassifierMulti + } } diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala index 6490378f..320e8781 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationApp.scala @@ -9,7 +9,7 @@ package edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation import edu.illinois.cs.cogcomp.core.datastructures.ViewNames import edu.illinois.cs.cogcomp.nlp.tokenizer.StatefulTokenizer import edu.illinois.cs.cogcomp.nlp.utility.TokenizerTextAnnotationBuilder -import edu.illinois.cs.cogcomp.saul.classifier.{ ClassifierUtils, JointTrainSparseNetwork } +import edu.illinois.cs.cogcomp.saul.classifier.{ JointTrainSparseNetwork, ClassifierUtils } import edu.illinois.cs.cogcomp.saul.util.Logging import edu.illinois.cs.cogcomp.saulexamples.EntityMentionRelation.datastruct.ConllRelation import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationClassifiers._ @@ -25,7 +25,7 @@ object EntityRelationApp extends Logging { def main(args: Array[String]): Unit = { /** Choose the experiment you're interested in by changing the following line */ - val testType = ERExperimentType.InteractiveMode + val testType = ERExperimentType.LPlusI testType match { case ERExperimentType.IndependentClassifiers => trainIndependentClassifiers() @@ -105,9 +105,9 @@ object EntityRelationApp extends Logging { ClassifierUtils.LoadClassifier(jarModelPath, PersonClassifier, OrganizationClassifier, LocationClassifier, WorksForClassifier, LivesInClassifier, LocatedInClassifier, OrgBasedInClassifier) - // Test using constrained classifiers + // Test using constrained classifiers ClassifierUtils.TestClassifiers(PerConstrainedClassifier, OrgConstrainedClassifier, LocConstrainedClassifier, - WorksFor_PerOrg_ConstrainedClassifier, LivesIn_PerOrg_relationConstrainedClassifier) + WorksForRelationConstrainedClassifier, LivesInRelationConstrainedClassifier) } /** here we meanwhile training classifiers, we use global inference, in order to overcome the poor local @@ -129,8 +129,8 @@ object EntityRelationApp extends Logging { JointTrainSparseNetwork.train[ConllRelation]( pairs, PerConstrainedClassifier :: OrgConstrainedClassifier :: LocConstrainedClassifier :: - WorksFor_PerOrg_ConstrainedClassifier :: LivesIn_PerOrg_relationConstrainedClassifier :: Nil, - jointTrainIteration, true + WorksForRelationConstrainedClassifier :: LivesInRelationConstrainedClassifier :: Nil, + jointTrainIteration, init = true ) // TODO: merge the following two tests @@ -138,8 +138,8 @@ object EntityRelationApp extends Logging { (testTokens, LocConstrainedClassifier)) ClassifierUtils.TestClassifiers( - (testRels, WorksFor_PerOrg_ConstrainedClassifier), - (testRels, LivesIn_PerOrg_relationConstrainedClassifier) + (testRels, WorksForRelationConstrainedClassifier), + (testRels, LivesInRelationConstrainedClassifier) ) } diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationClassifiers.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationClassifiers.scala index b7292bf5..6be0982a 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationClassifiers.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationClassifiers.scala @@ -15,7 +15,7 @@ import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationDat object EntityRelationClassifiers { /** independent entity classifiers */ object OrganizationClassifier extends Learnable(tokens) { - def label: Property[ConllRawToken] = entityType is "Org" + def label = entityType is "Org" override lazy val classifier = new SparseNetworkLearner() override def feature = using(word, posWindowFeature, phrase, containsSubPhraseMent, containsSubPhraseIng, wordLen) @@ -23,7 +23,7 @@ object EntityRelationClassifiers { } object PersonClassifier extends Learnable(tokens) { - def label: Property[ConllRawToken] = entityType is "Peop" + def label = entityType is "Peop" override def feature = using(word, posWindowFeature, phrase, containsSubPhraseMent, containsSubPhraseIng, wordLen) override lazy val classifier = new SparseNetworkLearner() @@ -31,7 +31,7 @@ object EntityRelationClassifiers { } object LocationClassifier extends Learnable(tokens) { - def label: Property[ConllRawToken] = entityType is "Loc" + def label = entityType is "Loc" override def feature = using(word, posWindowFeature, phrase, containsSubPhraseMent, containsSubPhraseIng, wordLen) override lazy val classifier = new SparseNetworkLearner() @@ -40,36 +40,36 @@ object EntityRelationClassifiers { /** independent relation classifiers */ object WorksForClassifier extends Learnable(pairs) { - def label: Property[ConllRelation] = relationType is "Work_For" + def label = relationType is "Work_For" override def feature = using(relFeature, relPos) override lazy val classifier = new SparseNetworkLearner() } object LivesInClassifier extends Learnable(pairs) { - def label: Property[ConllRelation] = relationType is "Live_In" + def label = relationType is "Live_In" override def feature = using(relFeature, relPos) override lazy val classifier = new SparseNetworkLearner() } object OrgBasedInClassifier extends Learnable(pairs) { - override def label: Property[ConllRelation] = relationType is "OrgBased_In" + override def label = relationType is "OrgBased_In" override lazy val classifier = new SparseNetworkLearner() } object LocatedInClassifier extends Learnable(pairs) { - override def label: Property[ConllRelation] = relationType is "Located_In" + override def label = relationType is "Located_In" override lazy val classifier = new SparseNetworkLearner() } /** relation pipeline classifiers */ object WorksForClassifierPipeline extends Learnable(pairs) { - override def label: Property[ConllRelation] = relationType is "Work_For" + override def label = relationType is "Work_For" override def feature = using(relFeature, relPos, entityPrediction) override lazy val classifier = new SparseNetworkLearner() } object LivesInClassifierPipeline extends Learnable(pairs) { - override def label: Property[ConllRelation] = relationType is "Live_In" + override def label = relationType is "Live_In" override def feature = using(relFeature, relPos, entityPrediction) override lazy val classifier = new SparseNetworkLearner() } diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationConstrainedClassifiers.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationConstrainedClassifiers.scala index 445b8f60..3a681bd1 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationConstrainedClassifiers.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationConstrainedClassifiers.scala @@ -6,43 +6,44 @@ */ package edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation -import edu.illinois.cs.cogcomp.infer.ilp.OJalgoHook -import edu.illinois.cs.cogcomp.saul.classifier.ConstrainedClassifier +import edu.illinois.cs.cogcomp.saul.classifier.infer.{ ConstrainedClassifier, OJAlgo } import edu.illinois.cs.cogcomp.saulexamples.EntityMentionRelation.datastruct.{ ConllRawToken, ConllRelation } -import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationClassifiers._ -import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationConstraints._ object EntityRelationConstrainedClassifiers { - val erSolver = new OJalgoHook - - object OrgConstrainedClassifier extends ConstrainedClassifier[ConllRawToken, ConllRelation](OrganizationClassifier) { - def subjectTo = relationArgumentConstraints - override val pathToHead = Some(-EntityRelationDataModel.pairTo2ndArg) + object OrgConstrainedClassifier extends ConstrainedClassifier[ConllRawToken, ConllRelation] { + override lazy val onClassifier = EntityRelationClassifiers.OrganizationClassifier + override def pathToHead = Some(-EntityRelationDataModel.pairTo2ndArg) + override def subjectTo = Some(EntityRelationConstraints.relationArgumentConstraints) override def filter(t: ConllRawToken, h: ConllRelation): Boolean = t.wordId == h.wordId2 - override val solver = erSolver + override def solverType = OJAlgo } - object PerConstrainedClassifier extends ConstrainedClassifier[ConllRawToken, ConllRelation](PersonClassifier) { - def subjectTo = relationArgumentConstraints - override val pathToHead = Some(-EntityRelationDataModel.pairTo1stArg) + object PerConstrainedClassifier extends ConstrainedClassifier[ConllRawToken, ConllRelation] { + override lazy val onClassifier = EntityRelationClassifiers.PersonClassifier + override def pathToHead = Some(-EntityRelationDataModel.pairTo1stArg) + override def subjectTo = Some(EntityRelationConstraints.relationArgumentConstraints) override def filter(t: ConllRawToken, h: ConllRelation): Boolean = t.wordId == h.wordId1 - override val solver = erSolver + override def solverType = OJAlgo } - object LocConstrainedClassifier extends ConstrainedClassifier[ConllRawToken, ConllRelation](LocationClassifier) { - def subjectTo = relationArgumentConstraints - override val pathToHead = Some(-EntityRelationDataModel.pairTo2ndArg) + object LocConstrainedClassifier extends ConstrainedClassifier[ConllRawToken, ConllRelation] { + override lazy val onClassifier = EntityRelationClassifiers.LocationClassifier + override def pathToHead = Some(-EntityRelationDataModel.pairTo2ndArg) + override def subjectTo = Some(EntityRelationConstraints.relationArgumentConstraints) override def filter(t: ConllRawToken, h: ConllRelation): Boolean = t.wordId == h.wordId2 - override val solver = erSolver + override def solverType = OJAlgo } - object WorksFor_PerOrg_ConstrainedClassifier extends ConstrainedClassifier[ConllRelation, ConllRelation](WorksForClassifier) { - def subjectTo = relationArgumentConstraints - override val solver = new OJalgoHook + object WorksForRelationConstrainedClassifier extends ConstrainedClassifier[ConllRelation, ConllRelation] { + override lazy val onClassifier = EntityRelationClassifiers.WorksForClassifier + override def subjectTo = Some(EntityRelationConstraints.relationArgumentConstraints) + override def solverType = OJAlgo } - object LivesIn_PerOrg_relationConstrainedClassifier extends ConstrainedClassifier[ConllRelation, ConllRelation](LivesInClassifier) { - def subjectTo = relationArgumentConstraints - override val solver = erSolver + object LivesInRelationConstrainedClassifier extends ConstrainedClassifier[ConllRelation, ConllRelation] { + override lazy val onClassifier = EntityRelationClassifiers.LivesInClassifier + override def subjectTo = Some(EntityRelationConstraints.relationArgumentConstraints) + override def solverType = OJAlgo } } + diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationConstraints.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationConstraints.scala index 637dff0a..6a016c07 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationConstraints.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationConstraints.scala @@ -6,47 +6,41 @@ */ package edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation -import edu.illinois.cs.cogcomp.saul.classifier.ConstrainedClassifier -import edu.illinois.cs.cogcomp.saul.constraint.ConstraintTypeConversion._ -import edu.illinois.cs.cogcomp.saulexamples.EntityMentionRelation.datastruct.{ ConllRawSentence, ConllRelation } +import edu.illinois.cs.cogcomp.saulexamples.EntityMentionRelation.datastruct.ConllRelation import EntityRelationClassifiers._ +import edu.illinois.cs.cogcomp.saul.classifier.infer.Constraint._ object EntityRelationConstraints { - // if x is works-for relation, it shouldn't be lives-in relation. - val relationArgumentConstraints = ConstrainedClassifier.constraint[ConllRelation] { x: ConllRelation => + def relationArgumentConstraints = EntityRelationDataModel.pairs.ForEach { x: ConllRelation => worksForConstraint(x) and livesInConstraint(x) and worksForImpliesNotLivesIn(x) } - // if x is lives-in realtion, then its first argument should be person, and second argument should be location. - val livesInConstraint = ConstrainedClassifier.constraint[ConllRelation] { x: ConllRelation => - ((LivesInClassifier on x) isTrue) ==> - (((PersonClassifier on x.e1) isTrue) and ((LocationClassifier on x.e2) isTrue)) + // if x is lives-in relation, then its first argument should be person, and second argument should be location. + def livesInConstraint(x: ConllRelation) = { + (LivesInClassifier on x isTrue) ==> ((PersonClassifier on x.e1 isTrue) and (LocationClassifier on x.e2 isTrue)) } // if x is works-for relation, then its first argument should be person, and second argument should be organization. - val worksForConstraint = ConstrainedClassifier.constraint[ConllRelation] { x: ConllRelation => - ((WorksForClassifier on x) isTrue) ==> - (((PersonClassifier on x.e1) isTrue) and ((OrganizationClassifier on x.e2) isTrue)) + def worksForConstraint(x: ConllRelation) = { + (WorksForClassifier on x isTrue) ==> ((PersonClassifier on x.e1 isTrue) and (OrganizationClassifier on x.e2 isTrue)) } // if x is works-for, it cannot be lives-in, and vice verca - val worksForImpliesNotLivesIn = ConstrainedClassifier.constraint[ConllRelation] { x: ConllRelation => - ((WorksForClassifier on x isTrue) ==> (LivesInClassifier on x isNotTrue)) and - ((LivesInClassifier on x isTrue) ==> (WorksForClassifier on x isNotTrue)) + def worksForImpliesNotLivesIn(x: ConllRelation) = { + ((WorksForClassifier on x isTrue) ==> (LivesInClassifier on x isFalse)) and + ((LivesInClassifier on x isTrue) ==> (WorksForClassifier on x isFalse)) } // TODO: create constrained classifiers for these constraints // if x is located-relation, its first argument must be a person or organization, while its second argument // must be a location - val locatedInConstrint = ConstrainedClassifier.constraint[ConllRelation] { x: ConllRelation => + def locatedInConstraint(x: ConllRelation) = { (LocatedInClassifier on x isTrue) ==> - (((PersonClassifier on x.e1 isTrue) or (OrganizationClassifier on x.e1 isTrue)) - and (LocationClassifier on x.e2 isTrue)) + (((PersonClassifier on x.e1 isTrue) or (OrganizationClassifier on x.e1 isTrue)) and (LocationClassifier on x.e2 isTrue)) } - val orgBasedInConstraint = ConstrainedClassifier.constraint[ConllRelation] { x: ConllRelation => - (OrgBasedInClassifier on x isTrue) ==> - ((OrganizationClassifier on x isTrue) and (LocationClassifier on x isTrue)) + def orgBasedInConstraint(x: ConllRelation) = { + (OrgBasedInClassifier on x isTrue) ==> ((OrganizationClassifier on x isTrue) and (LocationClassifier on x isTrue)) } } diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLApps.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLApps.scala index 9970b93b..8013d85a 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLApps.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLApps.scala @@ -6,13 +6,15 @@ */ package edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling -import java.io.File - import edu.illinois.cs.cogcomp.core.datastructures.ViewNames +import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager + import edu.illinois.cs.cogcomp.saul.classifier.{ ClassifierUtils, JointTrainSparseNetwork } import edu.illinois.cs.cogcomp.saul.util.Logging import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLClassifiers._ -import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLConstrainedClassifiers.argTypeConstraintClassifier +import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLConstrainedClassifiers.ArgTypeConstrainedClassifier + +import java.io.File object SRLscalaConfigurator { @@ -177,17 +179,17 @@ object RunningApps extends App with Logging { argumentTypeLearner.modelDir = modelDir + expName val outputFile = modelDir + SRL_OUTPUT_FILE logger.info("Global training... ") - JointTrainSparseNetwork(sentences, argTypeConstraintClassifier :: Nil, 30, init = true) + JointTrainSparseNetwork(sentences, ArgTypeConstrainedClassifier :: Nil, 30, init = true) argumentTypeLearner.save() - argTypeConstraintClassifier.test(relations.getTestingInstances, outputFile, 200, exclude = "candidate") + ArgTypeConstrainedClassifier.test(relations.getTestingInstances, outputFile, 200, exclude = "candidate") case "lTr" => argumentTypeLearner.modelDir = modelDir + expName val outputFile = modelDir + SRL_OUTPUT_FILE logger.info("Global training using loss augmented inference... ") - JointTrainSparseNetwork(sentences, argTypeConstraintClassifier :: Nil, 30, init = true, lossAugmented = true) + JointTrainSparseNetwork(sentences, ArgTypeConstrainedClassifier :: Nil, 30, init = true, lossAugmented = true) argumentTypeLearner.save() - argTypeConstraintClassifier.test(relations.getTestingInstances, outputFile, 200, exclude = "candidate") + ArgTypeConstrainedClassifier.test(relations.getTestingInstances, outputFile, 200, exclude = "candidate") } } @@ -214,7 +216,7 @@ object RunningApps extends App with Logging { case (false, true) => ClassifierUtils.LoadClassifier(SRLConfigurator.SRL_JAR_MODEL_PATH.value + "/models_aTr/", argumentTypeLearner) - argTypeConstraintClassifier.test(outputGranularity = 100, exclude = "candidate") + ArgTypeConstrainedClassifier.test(outputGranularity = 100, exclude = "candidate") case (false, false) => ClassifierUtils.LoadClassifier(SRLConfigurator.SRL_JAR_MODEL_PATH.value + "/models_aTr/", argumentTypeLearner) diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLClassifiers.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLClassifiers.scala index a4d4155a..89800530 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLClassifiers.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLClassifiers.scala @@ -17,13 +17,14 @@ object SRLClassifiers { import SRLApps.srlDataModelObject._ //TODO This needs to be overriden by the user; change it to be dynamic val parameters = new SparseAveragedPerceptron.Parameters() - object predicateClassifier extends Learnable[Constituent](predicates, parameters) { + object predicateClassifier extends Learnable[Constituent](predicates, parameters) { //TODO These are not used during Learner's initialization def label: Property[Constituent] = isPredicateGold override def feature = using(posTag, subcategorization, phraseType, headword, voice, verbClass, predPOSWindow, predWordWindow) override lazy val classifier = new SparseNetworkLearner() } + //This classifier has not been used in our current models object predicateSenseClassifier extends Learnable[Constituent](predicates, parameters) { def label = predicateSenseGold @@ -39,13 +40,11 @@ object SRLClassifiers { } object argumentXuIdentifierGivenApredicate extends Learnable[Relation](relations, parameters) { - def label = isArgumentXuGold override def feature = using(headwordRelation, syntacticFrameRelation, pathRelation, phraseTypeRelation, predPosTag, predLemmaR, linearPosition, argWordWindow, argPOSWindow, constituentLength, chunkLength, chunkEmbedding, chunkPathPattern, clauseFeatures, containsNEG, containsMOD) override lazy val classifier = new SparseNetworkLearner() } - } diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLConstrainedClassifiers.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLConstrainedClassifiers.scala index cab81f68..8d007f19 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLConstrainedClassifiers.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLConstrainedClassifiers.scala @@ -7,32 +7,29 @@ package edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling import edu.illinois.cs.cogcomp.core.datastructures.textannotation.{ Relation, TextAnnotation } -import edu.illinois.cs.cogcomp.infer.ilp.OJalgoHook -import edu.illinois.cs.cogcomp.saul.classifier.ConstrainedClassifier +import edu.illinois.cs.cogcomp.saul.classifier.infer.{ ConstrainedClassifier, Gurobi } import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLClassifiers.{ argumentTypeLearner, argumentXuIdentifierGivenApredicate } import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLConstraints._ -/** Created by Parisa on 12/27/15. - */ object SRLConstrainedClassifiers { import SRLApps.srlDataModelObject._ - val erSolver = new OJalgoHook - object argTypeConstraintClassifier extends ConstrainedClassifier[Relation, TextAnnotation](argumentTypeLearner) { - def subjectTo = r_and_c_args - override val solver = erSolver + object ArgTypeConstrainedClassifier extends ConstrainedClassifier[Relation, TextAnnotation] { + override def subjectTo = Some(allPredicateArgumentConstraints) + override def solverType = Gurobi + override lazy val onClassifier = argumentTypeLearner override val pathToHead = Some(-sentencesToRelations) } - object arg_Is_TypeConstraintClassifier extends ConstrainedClassifier[Relation, Relation](argumentTypeLearner) { - def subjectTo = arg_IdentifierClassifier_Constraint - override val solver = erSolver + object ArgIsTypeConstrainedClassifier extends ConstrainedClassifier[Relation, Relation] { + override def subjectTo = Some(arg_IdentifierClassifier_Constraint) + override def solverType = Gurobi + override lazy val onClassifier = argumentTypeLearner } - object arg_IdentifyConstraintClassifier extends ConstrainedClassifier[Relation, Relation](argumentXuIdentifierGivenApredicate) { - def subjectTo = arg_IdentifierClassifier_Constraint - override val solver = erSolver + object ArgIdentifyConstrainedClassifier extends ConstrainedClassifier[Relation, Relation] { + override def subjectTo = Some(arg_IdentifierClassifier_Constraint) + override def solverType = Gurobi + override lazy val onClassifier = argumentXuIdentifierGivenApredicate } - } - diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLConstraints.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLConstraints.scala index 5a494d02..1c1909ce 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLConstraints.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLConstraints.scala @@ -8,148 +8,98 @@ package edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling import edu.illinois.cs.cogcomp.core.datastructures.ViewNames import edu.illinois.cs.cogcomp.core.datastructures.textannotation._ -import edu.illinois.cs.cogcomp.lbjava.infer.{ FirstOrderConstant, FirstOrderConstraint } -import edu.illinois.cs.cogcomp.saul.classifier.ConstrainedClassifier -import edu.illinois.cs.cogcomp.saul.constraint.ConstraintTypeConversion._ +import edu.illinois.cs.cogcomp.saul.classifier.infer.Constraint._ import edu.illinois.cs.cogcomp.saulexamples.data.XuPalmerCandidateGenerator import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLApps.srlDataModelObject._ import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLClassifiers.{ argumentTypeLearner, argumentXuIdentifierGivenApredicate, predicateClassifier } import scala.collection.JavaConversions._ -/** Created by Parisa on 12/23/15. - */ -object SRLConstraints { - val noOverlap = ConstrainedClassifier.constraint[TextAnnotation] { - { - var a: FirstOrderConstraint = null - x: TextAnnotation => { - a = new FirstOrderConstant(true) - (sentences(x) ~> sentencesToRelations ~> relationsToPredicates).foreach { - y => - { - val argCandList = XuPalmerCandidateGenerator.generateCandidates(y, (sentences(y.getTextAnnotation) ~> sentencesToStringTree).head). - map(y => new Relation("candidate", y.cloneForNewView(y.getViewName), y.cloneForNewView(y.getViewName), 0.0)) - x.getView(ViewNames.TOKENS).asInstanceOf[TokenLabelView].getConstituents.toList.foreach { - t: Constituent => - { - val contains = argCandList.filter(z => z.getTarget.doesConstituentCover(t)) - a = a and contains.toList._atmost(1)({ p: Relation => (argumentTypeLearner on p).is("candidate") }) - } - } - } - } +object SRLConstraints { + def noOverlap = sentences.ForEach { x: TextAnnotation => + (sentences(x) ~> sentencesToRelations ~> relationsToPredicates).ForAll { y => + val argCandList = XuPalmerCandidateGenerator.generateCandidates(y, (sentences(y.getTextAnnotation) ~> sentencesToStringTree).head). + map(y => new Relation("candidate", y.cloneForNewView(y.getViewName), y.cloneForNewView(y.getViewName), 0.0)) + x.getView(ViewNames.TOKENS).getConstituents.ForAll { + t: Constituent => + val contains = argCandList.filter(_.getTarget.doesConstituentCover(t)) + contains.AtMost(1) { p: Relation => argumentTypeLearner on p is "candidate" } } - a } - } //end of NoOverlap constraint + } - val arg_IdentifierClassifier_Constraint = ConstrainedClassifier.constraint[Relation] { - x: Relation => - { - (argumentXuIdentifierGivenApredicate on x isNotTrue) ==> - (argumentTypeLearner on x is "candidate") - } + def arg_IdentifierClassifier_Constraint = relations.ForEach { x: Relation => + (argumentXuIdentifierGivenApredicate on x isFalse) ==> (argumentTypeLearner on x is "candidate") } - val predArg_IdentifierClassifier_Constraint = ConstrainedClassifier.constraint[Relation] { - x: Relation => - { - (predicateClassifier on x.getSource isTrue) and (argumentXuIdentifierGivenApredicate on x isTrue) ==> - (argumentTypeLearner on x isNot "candidate") - } + def predArg_IdentifierClassifier_Constraint = relations.ForEach { x: Relation => + (predicateClassifier on x.getSource isTrue) and (argumentXuIdentifierGivenApredicate on x isTrue) ==> + (argumentTypeLearner on x isNot "candidate") } - val r_arg_Constraint = ConstrainedClassifier.constraint[TextAnnotation] { - var a: FirstOrderConstraint = null - x: TextAnnotation => { - a = new FirstOrderConstant(true) - val values = Array("R-A1", "R-A2", "R-A3", "R-A4", "R-A5", "R-AA", "R-AM-ADV", "R-AM-CAU", "R-AM-EXT", "R-AM-LOC", "R-AM-MNR", "R-AM-PNC") - (sentences(x) ~> sentencesToRelations ~> relationsToPredicates).foreach { - y => - { - val argCandList = (predicates(y) ~> -relationsToPredicates).toList - argCandList.foreach { - t: Relation => - { - for (i <- 0 until values.length) - a = a and ((argumentTypeLearner on t) is values(i)) ==> - argCandList.filterNot(x => x.equals(t))._exists { - k: Relation => (argumentTypeLearner on k) is values(i).substring(2) - } - } - a - } - } + /** constraint for reference to an actual argument/adjunct of type arg */ + def rArgConstraint(x: TextAnnotation) = { + val values = Array("R-A1", "R-A2", "R-A3", "R-A4", "R-AA", "R-AM-ADV", "R-AM-CAU", "R-AM-EXT", "R-AM-LOC", "R-AM-MNR", "R-AM-PNC") + val constraints = for { + y <- sentences(x) ~> sentencesToRelations ~> relationsToPredicates + argCandList = (predicates(y) ~> -relationsToPredicates).toList + r: Relation <- argCandList + i <- values.indices + } yield ((argumentTypeLearner on r) is values(i)) ==> + argCandList.filterNot(x => x.equals(r)).Exists { + k: Relation => (argumentTypeLearner on k) is values(i).substring(2) } - } - a - } // end r-arg constraint + constraints.ForAll + } - val c_arg_Constraint = ConstrainedClassifier.constraint[TextAnnotation] { - var a: FirstOrderConstraint = null - x: TextAnnotation => { - a = new FirstOrderConstant(true) - val values = Array("C-A1", "C-A2", "C-A3", "C-A4", "C-A5", "C-AA", "C-AM-DIR", "C-AM-LOC", "C-AM-MNR", "C-AM-NEG", "C-AM-PNC") - (sentences(x) ~> sentencesToRelations ~> relationsToPredicates).foreach { - y => - { - val argCandList = (predicates(y) ~> -relationsToPredicates).toList - val sortedCandidates = argCandList.sortBy(x => x.getTarget.getStartSpan) - sortedCandidates.zipWithIndex.foreach { - case (t, ind) => - { - if (ind > 0) - for (i <- 0 until values.length) - a = a and ((argumentTypeLearner on t) is values(i)) ==> - sortedCandidates.subList(0, ind)._exists { - k: Relation => (argumentTypeLearner on k) is values(i).substring(2) - } - } - } - } + /** constraint for continuity of an argument/adjunct of type arg */ + def cArgConstraint(x: TextAnnotation) = { + val values = Array("C-A1", "C-A2", "C-A3", "C-A4", "C-A5", "C-AM-DIR", "C-AM-LOC", "C-AM-MNR", "C-AM-NEG", "C-AM-PNC") + val constraints = for { + y <- sentences(x) ~> sentencesToRelations ~> relationsToPredicates + argCandList = (predicates(y) ~> -relationsToPredicates).toList + sortedCandidates = argCandList.sortBy(x => x.getTarget.getStartSpan) + (t, ind) <- sortedCandidates.zipWithIndex.drop(1) + i <- values.indices + labelOnT = (argumentTypeLearner on t) is values(i) + labelsIsValid = sortedCandidates.subList(0, ind).Exists { + k: Relation => (argumentTypeLearner on k) is values(i).substring(2) } - } - a + } yield labelOnT ==> labelsIsValid + constraints.ForAll } - val legal_arguments_Constraint = ConstrainedClassifier.constraint[TextAnnotation] { x: TextAnnotation => + /** the label of the classifier should be valid */ + def legalArgumentsConstraint(x: TextAnnotation) = { + // these are the labels that are not used in the 'argumentTypeLearner' classifier + val excludedLabels = Set("R-AM-NEG", "R-AM-MOD", "", "R-AM-DIS", "R-AM-REC", "R-AM-PRD", "C-AM-REC", + "C-AM-PRD", "R-AM-DIR", "C-AM-MOD", "AM", "R-AM", "C-AM") val constraints = for { y <- sentences(x) ~> sentencesToRelations ~> relationsToPredicates argCandList = (predicates(y) ~> -relationsToPredicates).toList - argLegalList = legalArguments(y) + argLegalList = legalArguments(y).toSet diff excludedLabels z <- argCandList - } yield argLegalList._exists { t: String => argumentTypeLearner on z is t } or + } yield argLegalList.Exists { t: String => argumentTypeLearner on z is t } or (argumentTypeLearner on z is "candidate") - constraints.toSeq._forall(a => a) + constraints.ForAll } - val noDuplicate = ConstrainedClassifier.constraint[TextAnnotation] { - // Predicates have at most one argument of each type i.e. there shouldn't be any two arguments with the same type for each predicate + // Predicates have at most one argument of each type i.e. there shouldn't be any two arguments with the same type for each predicate + def noInconsistentPredicateLabels(x: TextAnnotation) = { val values = Array("A0", "A1", "A2", "A3", "A4", "A5", "AA") - var a: FirstOrderConstraint = null - x: TextAnnotation => { - a = new FirstOrderConstant(true) - (sentences(x) ~> sentencesToRelations ~> relationsToPredicates).foreach { - y => - { - val argCandList = (predicates(y) ~> -relationsToPredicates).toList - for (t1 <- 0 until argCandList.size - 1) { - for (t2 <- t1 + 1 until argCandList.size) { - a = a and (((argumentTypeLearner on argCandList.get(t1)) in values) ==> (((argumentTypeLearner on argCandList.get(t1)) isNot (argumentTypeLearner on argCandList.get(t2))))) - } - - } - } - } - a - } + val constraints = for { + y <- sentences(x) ~> sentencesToRelations ~> relationsToPredicates + argCandList = (predicates(y) ~> -relationsToPredicates).toList + idx1 <- 0 until argCandList.size - 1 + idx2 <- idx1 + 1 until argCandList.size + predictionIsValid = (argumentTypeLearner on argCandList.get(idx1)) isOneOf values + haveSameLabels = (argumentTypeLearner on argCandList.get(idx1)) equalsTo argCandList.get(idx2) + } yield predictionIsValid ==> haveSameLabels + constraints.ForAll } - val r_and_c_args = ConstrainedClassifier.constraint[TextAnnotation] { - x => - r_arg_Constraint(x) and c_arg_Constraint(x) and legal_arguments_Constraint(x) and noDuplicate(x) + def allPredicateArgumentConstraints = sentences.ForEach { x: TextAnnotation => + rArgConstraint(x) and cArgConstraint(x) and legalArgumentsConstraint(x) and noInconsistentPredicateLabels(x) } - -} // end srlConstraints +} diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLMultiGraphDataModel.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLMultiGraphDataModel.scala index 6ce4c391..4939722f 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLMultiGraphDataModel.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLMultiGraphDataModel.scala @@ -245,7 +245,7 @@ class SRLMultiGraphDataModel(parseViewName: String = null, frameManager: SRLFram x: Relation => val a: String = argumentXuIdentifierGivenApredicate(x) match { case "false" => "candidate" - case _ => argTypeConstraintClassifier(x) + case _ => ArgTypeConstrainedClassifier(x) } a } diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/setcover/SetCoverApp.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/setcover/SetCoverApp.scala index 2f6e8132..474ce128 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/setcover/SetCoverApp.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/setcover/SetCoverApp.scala @@ -11,17 +11,17 @@ import edu.illinois.cs.cogcomp.saul.util.Logging import scala.collection.JavaConversions._ object SetCoverApp extends Logging { - val cityInstances = new City("saul-examples/src/main/resources/SetCover/example.txt") + val cityInstances = new City("src/main/resources/SetCover/example.txt") val neighborhoodInstances = cityInstances.getNeighborhoods.toList def main(args: Array[String]) { + println("in main: allowable values: " + new ContainsStation().allowableValues.toSeq) SetCoverSolverDataModel.cities populate List(cityInstances) SetCoverSolverDataModel.neighborhoods populate neighborhoodInstances - SetCoverSolverDataModel.cityContainsNeighborhoods.populateWith(_ == _.getParentCity) - - /** printing the labels for each nrighborhood (whether they are choosen to be covered by a station, or not) */ + def getParentCity = (n: Neighborhood) => n.getParentCity + SetCoverSolverDataModel.cityContainsNeighborhoods.populateWith((c: City, n: Neighborhood) => n.getParentCity == c) cityInstances.getNeighborhoods.foreach { - n => logger.info(n.getNumber + ": " + ContainsStationConstraint(n)) + n => logger.info(n.getNumber + ": " + ConstrainedContainsStation(n)) } } } \ No newline at end of file diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/setcover/SetCoverDataModel.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/setcover/SetCoverDataModel.scala index 2727d130..17da70f5 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/setcover/SetCoverDataModel.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/setcover/SetCoverDataModel.scala @@ -6,10 +6,11 @@ */ package edu.illinois.cs.cogcomp.saulexamples.setcover -import edu.illinois.cs.cogcomp.infer.ilp.OJalgoHook -import edu.illinois.cs.cogcomp.saul.classifier.ConstrainedClassifier +import edu.illinois.cs.cogcomp.lbjava.learn.Learner +import edu.illinois.cs.cogcomp.saul.classifier.infer.Constraint._ +import edu.illinois.cs.cogcomp.saul.classifier.infer.{ ConstrainedClassifier, OJAlgo } import edu.illinois.cs.cogcomp.saul.datamodel.DataModel -import edu.illinois.cs.cogcomp.saul.constraint.ConstraintTypeConversion._ +import edu.illinois.cs.cogcomp.saul.lbjrelated.LBJLearnerEquivalent object SetCoverSolverDataModel extends DataModel { @@ -22,34 +23,30 @@ object SetCoverSolverDataModel extends DataModel { cityContainsNeighborhoods.populateWith((c, n) => c == n.getParentCity) /** definition of the constraints */ - val containStation = new ContainsStation() + val containStation: LBJLearnerEquivalent = new LBJLearnerEquivalent { + override val classifier: Learner = new ContainsStation() + } def atLeastANeighborOfNeighborhoodIsCovered = { n: Neighborhood => - n.getNeighbors._exists { neighbor: Neighborhood => containStation on neighbor isTrue } + n.getNeighbors.Exists { neighbor: Neighborhood => containStation on neighbor isTrue } } def neighborhoodContainsStation = { n: Neighborhood => - containStation on n isTrue + (containStation on n) isTrue } - def allCityNeiborhoodsAreCovered = { x: City => - x.getNeighborhoods._forall { n: Neighborhood => + def allCityNeighborhoodsAreCovered = { x: City => + x.getNeighborhoods.ForAll { n: Neighborhood => neighborhoodContainsStation(n) or atLeastANeighborOfNeighborhoodIsCovered(n) } } - def someCityNeiborhoodsAreCovered = { x: City => - x.getNeighborhoods._atleast(2) { n: Neighborhood => - neighborhoodContainsStation(n) //or atLeastANeighborOfNeighborhoodIsCovered(n) - } - } - - val containsStationConstraint = ConstrainedClassifier.constraint[City] { x: City => allCityNeiborhoodsAreCovered(x) } + def containsStationConstraint = SetCoverSolverDataModel.cities.ForAll { x: City => allCityNeighborhoodsAreCovered(x) } } -import SetCoverSolverDataModel._ -object ContainsStationConstraint extends ConstrainedClassifier[Neighborhood, City](new ContainsStation()) { - override val pathToHead = Some(-cityContainsNeighborhoods) - override def subjectTo = containsStationConstraint - override val solver = new OJalgoHook +object ConstrainedContainsStation extends ConstrainedClassifier[Neighborhood, City] { + override lazy val onClassifier = SetCoverSolverDataModel.containStation + override def pathToHead = Some(-SetCoverSolverDataModel.cityContainsNeighborhoods) + override def subjectTo = Some(SetCoverSolverDataModel.containsStationConstraint) + override def solverType = OJAlgo } diff --git a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/InferenceQuantifierTests.scala b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/InferenceQuantifierTests.scala index 3a39c922..be8c4802 100644 --- a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/InferenceQuantifierTests.scala +++ b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/InferenceQuantifierTests.scala @@ -6,74 +6,76 @@ */ package edu.illinois.cs.cogcomp.saulexamples -import edu.illinois.cs.cogcomp.infer.ilp.OJalgoHook -import edu.illinois.cs.cogcomp.saul.classifier.ConstrainedClassifier -import edu.illinois.cs.cogcomp.saul.constraint.ConstraintTypeConversion._ +import edu.illinois.cs.cogcomp.lbjava.learn.Learner +import edu.illinois.cs.cogcomp.saul.classifier.infer.Constraint._ +import edu.illinois.cs.cogcomp.saul.classifier.infer.{ ConstrainedClassifier, OJAlgo } import edu.illinois.cs.cogcomp.saul.datamodel.DataModel -import edu.illinois.cs.cogcomp.saulexamples.setcover.{ City, ContainsStation, Neighborhood } -import org.scalatest.{ Matchers, FlatSpec } +import edu.illinois.cs.cogcomp.saul.lbjrelated.LBJLearnerEquivalent +import edu.illinois.cs.cogcomp.saulexamples.setcover.{ City, ContainsStation, Neighborhood, SetCoverSolverDataModel } +import org.scalatest.{ FlatSpec, Matchers } import scala.collection.JavaConversions._ class InferenceQuantifierTests extends FlatSpec with Matchers { object SomeDM extends DataModel { - val cities = node[City] - val neighborhoods = node[Neighborhood] - val cityContainsNeighborhoods = edge(cities, neighborhoods) - cityContainsNeighborhoods.populateWith((c, n) => c == n.getParentCity) - /** definition of the constraints */ val containStation = new ContainsStation() - - def neighborhoodContainsStation = { n: Neighborhood => - containStation on n isTrue + val containStationLBJEquivalent: LBJLearnerEquivalent = new LBJLearnerEquivalent { + override val classifier: Learner = containStation } - val atLeastSomeNeighborsAreCoveredConstraint = ConstrainedClassifier.constraint[City] { x: City => - x.getNeighborhoods._atleast(2) { n: Neighborhood => neighborhoodContainsStation(n) } + /** definition of the constraints */ + def neighborhoodContainsStation(n: Neighborhood) = containStationLBJEquivalent on n isTrue + + def atLeastSomeNeighborsAreCoveredConstraint = cities.ForAll { x: City => + x.getNeighborhoods.AtLeast(2) { n: Neighborhood => neighborhoodContainsStation(n) } } - val atLeastSomeNeighborsAreCoveredConstraintUsingAtMost = ConstrainedClassifier.constraint[City] { x: City => - !x.getNeighborhoods._atmost(2) { n: Neighborhood => neighborhoodContainsStation(n) } + def atLeastSomeNeighborsAreCoveredConstraintUsingAtMost = cities.ForAll { x: City => + !x.getNeighborhoods.AtMost(2) { n: Neighborhood => neighborhoodContainsStation(n) } } - val allNeighborsAreCoveredConstraint = ConstrainedClassifier.constraint[City] { x: City => - x.getNeighborhoods._forall { n: Neighborhood => neighborhoodContainsStation(n) } + def allNeighborsAreCoveredConstraint = cities.ForAll { x: City => + x.getNeighborhoods.ForAll { n: Neighborhood => neighborhoodContainsStation(n) } } - val singleNeighborsAreCoveredConstraint = ConstrainedClassifier.constraint[City] { x: City => - x.getNeighborhoods._exists { n: Neighborhood => neighborhoodContainsStation(n) } + def singleNeighborsAreCoveredConstraint = cities.ForAll { x: City => + x.getNeighborhoods.Exists { n: Neighborhood => neighborhoodContainsStation(n) } } } import SomeDM._ - object AtLeastSomeNeighborhoods extends ConstrainedClassifier[Neighborhood, City](new ContainsStation()) { + object AtLeastSomeNeighborhoods extends ConstrainedClassifier[Neighborhood, City] { override val pathToHead = Some(-cityContainsNeighborhoods) - override def subjectTo = atLeastSomeNeighborsAreCoveredConstraint - override val solver = new OJalgoHook + override def subjectTo = Some(atLeastSomeNeighborsAreCoveredConstraint) + override val solverType = OJAlgo + override def onClassifier: LBJLearnerEquivalent = containStationLBJEquivalent } - object AtLeastSomeNeighborhoodsUsingAtMost extends ConstrainedClassifier[Neighborhood, City](new ContainsStation()) { + object AtLeastSomeNeighborhoodsUsingAtMost extends ConstrainedClassifier[Neighborhood, City] { override val pathToHead = Some(-cityContainsNeighborhoods) - override def subjectTo = atLeastSomeNeighborsAreCoveredConstraintUsingAtMost - override val solver = new OJalgoHook + override def subjectTo = Some(atLeastSomeNeighborsAreCoveredConstraintUsingAtMost) + override val solverType = OJAlgo + override def onClassifier: LBJLearnerEquivalent = containStationLBJEquivalent } - object AllNeighborhoods extends ConstrainedClassifier[Neighborhood, City](new ContainsStation()) { + object AllNeighborhoods extends ConstrainedClassifier[Neighborhood, City] { override val pathToHead = Some(-cityContainsNeighborhoods) - override def subjectTo = allNeighborsAreCoveredConstraint - override val solver = new OJalgoHook + override def subjectTo = Some(allNeighborsAreCoveredConstraint) + override val solverType = OJAlgo + override def onClassifier: LBJLearnerEquivalent = containStationLBJEquivalent } - object ASingleNeighborhood extends ConstrainedClassifier[Neighborhood, City](new ContainsStation()) { + object ASingleNeighborhood extends ConstrainedClassifier[Neighborhood, City] { override val pathToHead = Some(-cityContainsNeighborhoods) - override def subjectTo = singleNeighborsAreCoveredConstraint - override val solver = new OJalgoHook + override def subjectTo = Some(singleNeighborsAreCoveredConstraint) + override val solverType = OJAlgo + override def onClassifier: LBJLearnerEquivalent = containStationLBJEquivalent } val cityInstances = new City("../saul-examples/src/test/resources/SetCover/example.txt") @@ -81,7 +83,8 @@ class InferenceQuantifierTests extends FlatSpec with Matchers { SomeDM.cities populate List(cityInstances) SomeDM.neighborhoods populate neighborhoodInstances - SomeDM.cityContainsNeighborhoods.populateWith(_ == _.getParentCity) + def getParentCity = (n: Neighborhood) => n.getParentCity + SomeDM.cityContainsNeighborhoods.populateWith((c: City, n: Neighborhood) => n.getParentCity == c) "Quantifier atleast " should " work " in { cityInstances.getNeighborhoods.count(n => AtLeastSomeNeighborhoods(n) == "true") should be(2) @@ -89,7 +92,8 @@ class InferenceQuantifierTests extends FlatSpec with Matchers { // negation of atmost(2) is equivalent to atleast(2) "Quantifier atmost " should " work " in { - cityInstances.getNeighborhoods.count(n => AtLeastSomeNeighborhoodsUsingAtMost(n) == "true") should be(3) + cityInstances.getNeighborhoods.count(n => AtLeastSomeNeighborhoodsUsingAtMost(n) == "true") should be(2) + info("cityInstances.getNeighborhoods: " + cityInstances.getNeighborhoods.size()) } "Quantifier forall " should " work " in { diff --git a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/SetCoverTest.scala b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/SetCoverTest.scala index 3a8319fe..ec88be97 100644 --- a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/SetCoverTest.scala +++ b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/SetCoverTest.scala @@ -6,66 +6,64 @@ */ package edu.illinois.cs.cogcomp.saulexamples -import edu.illinois.cs.cogcomp.saulexamples.setcover.{ City, ContainsStationConstraint, SetCoverSolverDataModel } +import edu.illinois.cs.cogcomp.saulexamples.setcover._ import org.scalatest.{ FlatSpec, Matchers } import scala.collection.JavaConversions._ class SetCoverTest extends FlatSpec with Matchers { - import SetCoverSolverDataModel._ - - val prefix = "../saul-examples/src/test/resources/SetCover/" + def prefix(t: String = "test") = s"../saul-examples/src/$t/resources/SetCover/" "SetCover " should " be solved correctly for example.txt " in { - clearInstances - val citiesInstance = new City(prefix + "example.txt") + SetCoverSolverDataModel.clearInstances + val citiesInstance = new City(prefix("main") + "example.txt") val neighborhoodInstances = citiesInstance.getNeighborhoods.toList - cities populate List(citiesInstance) - neighborhoods populate neighborhoodInstances - cityContainsNeighborhoods.populateWith(_ == _.getParentCity) + SetCoverSolverDataModel.cities populate List(citiesInstance) + SetCoverSolverDataModel.neighborhoods populate neighborhoodInstances + SetCoverSolverDataModel.cityContainsNeighborhoods.populateWith(_ == _.getParentCity) val neighborhoodLabels = Map(1 -> true, 2 -> false, 3 -> false, 4 -> false, 5 -> false, 6 -> true, 7 -> false, 8 -> false, 9 -> false) citiesInstance.getNeighborhoods.forall { n => - ContainsStationConstraint(n) == neighborhoodLabels(n.getNumber).toString + ConstrainedContainsStation(n) == neighborhoodLabels(n.getNumber).toString } should be(true) val neighborhoodOutput = "List(neighborhood #1, neighborhood #2, neighborhood #3, neighborhood #4, neighborhood #5, neighborhood #6, neighborhood #7, neighborhood #8, neighborhood #9)" - ContainsStationConstraint.getCandidates(citiesInstance).toList.toString should be(neighborhoodOutput) - cityContainsNeighborhoods(citiesInstance).toList.sorted.toString should be(neighborhoodOutput) + ConstrainedContainsStation.getCandidates(citiesInstance).toList.toString should be(neighborhoodOutput) + SetCoverSolverDataModel.cityContainsNeighborhoods(citiesInstance).toList.sorted.toString should be(neighborhoodOutput) } "SetCover " should " be solved correctly for example2.txt " in { - clearInstances - val citiesInstance = new City(prefix + "example2.txt") + SetCoverSolverDataModel.clearInstances + val citiesInstance = new City(prefix() + "example2.txt") val neighborhoodInstances = citiesInstance.getNeighborhoods.toList - cities populate List(citiesInstance) - neighborhoods populate neighborhoodInstances - cityContainsNeighborhoods.populateWith(_ == _.getParentCity) + SetCoverSolverDataModel.cities populate List(citiesInstance) + SetCoverSolverDataModel.neighborhoods populate neighborhoodInstances + SetCoverSolverDataModel.cityContainsNeighborhoods.populateWith(_ == _.getParentCity) val neighborhoodLabels = Map(1 -> true, 2 -> true, 3 -> false, 4 -> false, 5 -> false, 6 -> false, 7 -> false, 8 -> false) citiesInstance.getNeighborhoods.forall { n => - ContainsStationConstraint(n) == neighborhoodLabels(n.getNumber).toString + ConstrainedContainsStation(n) == neighborhoodLabels(n.getNumber).toString } should be(true) } "SetCover " should " be solved correctly for example3.txt " in { - clearInstances - val citiesInstance = new City(prefix + "example3.txt") + SetCoverSolverDataModel.clearInstances + val citiesInstance = new City(prefix() + "example3.txt") val neighborhoodInstances = citiesInstance.getNeighborhoods.toList - cities populate List(citiesInstance) - neighborhoods populate neighborhoodInstances - cityContainsNeighborhoods.populateWith(_ == _.getParentCity) + SetCoverSolverDataModel.cities populate List(citiesInstance) + SetCoverSolverDataModel.neighborhoods populate neighborhoodInstances + SetCoverSolverDataModel.cityContainsNeighborhoods.populateWith(_ == _.getParentCity) val neighborhoodLabels = Map(1 -> true, 2 -> false, 3 -> false) citiesInstance.getNeighborhoods.forall { n => - ContainsStationConstraint(n) == neighborhoodLabels(n.getNumber).toString + ConstrainedContainsStation(n) == neighborhoodLabels(n.getNumber).toString } should be(true) } } diff --git a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala index 3ce4f58a..b3c96d91 100644 --- a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala +++ b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/EntityRelation/EntityRelationTests.scala @@ -7,7 +7,7 @@ package edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation import edu.illinois.cs.cogcomp.lbjava.learn.{ LinearThresholdUnit, SparseNetworkLearner } -import edu.illinois.cs.cogcomp.saul.classifier.{ ClassifierUtils, JointTrainSparseNetwork } +import edu.illinois.cs.cogcomp.saul.classifier.{ JointTrainSparseNetwork, ClassifierUtils } import edu.illinois.cs.cogcomp.saulexamples.EntityMentionRelation.datastruct.ConllRelation import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationClassifiers._ import edu.illinois.cs.cogcomp.saulexamples.nlp.EntityRelation.EntityRelationConstrainedClassifiers._ @@ -23,8 +23,8 @@ class EntityRelationTests extends FlatSpec with Matchers { PersonClassifier, OrganizationClassifier, LocationClassifier ) val scores = List(PersonClassifier.test(), OrganizationClassifier.test(), LocationClassifier.test()) - scores.foreach { case score => (score.average.f1 > minScore) should be(true) } - scores.foreach { case score => (score.overall.f1 > minScore) should be(true) } + scores.foreach { score => (score.average.f1 > minScore) should be(true) } + scores.foreach { score => (score.overall.f1 > minScore) should be(true) } } "independent relation classifier " should " work. " in { @@ -35,8 +35,8 @@ class EntityRelationTests extends FlatSpec with Matchers { ) val scores = List(WorksForClassifier.test(), LivesInClassifier.test(), LocatedInClassifier.test(), OrgBasedInClassifier.test()) - scores.foreach { case score => (score.average.f1 > minScore) should be(true) } - scores.foreach { case score => (score.overall.f1 > minScore) should be(true) } + scores.foreach { score => (score.average.f1 > minScore) should be(true) } + scores.foreach { score => (score.overall.f1 > minScore) should be(true) } } "pipeline relation classifiers " should " work. " in { @@ -47,8 +47,8 @@ class EntityRelationTests extends FlatSpec with Matchers { WorksForClassifierPipeline, LivesInClassifierPipeline ) val scores = List(WorksForClassifierPipeline.test(), LivesInClassifierPipeline.test()) - scores.foreach { case score => (score.average.f1 > minScore) should be(true) } - scores.foreach { case score => (score.overall.f1 > minScore) should be(true) } + scores.foreach { score => (score.average.f1 > minScore) should be(true) } + scores.foreach { score => (score.overall.f1 > minScore) should be(true) } } "L+I entity-relation classifiers " should " work. " in { @@ -58,25 +58,35 @@ class EntityRelationTests extends FlatSpec with Matchers { PersonClassifier, OrganizationClassifier, LocationClassifier, WorksForClassifier, LivesInClassifier, LocatedInClassifier, OrgBasedInClassifier ) - val scores = List(PerConstrainedClassifier.test(), WorksFor_PerOrg_ConstrainedClassifier.test()) - scores.foreach { case score => (score.average.f1 > minScore) should be(true) } - scores.foreach { case score => (score.overall.f1 > minScore) should be(true) } + val personClassifierScore = PerConstrainedClassifier.test() + personClassifierScore.perLabel.foreach(_.f1 should be > 0.8) + personClassifierScore.perLabel.foreach(_.f1 should be > 0.8) + + val orgConstrainedClassifierScore = OrgConstrainedClassifier.test() + orgConstrainedClassifierScore.perLabel.foreach(_.f1 should be > 0.75) + orgConstrainedClassifierScore.perLabel.foreach(_.f1 should be > 0.75) + + val worksForClassifierScore = WorksForRelationConstrainedClassifier.test() + worksForClassifierScore.perLabel.foreach(_.f1 should be > 0.9) + worksForClassifierScore.perLabel.foreach(_.f1 should be > 0.9) } "crossValidation on ER " should " work. " in { - EntityRelationDataModel.clearInstances + EntityRelationDataModel.clearInstances() sentences.populate(EntityRelationSensors.sentencesSmallSetTest) PersonClassifier.crossValidation(5) val results = PersonClassifier.crossValidation(5) - results.foreach { case score => (score.overall.f1 > minScore) should be(true) } + results.foreach { score => (score.overall.f1 > minScore) should be(true) } } + "Initialization on ER " should "work." in { EntityRelationDataModel.clearInstances() EntityRelationDataModel.populateWithConllSmallSet() val cls_base = List(PersonClassifier, OrganizationClassifier, LocationClassifier, WorksForClassifier, LivesInClassifier) - val cls = List(PerConstrainedClassifier, OrgConstrainedClassifier, LocConstrainedClassifier, WorksFor_PerOrg_ConstrainedClassifier, LivesIn_PerOrg_relationConstrainedClassifier) + val cls = List(PerConstrainedClassifier, OrgConstrainedClassifier, LocConstrainedClassifier, + WorksForRelationConstrainedClassifier, LivesInRelationConstrainedClassifier) ClassifierUtils.ForgetAll(cls_base: _*) @@ -98,6 +108,5 @@ class EntityRelationTests extends FlatSpec with Matchers { JointTrainSparseNetwork.train[ConllRelation](pairs, cls, jointTrainIteration, init = true) PerConstrainedClassifier.onClassifier.classifier.asInstanceOf[SparseNetworkLearner].getNetwork.get(0).asInstanceOf[LinearThresholdUnit].getWeightVector.size() should be(81) - } -} \ No newline at end of file +} diff --git a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/ConstraintsTest.scala b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/ConstraintsTest.scala deleted file mode 100644 index db244ae2..00000000 --- a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/ConstraintsTest.scala +++ /dev/null @@ -1,175 +0,0 @@ -/** This software is released under the University of Illinois/Research and Academic Use License. See - * the LICENSE file in the root folder for details. Copyright (c) 2016 - * - * Developed by: The Cognitive Computations Group, University of Illinois at Urbana-Champaign - * http://cogcomp.cs.illinois.edu/ - */ -package edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling - -import edu.illinois.cs.cogcomp.core.datastructures.ViewNames -import edu.illinois.cs.cogcomp.core.datastructures.textannotation.{ Constituent, Relation, TextAnnotation } -import edu.illinois.cs.cogcomp.core.datastructures.trees.Tree -import edu.illinois.cs.cogcomp.core.utilities.DummyTextAnnotationGenerator -import edu.illinois.cs.cogcomp.lbjava.infer.{ FirstOrderConstant, FirstOrderConstraint } -import edu.illinois.cs.cogcomp.lbjava.learn.SparseNetworkLearner -import edu.illinois.cs.cogcomp.saul.classifier.{ ConstrainedClassifier, Learnable } -import edu.illinois.cs.cogcomp.saul.constraint.ConstraintTypeConversion._ -import edu.illinois.cs.cogcomp.saul.datamodel.DataModel -import edu.illinois.cs.cogcomp.saulexamples.nlp.CommonSensors._ -import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLClassifiers.argumentTypeLearner -import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLSensors._ -import org.scalatest.{ FlatSpec, Matchers } -import scala.collection.JavaConversions._ - -class ConstraintsTest extends FlatSpec with Matchers { - object TestTextAnnotation extends DataModel { - val predicates = node[Constituent]((x: Constituent) => x.getTextAnnotation.getCorpusId + ":" + x.getTextAnnotation.getId + ":" + x.getSpan) - - val arguments = node[Constituent]((x: Constituent) => x.getTextAnnotation.getCorpusId + ":" + x.getTextAnnotation.getId + ":" + x.getSpan) - - val relations = node[Relation]((x: Relation) => "S" + x.getSource.getTextAnnotation.getCorpusId + ":" + x.getSource.getTextAnnotation.getId + ":" + x.getSource.getSpan + - "D" + x.getTarget.getTextAnnotation.getCorpusId + ":" + x.getTarget.getTextAnnotation.getId + ":" + x.getTarget.getSpan) - - val sentences = node[TextAnnotation]((x: TextAnnotation) => x.getCorpusId + ":" + x.getId) - - val trees = node[Tree[Constituent]] - - val stringTree = node[Tree[String]] - - val tokens = node[Constituent]((x: Constituent) => x.getTextAnnotation.getCorpusId + ":" + x.getTextAnnotation.getId + ":" + x.getSpan) - - val sentencesToTrees = edge(sentences, trees) - val sentencesToStringTree = edge(sentences, stringTree) - val sentencesToTokens = edge(sentences, tokens) - val sentencesToRelations = edge(sentences, relations) - val relationsToPredicates = edge(relations, predicates) - val relationsToArguments = edge(relations, arguments) - - sentencesToRelations.addSensor(textAnnotationToRelation _) - sentencesToRelations.addSensor(textAnnotationToRelationMatch _) - relationsToArguments.addSensor(relToArgument _) - relationsToPredicates.addSensor(relToPredicate _) - sentencesToStringTree.addSensor(textAnnotationToStringTree _) - val posTag = property(predicates, "posC") { - x: Constituent => getPosTag(x) - } - val argumentLabelGold = property(relations, "l") { - r: Relation => r.getRelationName - } - } - - import TestTextAnnotation._ - - object ArgumentTypeLearner extends Learnable[Relation](relations) { - def label = argumentLabelGold - - override lazy val classifier = new SparseNetworkLearner() - } - - object TestConstraints { - - val r_arg_Constraint = ConstrainedClassifier.constraint[TextAnnotation] { - var a: FirstOrderConstraint = null - x: TextAnnotation => { - a = new FirstOrderConstant(true) - val values = Array("R-A1", "R-A2", "R-A3", "R-A4", "R-A5", "R-AA", "R-AM-ADV", "R-AM-CAU", "R-AM-EXT", "R-AM-LOC", "R-AM-MNR", "R-AM-PNC") - (sentences(x) ~> sentencesToRelations ~> relationsToPredicates).foreach { - y => - { - val argCandList = (predicates(y) ~> -relationsToPredicates).toList - argCandList.foreach { - t: Relation => - { - for (i <- 0 until values.length) - a = a and new FirstOrderConstant((argumentTypeLearner.classifier.getLabeler.discreteValue(t).equals(values(i)))) ==> - argCandList.filterNot(x => x.equals(t))._exists { - k: Relation => new FirstOrderConstant((argumentTypeLearner.classifier.getLabeler.discreteValue(k)).equals(values(i).substring(2))) - } - } - a - } - } - } - } - a - } // end r-arg constraint - - val c_arg_Constraint = ConstrainedClassifier.constraint[TextAnnotation] { - var a: FirstOrderConstraint = null - x: TextAnnotation => { - a = new FirstOrderConstant(true) - val values = Array("C-A1", "C-A2", "C-A3", "C-A4", "C-A5", "C-AA", "C-AM-DIR", "C-AM-LOC", "C-AM-MNR", "C-AM-NEG", "C-AM-PNC") - (sentences(x) ~> sentencesToRelations ~> relationsToPredicates).foreach { - y => - { - val argCandList = (predicates(y) ~> -relationsToPredicates).toList - val sortedCandidates = argCandList.sortBy(x => x.getTarget.getStartSpan) - sortedCandidates.zipWithIndex.foreach { - case (t, ind) => { - if (ind > 0) - for (i <- 0 until values.length) - a = a and new FirstOrderConstant((argumentTypeLearner.classifier.getLabeler.discreteValue(t).equals(values(i)))) ==> - sortedCandidates.subList(0, ind)._exists { - k: Relation => new FirstOrderConstant((argumentTypeLearner.classifier.getLabeler.discreteValue(k)).equals(values(i).substring(2))) - } - } - } - } - } - } - a - } - - val noDuplicate = ConstrainedClassifier.constraint[TextAnnotation] { - // Predicates have at most one argument of each type i.e. there shouldn't be any two arguments with the same type for each predicate - val values = Array("A0", "A1", "A2", "A3", "A4", "A5", "AA") - var a: FirstOrderConstraint = null - x: TextAnnotation => { - a = new FirstOrderConstant(true) - (sentences(x) ~> sentencesToRelations ~> relationsToPredicates).foreach { - y => - { - val argCandList = (predicates(y) ~> -relationsToPredicates).toList - for (t1 <- 0 until argCandList.size - 1) { - for (t2 <- t1 + 1 until argCandList.size) { - a = a and (new FirstOrderConstant(values.contains(argumentTypeLearner.classifier.getLabeler.discreteValue(argCandList.get(t1)))) ==> new FirstOrderConstant(argumentTypeLearner.classifier.getLabeler.discreteValue(argCandList.get(t1)).ne(argumentTypeLearner.classifier.getLabeler.discreteValue(argCandList.get(t2))))) - } - } - } - } - a - } - } - } - - val viewsToAdd = Array(ViewNames.LEMMA, ViewNames.POS, ViewNames.SHALLOW_PARSE, ViewNames.PARSE_GOLD, ViewNames.SRL_VERB) - val ta: TextAnnotation = DummyTextAnnotationGenerator.generateAnnotatedTextAnnotation(viewsToAdd, true, 1) - - import TestConstraints._ - import TestTextAnnotation._ - sentencesToTokens.addSensor(textAnnotationToTokens _) - sentences.populate(Seq(ta)) - val predicateTrainCandidates = tokens.getTrainingInstances.filter((x: Constituent) => posTag(x).startsWith("IN")) - .map(c => c.cloneForNewView(ViewNames.SRL_VERB)) - predicates.populate(predicateTrainCandidates) - val XuPalmerCandidateArgsTraining = predicates.getTrainingInstances.flatMap(x => xuPalmerCandidate(x, (sentences(x.getTextAnnotation) ~> sentencesToStringTree).head)) - sentencesToRelations.addSensor(textAnnotationToRelationMatch _) - relations.populate(XuPalmerCandidateArgsTraining) - - "manually defined has codes" should "avoid duplications in edges and reverse edges" in { - predicates().size should be((relations() ~> relationsToPredicates).size) - (predicates() ~> -relationsToPredicates).size should be(relations().size) - (predicates(predicates().head) ~> -relationsToPredicates).size should be(4) - } - - "the no duplicate constraint" should "be true" in { - noDuplicate(ta).evaluate() should be(true) - } - "the r-arg constraint" should "be true" in { - r_arg_Constraint(ta).evaluate() should be(true) - } - - "the c-arg constraint" should "be true" in { - c_arg_Constraint(ta).evaluate() should be(true) - } -} \ No newline at end of file diff --git a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/ModelsTest.scala b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/ModelsTest.scala index 452878d5..70d7fa6e 100644 --- a/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/ModelsTest.scala +++ b/saul-examples/src/test/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/ModelsTest.scala @@ -8,10 +8,10 @@ package edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling import edu.illinois.cs.cogcomp.saul.classifier.ClassifierUtils import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLClassifiers._ +import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLConstrainedClassifiers.ArgTypeConstrainedClassifier import org.scalatest.{ FlatSpec, Matchers } class ModelsTest extends FlatSpec with Matchers { - "argument type classifier (aTr)" should "work." in { ClassifierUtils.LoadClassifier(SRLConfigurator.SRL_JAR_MODEL_PATH.value + "/models_aTr/", argumentTypeLearner) val results = argumentTypeLearner.test(exclude = "candidate") @@ -37,20 +37,20 @@ class ModelsTest extends FlatSpec with Matchers { } } - "L+I argument type classifier (aTr)" should "work." in { - //TODO solve the test problem with Gurobi licencing vs. OJalgoHook inefficiency - // ClassifierUtils.LoadClassifier(SRLConfigurator.SRL_JAR_MODEL_PATH.value + "/models_aTr/", argumentTypeLearner) - // val scores = argTypeConstraintClassifier.test(exclude = "candidate") - // scores.foreach { - // case (label, score) => { - // label match { - // case "A0" => (score._1 >= 0.9) should be(true) - // case "A1" => (score._1 >= 0.9) should be(true) - // case "A2" => (score._1 >= 0.6) should be(true) - // case _ => "" - // } - // } - // } + //TODO fix the issue with OjAlgo, and un-ignore this + "L+I argument type classifier (cTr)" should "work." ignore { + ClassifierUtils.LoadClassifier(SRLConfigurator.SRL_JAR_MODEL_PATH.value + "/models_cTr/", argumentTypeLearner) + val scores = ArgTypeConstrainedClassifier.test(exclude = "candidate") + println("scores = ") + println(scores) + scores.perLabel.foreach { resultPerLabel => + resultPerLabel.label match { + case "A0" => resultPerLabel.f1 should be(0.96 +- 0.02) + case "A1" => resultPerLabel.f1 should be(0.93 +- 0.02) + case "A2" => resultPerLabel.f1 should be(0.85 +- 0.02) + case _ => "" + } + } } "argument identifier (bTr)" should "perform higher than 0.95." in {