From d75a4f683a4aab5fed78438ad2df612c71dcf8ed Mon Sep 17 00:00:00 2001 From: Bhargav Mangipudi Date: Sat, 8 Apr 2017 21:01:58 -0500 Subject: [PATCH 1/2] Basic feature vector caching. --- .../cogcomp/saul/classifier/Learnable.scala | 2 +- .../cs/cogcomp/saul/datamodel/DataModel.scala | 26 ++++++- .../cs/cogcomp/saul/datamodel/node/Node.scala | 4 +- .../property/CombinedDiscreteProperty.scala | 22 +++++- .../saul/datamodel/property/Property.scala | 2 + .../cogcomp/saul/datamodel/propertyTest.scala | 51 +++++++++++++ .../SRLMultiGraphDataModel.scala | 76 +++++++++++-------- 7 files changed, 142 insertions(+), 41 deletions(-) diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/Learnable.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/Learnable.scala index ecb0473c..782cfcc1 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/Learnable.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/Learnable.scala @@ -18,7 +18,7 @@ import edu.illinois.cs.cogcomp.lbjava.parse.FoldParser.SplitPolicy import edu.illinois.cs.cogcomp.lbjava.parse.{ FoldParser, Parser } import edu.illinois.cs.cogcomp.saul.datamodel.DataModel import edu.illinois.cs.cogcomp.saul.datamodel.edge.Link -import edu.illinois.cs.cogcomp.saul.datamodel.node.Node +import edu.illinois.cs.cogcomp.saul.datamodel.node.{ Node, NodeProperty } import edu.illinois.cs.cogcomp.saul.datamodel.property.{ CombinedDiscreteProperty, Property, PropertyWithWindow } import edu.illinois.cs.cogcomp.saul.lbjrelated.LBJLearnerEquivalent import edu.illinois.cs.cogcomp.saul.parser.{ IterableToLBJavaParser, LBJavaParserToIterable } diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/DataModel.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/DataModel.scala index 7ec1430d..5ab5e03e 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/DataModel.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/DataModel.scala @@ -160,7 +160,13 @@ trait DataModel extends Logging { e } - class PropertyApply[T <: AnyRef] private[DataModel] (val node: Node[T], name: String, cache: Boolean, ordered: Boolean) { + class PropertyApply[T <: AnyRef] private[DataModel] ( + val node: Node[T], + name: String, + cache: Boolean, + ordered: Boolean, + cacheFeatureVector: Boolean + ) { papply => // TODO(danielk): make the hashmaps immutable @@ -174,7 +180,11 @@ trait DataModel extends Logging { def apply(f: T => Boolean)(implicit tag: ClassTag[T]): BooleanProperty[T] = { def cachedF = if (cache) { x: T => getOrUpdate(x, f).asInstanceOf[Boolean] } else f - val a = new BooleanProperty[T](name, cachedF) with NodeProperty[T] { override def node: Node[T] = papply.node } + val a = new BooleanProperty[T](name, cachedF) with NodeProperty[T] { + override def node: Node[T] = papply.node + override val cacheFeatureVector: Boolean = papply.cacheFeatureVector + } + papply.node.properties += a properties += a a @@ -186,10 +196,12 @@ trait DataModel extends Logging { val a = if (ordered) { new RealArrayProperty[T](name, newf) with NodeProperty[T] { override def node: Node[T] = papply.node + override val cacheFeatureVector: Boolean = papply.cacheFeatureVector } } else { new RealGenProperty[T](name, newf) with NodeProperty[T] { override def node: Node[T] = papply.node + override val cacheFeatureVector: Boolean = papply.cacheFeatureVector } } papply.node.properties += a @@ -203,6 +215,7 @@ trait DataModel extends Logging { val newf: T => Double = { t => cachedF(t).toDouble } val a = new RealProperty[T](name, newf) with NodeProperty[T] { override def node: Node[T] = papply.node + override val cacheFeatureVector: Boolean = papply.cacheFeatureVector } papply.node.properties += a properties += a @@ -215,6 +228,7 @@ trait DataModel extends Logging { def cachedF = if (cache) { x: T => getOrUpdate(x, f).asInstanceOf[List[Double]] } else f val a = new RealCollectionProperty[T](name, cachedF, ordered) with NodeProperty[T] { override def node: Node[T] = papply.node + override val cacheFeatureVector: Boolean = papply.cacheFeatureVector } papply.node.properties += a properties += a @@ -227,6 +241,7 @@ trait DataModel extends Logging { def cachedF = if (cache) { x: T => getOrUpdate(x, f).asInstanceOf[Double] } else f val a = new RealProperty[T](name, cachedF) with NodeProperty[T] { override def node: Node[T] = papply.node + override val cacheFeatureVector: Boolean = papply.cacheFeatureVector } papply.node.properties += a properties += a @@ -239,6 +254,7 @@ trait DataModel extends Logging { def cachedF = if (cache) { x: T => getOrUpdate(x, f).asInstanceOf[String] } else f val a = new DiscreteProperty[T](name, cachedF, None) with NodeProperty[T] { override def node: Node[T] = papply.node + override val cacheFeatureVector: Boolean = papply.cacheFeatureVector } papply.node.properties += a properties += a @@ -251,6 +267,7 @@ trait DataModel extends Logging { def cachedF = if (cache) { x: T => getOrUpdate(x, f).asInstanceOf[List[String]] } else f val a = new DiscreteCollectionProperty[T](name, cachedF, !ordered) with NodeProperty[T] { override def node: Node[T] = papply.node + override val cacheFeatureVector: Boolean = papply.cacheFeatureVector } papply.node.properties += a properties += a @@ -265,6 +282,7 @@ trait DataModel extends Logging { val r = range.toList val a = new DiscreteProperty[T](name, cachedF, Some(r)) with NodeProperty[T] { override def node: Node[T] = papply.node + override val cacheFeatureVector: Boolean = papply.cacheFeatureVector } papply.node.properties += a properties += a @@ -272,8 +290,8 @@ trait DataModel extends Logging { } } - def property[T <: AnyRef](node: Node[T], name: String = "prop" + properties.size, cache: Boolean = false, ordered: Boolean = false) = - new PropertyApply[T](node, name, cache, ordered) + def property[T <: AnyRef](node: Node[T], name: String = "prop" + properties.size, cache: Boolean = false, ordered: Boolean = false, cacheFeatureVector: Boolean = false) = + new PropertyApply[T](node, name, cache, ordered, cacheFeatureVector) /** Methods for caching Data Model */ var hasDerivedInstances = false diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/node/Node.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/node/Node.scala index 6ba00ef9..dc76f13c 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/node/Node.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/node/Node.scala @@ -312,8 +312,10 @@ class Node[T <: AnyRef](val keyFunc: T => Any = (x: T) => x, val tag: ClassTag[T } } + private[saul] final val propertyFeatureVectorCache = new mutable.WeakHashMap[T, mutable.HashMap[Property[_], FeatureVector]]() + /** list of hashmaps used inside properties for caching sensor values */ - final val propertyCacheList = new ListBuffer[MutableHashMap[_, Any]]() + private[saul] final val propertyCacheList = new ListBuffer[MutableHashMap[_, Any]]() def clearPropertyCache(): Unit = { if (propertyCacheList.nonEmpty) { diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/CombinedDiscreteProperty.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/CombinedDiscreteProperty.scala index fbe65ee8..21ffbd81 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/CombinedDiscreteProperty.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/CombinedDiscreteProperty.scala @@ -6,10 +6,12 @@ */ package edu.illinois.cs.cogcomp.saul.datamodel.property -import java.util - import edu.illinois.cs.cogcomp.lbjava.classify.{ Classifier, FeatureVector } +import edu.illinois.cs.cogcomp.saul.datamodel.node.NodeProperty + +import java.util +import scala.collection.mutable import scala.reflect.ClassTag case class CombinedDiscreteProperty[T <: AnyRef](atts: List[Property[T]])(implicit val tag: ClassTag[T]) extends TypedProperty[T, List[_]] { @@ -26,7 +28,21 @@ case class CombinedDiscreteProperty[T <: AnyRef](atts: List[Property[T]])(implic override def featureVector(instance: T): FeatureVector = { val featureVector = new FeatureVector() - atts.foreach(property => featureVector.addFeatures(property.featureVector(instance))) + atts.foreach(property => { + val extractedFeatureVector = { + // Handle caching of Feature Vector + if (property.cacheFeatureVector && property.isInstanceOf[NodeProperty[T]]) { + val nodeProperty = property.asInstanceOf[NodeProperty[T]] + val instanceCacheMap = nodeProperty.node.propertyFeatureVectorCache + .getOrElseUpdate(instance, new mutable.HashMap[Property[_], FeatureVector]()) + instanceCacheMap.getOrElseUpdate(property, property.featureVector(instance)) + } else { + property.featureVector(instance) + } + } + + featureVector.addFeatures(extractedFeatureVector) + }) featureVector } diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/Property.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/Property.scala index 53480da9..49059f2d 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/Property.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/Property.scala @@ -28,6 +28,8 @@ trait Property[T] { val sensor: T => S + val cacheFeatureVector: Boolean = false + def apply(instance: T): S = sensor(instance) def featureVector(instance: T): FeatureVector diff --git a/saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/datamodel/propertyTest.scala b/saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/datamodel/propertyTest.scala index 1de8c5b3..28589963 100644 --- a/saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/datamodel/propertyTest.scala +++ b/saul-core/src/test/scala/edu/illinois/cs/cogcomp/saul/datamodel/propertyTest.scala @@ -6,6 +6,8 @@ */ package edu.illinois.cs.cogcomp.saul.datamodel +import edu.illinois.cs.cogcomp.lbjava.learn.{ SparseNetworkLearner, SparsePerceptron } +import edu.illinois.cs.cogcomp.saul.classifier.Learnable import edu.illinois.cs.cogcomp.saul.datamodel.property.PairwiseConjunction import org.scalatest._ @@ -56,6 +58,55 @@ class propertyTest extends FlatSpec with Matchers { Set(p1.name, p2.name, p3.name).size should be(3) } } + + "property feature vector caching" should "work" in { + var counterNonStatic: Int = 0 + var counterStatic: Int = 0 + + object testModel extends DataModel { + val n = node[String] + + val testLabel = property(n) { s: String => s } + val nonStaticProperty = property(n) { s: String => counterNonStatic += 1; s } + val staticProperty = property(n, cacheFeatureVector = true) { s: String => counterStatic += 1; s } + } + + import testModel._ + + object testClassifier extends Learnable[String](n) { + def label = testLabel + override lazy val classifier = new SparseNetworkLearner() + override def feature = using(staticProperty, nonStaticProperty) + } + + val dataset = List("test", "testSecond") + + n.populate(dataset) + testClassifier.learn(5) + + counterNonStatic should be(10) // 5 iterations * 2 items + counterStatic should be(2) // 2 items + + // To indicate that sensor values are not cached + nonStaticProperty("test") + staticProperty("test") + + counterNonStatic should be(11) + counterStatic should be(3) + + // To indicate that feature vectors are only cached in the training workflow + // Accessing featureVector directly does not return cached value. + nonStaticProperty.featureVector("test") + staticProperty.featureVector("test") + + counterNonStatic should be(12) + counterStatic should be(4) + + testClassifier.learn(5) + + counterNonStatic should be(22) + counterStatic should be(4) + } } object toyDataModel extends DataModel { diff --git a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLMultiGraphDataModel.scala b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLMultiGraphDataModel.scala index 3945d63d..16eb68c8 100644 --- a/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLMultiGraphDataModel.scala +++ b/saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/SemanticRoleLabeling/SRLMultiGraphDataModel.scala @@ -21,7 +21,18 @@ import edu.illinois.cs.cogcomp.saulexamples.nlp.SemanticRoleLabeling.SRLConstrai import scala.collection.JavaConversions._ -class SRLMultiGraphDataModel(parseViewName: String = null, frameManager: SRLFrameManager = null) extends DataModel { +/** Data Model graph for the Semantic Role Labeling task. Represents the data model and feature definitions. + * + * @param parseViewName View name for the gold SRL annotation. + * @param frameManager Instance of the SRL Frame Manager. + * @param cacheStaticFeatures Boolean to decide if static features should be cached during experimentation. Caching + * feature vectors trades off RAM usage with feature extraction time. + */ +class SRLMultiGraphDataModel( + parseViewName: String = null, + frameManager: SRLFrameManager = null, + cacheStaticFeatures: Boolean = false +) extends DataModel { val predicates = node[Constituent]((x: Constituent) => x.getTextAnnotation.getCorpusId + ":" + x.getTextAnnotation.getId + ":" + x.getSpan) @@ -55,7 +66,7 @@ class SRLMultiGraphDataModel(parseViewName: String = null, frameManager: SRLFram } // Classification labels - val isPredicateGold = property(predicates, "p") { + val isPredicateGold = property(predicates, "p", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => x.getLabel.equals("Predicate") } val predicateSenseGold = property(predicates, "s") { @@ -70,26 +81,27 @@ class SRLMultiGraphDataModel(parseViewName: String = null, frameManager: SRLFram } // Features properties - val posTag = property(predicates, "posC") { + val posTag = property(predicates, "posC", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => getPOS(x) } - val subcategorization = property(predicates, "subcatC") { + val subcategorization = property(predicates, "subcatC", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => fexFeatureExtractor(x, new SubcategorizationFrame(parseViewName)) } - val phraseType = property(predicates, "phraseTypeC") { + val phraseType = property(predicates, "phraseTypeC", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => fexFeatureExtractor(x, new ParsePhraseType(parseViewName)) } - val headword = property(predicates, "headC") { + val headword = property(predicates, "headC", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => fexFeatureExtractor(x, new ParseHeadWordPOS(parseViewName)) } - val syntacticFrame = property(predicates, "synFrameC") { + val syntacticFrame = property(predicates, "synFrameC", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => fexFeatureExtractor(x, new SyntacticFrame(parseViewName)) } - val path = property(predicates, "pathC") { + + val path = property(predicates, "pathC", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => fexFeatureExtractor(x, new ParsePath(parseViewName)) } @@ -97,100 +109,100 @@ class SRLMultiGraphDataModel(parseViewName: String = null, frameManager: SRLFram // x: Relation => fexFeatureExtractor(x.getTarget, new SubcategorizationFrame(parseViewName)) // } - val phraseTypeRelation = property(relations, "phraseType") { + val phraseTypeRelation = property(relations, "phraseType", cacheFeatureVector = cacheStaticFeatures) { x: Relation => fexFeatureExtractor(x.getTarget, new ParsePhraseType(parseViewName)) } - val headwordRelation = property(relations, "head") { + val headwordRelation = property(relations, "head", cacheFeatureVector = cacheStaticFeatures) { x: Relation => fexFeatureExtractor(x.getTarget, new ParseHeadWordPOS(parseViewName)) } - val syntacticFrameRelation = property(relations, "synFrame") { + val syntacticFrameRelation = property(relations, "synFrame", cacheFeatureVector = cacheStaticFeatures) { x: Relation => fexFeatureExtractor(x.getTarget, new SyntacticFrame(parseViewName)) } - val pathRelation = property(relations, "path") { + val pathRelation = property(relations, "path", cacheFeatureVector = cacheStaticFeatures) { x: Relation => fexFeatureExtractor(x.getTarget, new ParsePath(parseViewName)) } - val predPosTag = property(relations, "pPos") { + val predPosTag = property(relations, "pPos", cacheFeatureVector = cacheStaticFeatures) { x: Relation => getPOS(x.getSource) } - val predLemmaR = property(relations, "pLem") { + val predLemmaR = property(relations, "pLem", cacheFeatureVector = cacheStaticFeatures) { x: Relation => getLemma(x.getSource) } - val predLemmaP = property(predicates, "pLem") { + val predLemmaP = property(predicates, "pLem", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => getLemma(x) } - val linearPosition = property(relations, "position") { + val linearPosition = property(relations, "position", cacheFeatureVector = cacheStaticFeatures) { x: Relation => fexFeatureExtractor(x.getTarget, new LinearPosition()) } - val voice = property(predicates, "voice") { + val voice = property(predicates, "voice", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => fexFeatureExtractor(x, new VerbVoiceIndicator(parseViewName)) } - val predWordWindow = property(predicates, "predWordWindow") { + val predWordWindow = property(predicates, "predWordWindow", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => fexContextFeats(x, WordFeatureExtractorFactory.word) } - val predPOSWindow = property(predicates, "predPOSWindow") { + val predPOSWindow = property(predicates, "predPOSWindow", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => fexContextFeats(x, WordFeatureExtractorFactory.pos) } - val argWordWindow = property(relations, "argWordWindow") { + val argWordWindow = property(relations, "argWordWindow", cacheFeatureVector = cacheStaticFeatures) { rel: Relation => fexContextFeats(rel.getTarget, WordFeatureExtractorFactory.word) } - val argPOSWindow = property(relations, "argPOSWindow") { + val argPOSWindow = property(relations, "argPOSWindow", cacheFeatureVector = cacheStaticFeatures) { rel: Relation => fexContextFeats(rel.getTarget, WordFeatureExtractorFactory.pos) } - val verbClass = property(predicates, "verbClass") { + val verbClass = property(predicates, "verbClass", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => frameManager.getAllClasses(getLemma(x)).toList } - val constituentLength = property(relations, "constLength") { + val constituentLength = property(relations, "constLength", cacheFeatureVector = cacheStaticFeatures) { rel: Relation => rel.getTarget.getEndSpan - rel.getTarget.getStartSpan } - val chunkLength = property(relations, "chunkLength") { + val chunkLength = property(relations, "chunkLength", cacheFeatureVector = cacheStaticFeatures) { rel: Relation => rel.getTarget.getTextAnnotation.getView(ViewNames.SHALLOW_PARSE).getConstituentsCovering(rel.getTarget).length } - val chunkEmbedding = property(relations, "chunkEmbedding") { + val chunkEmbedding = property(relations, "chunkEmbedding", cacheFeatureVector = cacheStaticFeatures) { rel: Relation => fexFeatureExtractor(rel.getTarget, new ChunkEmbedding(ViewNames.SHALLOW_PARSE)) } - val chunkPathPattern = property(relations, "chunkPath") { + val chunkPathPattern = property(relations, "chunkPath", cacheFeatureVector = cacheStaticFeatures) { rel: Relation => fexFeatureExtractor(rel.getTarget, new ChunkPathPattern(ViewNames.SHALLOW_PARSE)) } /** Combines clause relative position and clause coverage */ - val clauseFeatures = property(relations, "clauseFeats") { + val clauseFeatures = property(relations, "clauseFeats", cacheFeatureVector = cacheStaticFeatures) { rel: Relation => val clauseViewName = if (parseViewName.equals(ViewNames.PARSE_GOLD)) "CLAUSES_GOLD" else ViewNames.CLAUSES_STANFORD fexFeatureExtractor(rel.getTarget, new ClauseFeatureExtractor(parseViewName, clauseViewName)) } - val containsNEG = property(relations, "containsNEG") { + val containsNEG = property(relations, "containsNEG", cacheFeatureVector = cacheStaticFeatures) { rel: Relation => fexFeatureExtractor(rel.getTarget, ChunkPropertyFeatureFactory.isNegated) } - val containsMOD = property(relations, "containsMOD") { + val containsMOD = property(relations, "containsMOD", cacheFeatureVector = cacheStaticFeatures) { rel: Relation => fexFeatureExtractor(rel.getTarget, ChunkPropertyFeatureFactory.hasModalVerb) } // Frame properties - val legalSenses = property(relations, "legalSens") { + val legalSenses = property(relations, "legalSens", cacheFeatureVector = cacheStaticFeatures) { x: Relation => frameManager.getLegalSenses(predLemmaR(x)).toList } - val legalArguments = property(predicates, "legalArgs") { + val legalArguments = property(predicates, "legalArgs", cacheFeatureVector = cacheStaticFeatures) { x: Constituent => frameManager.getLegalArguments(predLemmaP(x)).toList } @@ -245,7 +257,7 @@ class SRLMultiGraphDataModel(parseViewName: String = null, frameManager: SRLFram } a } - val propertyConjunction = property(relations) { + val propertyConjunction = property(relations, cacheFeatureVector = cacheStaticFeatures) { x: Relation => PairwiseConjunction(List(containsMOD, containsNEG), x) } From 08dec8bf915845422680954042e9e5aa6fb76b6e Mon Sep 17 00:00:00 2001 From: Bhargav Mangipudi Date: Sat, 8 Apr 2017 23:10:13 -0500 Subject: [PATCH 2/2] More documentation and minor changes. --- saul-core/doc/CONCEPTUALSTRUCTURE.md | 21 +++++++--- .../cogcomp/saul/classifier/Learnable.scala | 8 +++- .../cs/cogcomp/saul/datamodel/DataModel.scala | 42 +++++++++++++++++++ .../cs/cogcomp/saul/datamodel/node/Node.scala | 10 ++++- .../property/CombinedDiscreteProperty.scala | 11 ++++- 5 files changed, 80 insertions(+), 12 deletions(-) diff --git a/saul-core/doc/CONCEPTUALSTRUCTURE.md b/saul-core/doc/CONCEPTUALSTRUCTURE.md index bfa4523b..2432e2ce 100644 --- a/saul-core/doc/CONCEPTUALSTRUCTURE.md +++ b/saul-core/doc/CONCEPTUALSTRUCTURE.md @@ -37,19 +37,28 @@ In this definition `pos` is defined to be a property of nodes of type token. The inside `{ .... }` is the definition of a sensor which given an object of type `ConllRawToken` i.e. the tye of node and generates an output property value (in this case, using the POS tag of an object of type `ConllRawToken`). -If the content of a property is computationally intensive to compute, you can cache its value, by setting `cache` to be +If the content of a property is computationally intensive to compute, you can cache its Feature Vector, by setting `cacheFeatureVector` to be `true`: ```scala -val pos = property(token, cache = true) { +val pos = property(token, cacheFeatureVector = true) { (t: ConllRawToken) => t.POS } ``` -The first time that a property is called with a specific value, it would you remember the corresponding output, -so next time it just looks up the value from the cache. +During the first training iteration, the feature vector is computed and cached during further iterations of training/testing. This value is cached in-memory for the lifetime of the app. Using this feature judiciously and make sure you have enough free memory (RAM) available. -Note that when training, the property cache is remove between two training interation in order not to interrupt -the trainng procedure. +If you want to cache the value of a feature during a single iteration, use the `cache` parameter. + +The `cache` parameter allows the value to be cached within a training/testing iteration. This is useful if you one of your features depends on evaluation of a Classifier on other instances as well. This recursive evaluation of the Classifier might be expensive and caching would speed-up performance. Look at a sample usage of this parameter in the [POSTagging Example](../../saul-examples/src/main/scala/edu/illinois/cs/cogcomp/saulexamples/nlp/POSTagger/POSDataModel.scala#L66). + + Usage: + ```scala + val posWindow = property(token, cache = true) { + (t: ConllRawToken) => t.getNeighbors.map(n => posWindow(n)) + } + ``` + + The value of these properties are cleared at the end of each training iteration. #### Parameterized properties Suppose you want to define properties which get some parameters; this can be important when we want to programmatically diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/Learnable.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/Learnable.scala index 782cfcc1..ee734a4d 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/Learnable.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/classifier/Learnable.scala @@ -51,8 +51,12 @@ abstract class Learnable[T <: AnyRef](val node: Node[T], val parameters: Paramet def feature: List[Property[T]] = node.properties.toList /** filter out the label from the features */ - def combinedProperties = if (label != null) new CombinedDiscreteProperty[T](this.feature.filterNot(_.name == label.name)) - else new CombinedDiscreteProperty[T](this.feature) + def combinedProperties = { + val features = if (label != null) this.feature.filterNot(_.name == label.name) else this.feature + + // Support Feature Vector Caching during training. + new CombinedDiscreteProperty[T](features, supportsFeatureVectorCaching = true) + } def lbpFeatures = Property.convertToClassifier(combinedProperties) diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/DataModel.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/DataModel.scala index 5ab5e03e..a1289ea4 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/DataModel.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/DataModel.scala @@ -160,6 +160,27 @@ trait DataModel extends Logging { e } + /** Helper class to facilitate creating new [[Property]] instances. + * + * Note: + * - The `cache` parameter is used to cache a property sensor's value within a single training iteration. This is + * useful if properties are defined recursively. + * - The `cacheFeatureVector` parameter caches the FeatureVector for instances of this property. Thus, feature + * extraction is performed only once during the training/testing process. This can lead to an increase in RAM + * usage but will lead to speed-up in training iterations. Recommended to use with static features which have high + * feature extraction effort. + * + * @param node [[Node]] instance to add the current property to. + * @param name Name of the property. + * @param cache Boolean indicating if this property sensor's value should be cached within training iterations. + * @param ordered Denoting if the order among the values in this property needs to be preserved. Only applies to + * collection based properties. + * @param cacheFeatureVector Boolean indicating if this property's feature vector should be cached during + * training/testing. Caching feature vector saves redundant feature extraction during + * training/testing. + * @tparam T Data type of the node that this property is associated with. + * @return [[PropertyApply]] instance + */ class PropertyApply[T <: AnyRef] private[DataModel] ( val node: Node[T], name: String, @@ -290,6 +311,27 @@ trait DataModel extends Logging { } } + /** Function to create a new [[Property]] instance inside a DataModel. + * + * Note: + * - The `cache` parameter is used to cache a property sensor's value within a single training iteration. This is + * useful if properties are defined recursively. + * - The `cacheFeatureVector` parameter caches the FeatureVector for instances of this property. Thus, feature + * extraction is performed only once during the training/testing process. This can lead to an increase in RAM + * usage but will lead to speed-up in training iterations. Recommended to use with static features which have high + * feature extraction effort. + * + * @param node [[Node]] instance to add the current property to. + * @param name Name of the property. + * @param cache Boolean indicating if this property sensor's value should be cached within training iterations. + * @param ordered Denoting if the order among the values in this property needs to be preserved. Only applies to + * collection based properties. + * @param cacheFeatureVector Boolean indicating if this property's feature vector should be cached during + * training/testing. Caching feature vector saves redundant feature extraction during + * training/testing. + * @tparam T Data type of the node that this property is associated with. + * @return Property instance wrapped in a helper class [[PropertyApply]] + */ def property[T <: AnyRef](node: Node[T], name: String = "prop" + properties.size, cache: Boolean = false, ordered: Boolean = false, cacheFeatureVector: Boolean = false) = new PropertyApply[T](node, name, cache, ordered, cacheFeatureVector) diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/node/Node.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/node/Node.scala index dc76f13c..d733cdf7 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/node/Node.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/node/Node.scala @@ -80,11 +80,16 @@ class Node[T <: AnyRef](val keyFunc: T => Any = (x: T) => x, val tag: ClassTag[T } def clear(): Unit = { + // Clear property caches + propertyCacheList.foreach(_.clear()) + propertyFeatureVectorCache.clear() + collection.clear trainingSet.clear testingSet.clear - for (e <- incoming) e.clear - for (e <- outgoing) e.clear + + for (e <- incoming) e.clear() + for (e <- outgoing) e.clear() } private var count: AtomicInteger = new AtomicInteger() @@ -312,6 +317,7 @@ class Node[T <: AnyRef](val keyFunc: T => Any = (x: T) => x, val tag: ClassTag[T } } + /** WeakHashMap instance to cache property's [[FeatureVector]] instances during training/testing */ private[saul] final val propertyFeatureVectorCache = new mutable.WeakHashMap[T, mutable.HashMap[Property[_], FeatureVector]]() /** list of hashmaps used inside properties for caching sensor values */ diff --git a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/CombinedDiscreteProperty.scala b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/CombinedDiscreteProperty.scala index 21ffbd81..d7141594 100644 --- a/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/CombinedDiscreteProperty.scala +++ b/saul-core/src/main/scala/edu/illinois/cs/cogcomp/saul/datamodel/property/CombinedDiscreteProperty.scala @@ -14,7 +14,14 @@ import java.util import scala.collection.mutable import scala.reflect.ClassTag -case class CombinedDiscreteProperty[T <: AnyRef](atts: List[Property[T]])(implicit val tag: ClassTag[T]) extends TypedProperty[T, List[_]] { +/** Represents a collection of properties. + * + * @param atts List of properties (attributes). + * @param supportsFeatureVectorCaching Boolean to denote if feature vector caching should be supported. + * @param tag ClassTag of the property's input type. + * @tparam T Property's input type. + */ +case class CombinedDiscreteProperty[T <: AnyRef](atts: List[Property[T]], supportsFeatureVectorCaching: Boolean = false)(implicit val tag: ClassTag[T]) extends TypedProperty[T, List[_]] { override val sensor: (T) => List[_] = { t: T => atts.map(att => att.sensor(t)) @@ -31,7 +38,7 @@ case class CombinedDiscreteProperty[T <: AnyRef](atts: List[Property[T]])(implic atts.foreach(property => { val extractedFeatureVector = { // Handle caching of Feature Vector - if (property.cacheFeatureVector && property.isInstanceOf[NodeProperty[T]]) { + if (supportsFeatureVectorCaching && property.cacheFeatureVector && property.isInstanceOf[NodeProperty[T]]) { val nodeProperty = property.asInstanceOf[NodeProperty[T]] val instanceCacheMap = nodeProperty.node.propertyFeatureVectorCache .getOrElseUpdate(instance, new mutable.HashMap[Property[_], FeatureVector]())