From 8d91502e14b6aa1d56ac6588c1ad30ff6bc9a1dd Mon Sep 17 00:00:00 2001 From: Gianmarco De Francisci Morales Date: Tue, 2 Apr 2013 11:59:20 +0200 Subject: [PATCH 1/3] enable passing parameters via command line (-D) switches --- src/main/java/cc/mrlda/DisplayTopic.java | 2 +- src/main/java/cc/mrlda/InformedPrior.java | 2 +- src/main/java/cc/mrlda/ParseCorpus.java | 2 +- src/main/java/cc/mrlda/VariationalInference.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/cc/mrlda/DisplayTopic.java b/src/main/java/cc/mrlda/DisplayTopic.java index 79e89f1..1fdc1c4 100644 --- a/src/main/java/cc/mrlda/DisplayTopic.java +++ b/src/main/java/cc/mrlda/DisplayTopic.java @@ -163,7 +163,7 @@ public int run(String[] args) throws Exception { } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(new Configuration(), new DisplayTopic(), args); + int res = ToolRunner.run(new DisplayTopic(), args); System.exit(res); } } \ No newline at end of file diff --git a/src/main/java/cc/mrlda/InformedPrior.java b/src/main/java/cc/mrlda/InformedPrior.java index 94c0112..839cd3c 100644 --- a/src/main/java/cc/mrlda/InformedPrior.java +++ b/src/main/java/cc/mrlda/InformedPrior.java @@ -202,7 +202,7 @@ public static HMapIV> importEta(SequenceFile.Reader sequenceFileRea } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(new Configuration(), new InformedPrior(), args); + int res = ToolRunner.run(new InformedPrior(), args); System.exit(res); } } \ No newline at end of file diff --git a/src/main/java/cc/mrlda/ParseCorpus.java b/src/main/java/cc/mrlda/ParseCorpus.java index e771875..07be283 100644 --- a/src/main/java/cc/mrlda/ParseCorpus.java +++ b/src/main/java/cc/mrlda/ParseCorpus.java @@ -672,7 +672,7 @@ public static Map importParameter(SequenceFile.Reader sequenceF } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(new Configuration(), new ParseCorpus(), args); + int res = ToolRunner.run(new ParseCorpus(), args); System.exit(res); } diff --git a/src/main/java/cc/mrlda/VariationalInference.java b/src/main/java/cc/mrlda/VariationalInference.java index 8733cb4..af75ad4 100644 --- a/src/main/java/cc/mrlda/VariationalInference.java +++ b/src/main/java/cc/mrlda/VariationalInference.java @@ -763,7 +763,7 @@ public static void exportAlpha(SequenceFile.Writer sequenceFileWriter, double[] } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(new Configuration(), new VariationalInference(), args); + int res = ToolRunner.run(new VariationalInference(), args); System.exit(res); } } \ No newline at end of file From 1df4910fc90b2fadd0c6ad39af3b15c3d0770292 Mon Sep 17 00:00:00 2001 From: Gianmarco De Francisci Morales Date: Tue, 2 Apr 2013 11:59:20 +0200 Subject: [PATCH 2/3] enable passing parameters via command line (-D) switches --- src/main/java/cc/mrlda/DisplayTopic.java | 4 ++-- src/main/java/cc/mrlda/InformedPrior.java | 4 ++-- src/main/java/cc/mrlda/ParseCorpus.java | 12 ++++++------ src/main/java/cc/mrlda/VariationalInference.java | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/cc/mrlda/DisplayTopic.java b/src/main/java/cc/mrlda/DisplayTopic.java index 79e89f1..948f9e7 100644 --- a/src/main/java/cc/mrlda/DisplayTopic.java +++ b/src/main/java/cc/mrlda/DisplayTopic.java @@ -101,7 +101,7 @@ public int run(String[] args) throws Exception { System.exit(0); } - JobConf conf = new JobConf(DisplayTopic.class); + JobConf conf = new JobConf(getConf()); FileSystem fs = FileSystem.get(conf); Path indexPath = new Path(indexString); @@ -163,7 +163,7 @@ public int run(String[] args) throws Exception { } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(new Configuration(), new DisplayTopic(), args); + int res = ToolRunner.run(new DisplayTopic(), args); System.exit(res); } } \ No newline at end of file diff --git a/src/main/java/cc/mrlda/InformedPrior.java b/src/main/java/cc/mrlda/InformedPrior.java index 94c0112..cdf87c5 100644 --- a/src/main/java/cc/mrlda/InformedPrior.java +++ b/src/main/java/cc/mrlda/InformedPrior.java @@ -102,7 +102,7 @@ public int run(String[] args) throws Exception { } // Delete the output directory if it exists already - JobConf conf = new JobConf(InformedPrior.class); + JobConf conf = new JobConf(getConf()); FileSystem fs = FileSystem.get(conf); Path inputPath = new Path(input); @@ -202,7 +202,7 @@ public static HMapIV> importEta(SequenceFile.Reader sequenceFileRea } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(new Configuration(), new InformedPrior(), args); + int res = ToolRunner.run(new InformedPrior(), args); System.exit(res); } } \ No newline at end of file diff --git a/src/main/java/cc/mrlda/ParseCorpus.java b/src/main/java/cc/mrlda/ParseCorpus.java index e771875..d1a36e1 100644 --- a/src/main/java/cc/mrlda/ParseCorpus.java +++ b/src/main/java/cc/mrlda/ParseCorpus.java @@ -188,7 +188,7 @@ public int run(String[] args) throws Exception { String indexPath = outputPath + INDEX; // Delete the output directory if it exists already - FileSystem fs = FileSystem.get(new JobConf(ParseCorpus.class)); + FileSystem fs = FileSystem.get(getConf()); fs.delete(new Path(outputPath), true); try { @@ -326,7 +326,7 @@ public PairOfInts tokenizeDocument(String inputPath, String outputPath, int numb sLogger.info(" - number of mappers: " + numberOfMappers); sLogger.info(" - number of reducers: " + numberOfReducers); - JobConf conf = new JobConf(ParseCorpus.class); + JobConf conf = new JobConf(getConf()); conf.setJobName(ParseCorpus.class.getSimpleName() + " - tokenize document"); FileSystem fs = FileSystem.get(conf); @@ -371,7 +371,7 @@ public PairOfInts tokenizeDocument(String inputPath, String outputPath, int numb public Path indexTitle(String inputTitles, String outputTitle, int numberOfMappers) throws Exception { - JobConf conf = new JobConf(ParseCorpus.class); + JobConf conf = new JobConf(getConf()); FileSystem fs = FileSystem.get(conf); Path titleIndexPath = new Path(outputTitle); @@ -446,7 +446,7 @@ public Path indexTerm(String inputTerms, String outputTerm, int numberOfMappers, Path inputTermFiles = new Path(inputTerms); Path outputTermFile = new Path(outputTerm); - JobConf conf = new JobConf(ParseCorpus.class); + JobConf conf = new JobConf(getConf()); FileSystem fs = FileSystem.get(conf); sLogger.info("Tool: " + ParseCorpus.class.getSimpleName()); @@ -594,7 +594,7 @@ public Path indexDocument(String inputDocument, String outputDocument, String te Path termIndexPath = new Path(termIndex); Path titleIndexPath = new Path(titleIndex); - JobConf conf = new JobConf(ParseCorpus.class); + JobConf conf = new JobConf(getConf()); FileSystem fs = FileSystem.get(conf); sLogger.info("Tool: " + ParseCorpus.class.getSimpleName()); @@ -672,7 +672,7 @@ public static Map importParameter(SequenceFile.Reader sequenceF } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(new Configuration(), new ParseCorpus(), args); + int res = ToolRunner.run(new ParseCorpus(), args); System.exit(res); } diff --git a/src/main/java/cc/mrlda/VariationalInference.java b/src/main/java/cc/mrlda/VariationalInference.java index 8733cb4..9a92b05 100644 --- a/src/main/java/cc/mrlda/VariationalInference.java +++ b/src/main/java/cc/mrlda/VariationalInference.java @@ -319,7 +319,7 @@ private int run(String inputPath, String outputPath, int numberOfTopics, int num sLogger.info(" - truncation beta: " + truncateBeta); sLogger.info(" - informed prior: " + informedPrior); - JobConf conf = new JobConf(VariationalInference.class); + JobConf conf = new JobConf(getConf()); FileSystem fs = FileSystem.get(conf); // delete the overall output path @@ -387,7 +387,7 @@ private int run(String inputPath, String outputPath, int numberOfTopics, int num int numberOfDocuments = 0; do { - conf = new JobConf(VariationalInference.class); + conf = new JobConf(getConf()); if (training) { conf.setJobName(VariationalInference.class.getSimpleName() + " - Iteration " + (iterationCount + 1)); @@ -763,7 +763,7 @@ public static void exportAlpha(SequenceFile.Writer sequenceFileWriter, double[] } public static void main(String[] args) throws Exception { - int res = ToolRunner.run(new Configuration(), new VariationalInference(), args); + int res = ToolRunner.run(new VariationalInference(), args); System.exit(res); } } \ No newline at end of file From aae2b003175d213e593146ad6d5c425b95b564f3 Mon Sep 17 00:00:00 2001 From: Gianmarco De Francisci Morales Date: Tue, 2 Apr 2013 12:24:42 +0200 Subject: [PATCH 3/3] fixed user jar location missing --- src/main/java/cc/mrlda/DisplayTopic.java | 1 + src/main/java/cc/mrlda/InformedPrior.java | 1 + src/main/java/cc/mrlda/ParseCorpus.java | 4 ++++ src/main/java/cc/mrlda/VariationalInference.java | 2 ++ 4 files changed, 8 insertions(+) diff --git a/src/main/java/cc/mrlda/DisplayTopic.java b/src/main/java/cc/mrlda/DisplayTopic.java index 948f9e7..a6e5dce 100644 --- a/src/main/java/cc/mrlda/DisplayTopic.java +++ b/src/main/java/cc/mrlda/DisplayTopic.java @@ -102,6 +102,7 @@ public int run(String[] args) throws Exception { } JobConf conf = new JobConf(getConf()); + conf.setJarByClass(getClass()); FileSystem fs = FileSystem.get(conf); Path indexPath = new Path(indexString); diff --git a/src/main/java/cc/mrlda/InformedPrior.java b/src/main/java/cc/mrlda/InformedPrior.java index cdf87c5..d6fdeb5 100644 --- a/src/main/java/cc/mrlda/InformedPrior.java +++ b/src/main/java/cc/mrlda/InformedPrior.java @@ -103,6 +103,7 @@ public int run(String[] args) throws Exception { // Delete the output directory if it exists already JobConf conf = new JobConf(getConf()); + conf.setJarByClass(getClass()); FileSystem fs = FileSystem.get(conf); Path inputPath = new Path(input); diff --git a/src/main/java/cc/mrlda/ParseCorpus.java b/src/main/java/cc/mrlda/ParseCorpus.java index d1a36e1..ea27110 100644 --- a/src/main/java/cc/mrlda/ParseCorpus.java +++ b/src/main/java/cc/mrlda/ParseCorpus.java @@ -327,6 +327,7 @@ public PairOfInts tokenizeDocument(String inputPath, String outputPath, int numb sLogger.info(" - number of reducers: " + numberOfReducers); JobConf conf = new JobConf(getConf()); + conf.setJarByClass(getClass()); conf.setJobName(ParseCorpus.class.getSimpleName() + " - tokenize document"); FileSystem fs = FileSystem.get(conf); @@ -372,6 +373,7 @@ public PairOfInts tokenizeDocument(String inputPath, String outputPath, int numb public Path indexTitle(String inputTitles, String outputTitle, int numberOfMappers) throws Exception { JobConf conf = new JobConf(getConf()); + conf.setJarByClass(getClass()); FileSystem fs = FileSystem.get(conf); Path titleIndexPath = new Path(outputTitle); @@ -447,6 +449,7 @@ public Path indexTerm(String inputTerms, String outputTerm, int numberOfMappers, Path outputTermFile = new Path(outputTerm); JobConf conf = new JobConf(getConf()); + conf.setJarByClass(getClass()); FileSystem fs = FileSystem.get(conf); sLogger.info("Tool: " + ParseCorpus.class.getSimpleName()); @@ -595,6 +598,7 @@ public Path indexDocument(String inputDocument, String outputDocument, String te Path titleIndexPath = new Path(titleIndex); JobConf conf = new JobConf(getConf()); + conf.setJarByClass(getClass()); FileSystem fs = FileSystem.get(conf); sLogger.info("Tool: " + ParseCorpus.class.getSimpleName()); diff --git a/src/main/java/cc/mrlda/VariationalInference.java b/src/main/java/cc/mrlda/VariationalInference.java index 9a92b05..42a67e7 100644 --- a/src/main/java/cc/mrlda/VariationalInference.java +++ b/src/main/java/cc/mrlda/VariationalInference.java @@ -320,6 +320,7 @@ private int run(String inputPath, String outputPath, int numberOfTopics, int num sLogger.info(" - informed prior: " + informedPrior); JobConf conf = new JobConf(getConf()); + conf.setJarByClass(getClass()); FileSystem fs = FileSystem.get(conf); // delete the overall output path @@ -388,6 +389,7 @@ private int run(String inputPath, String outputPath, int numberOfTopics, int num do { conf = new JobConf(getConf()); + conf.setJarByClass(getClass()); if (training) { conf.setJobName(VariationalInference.class.getSimpleName() + " - Iteration " + (iterationCount + 1));