From d6820dc848d0dd4e734e11410cfb91b8cafc3865 Mon Sep 17 00:00:00 2001 From: Oleh Shklyar Date: Tue, 10 Dec 2019 13:19:24 +0200 Subject: [PATCH 1/9] fix generic --- src/plugins/Library/util/SkeletonBTreeMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/Library/util/SkeletonBTreeMap.java b/src/plugins/Library/util/SkeletonBTreeMap.java index 84628770..aa326304 100644 --- a/src/plugins/Library/util/SkeletonBTreeMap.java +++ b/src/plugins/Library/util/SkeletonBTreeMap.java @@ -562,7 +562,7 @@ public interface SkeletonMap final Queue nodequeue = new PriorityQueue(); - Map, ProgressTracker> ids = null; + Map, ProgressTracker> ids = null; ProgressTracker ntracker = null;; if (nsrl instanceof Serialiser.Trackable) { From 2f4d2103d2f2157fdb3309c4f133d815bb0cd9bc Mon Sep 17 00:00:00 2001 From: Oleh Shklyar Date: Tue, 10 Dec 2019 13:29:45 +0200 Subject: [PATCH 2/9] explanatory comment added --- src/plugins/Library/util/SkeletonTreeMap.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/Library/util/SkeletonTreeMap.java b/src/plugins/Library/util/SkeletonTreeMap.java index 866fb2bc..1200f5c1 100644 --- a/src/plugins/Library/util/SkeletonTreeMap.java +++ b/src/plugins/Library/util/SkeletonTreeMap.java @@ -321,10 +321,10 @@ public static Map app(SkeletonTreeMap map, Map SkeletonTreeMap rev(Map intm, SkeletonTreeMap map, Translator ktr) throws DataFormatException { - if (ktr == null) { + if (ktr == null) { // this case when K is String and therefore no keyTranslator is needed try { for (Map.Entry en: intm.entrySet()) { - map.putGhost((K)en.getKey(), en.getValue()); + map.putGhost((K) en.getKey(), en.getValue()); } } catch (ClassCastException e) { throw new DataFormatException("TreeMapTranslator: reverse translation failed. Try supplying a non-null key-translator.", e, intm, null, null); From 86a7d0a2fc972489d5988575ba5c9680f97ee730 Mon Sep 17 00:00:00 2001 From: Oleh Shklyar Date: Tue, 10 Dec 2019 15:31:34 +0200 Subject: [PATCH 3/9] update java version to 8 and snakeyaml to 1.25 --- build.xml | 20 +++++++------- src/plugins/Library/io/YamlReaderWriter.java | 27 +++++++------------ .../Library/io/serial/YamlMapTest.java | 14 +++------- 3 files changed, 24 insertions(+), 37 deletions(-) diff --git a/build.xml b/build.xml index fa7a6670..c0cee4e3 100644 --- a/build.xml +++ b/build.xml @@ -2,10 +2,10 @@ - + - - + + @@ -64,14 +64,14 @@ - - - + + + - - - + + + @@ -80,7 +80,7 @@ - + - + @@ -146,7 +146,7 @@ - + @@ -173,7 +173,7 @@ - + diff --git a/src/plugins/Library/io/YamlReaderWriter.java b/src/plugins/Library/io/YamlReaderWriter.java index a2b0977a..1d99dc6a 100644 --- a/src/plugins/Library/io/YamlReaderWriter.java +++ b/src/plugins/Library/io/YamlReaderWriter.java @@ -17,6 +17,8 @@ import org.yaml.snakeyaml.constructor.Constructor; import org.yaml.snakeyaml.constructor.AbstractConstruct; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Arrays; import java.util.Map; @@ -35,20 +37,18 @@ import plugins.Library.index.TermTermEntry; import freenet.keys.FreenetURI; - /** -** Converts between an object and a stream containing a YAML document. By -** default, this uses a {@link Yaml} processor with additional object and tag -** definitions relevant to the Library plugin. -** -** (Ideally this would implement {@link java.io.ObjectInput} and {@link -** java.io.ObjectOutput} but they have too many methods to bother with...) -** -** @see Yaml -** @author infinity0 +* Converts between an object and a stream containing a YAML document. By +* default, this uses a {@link Yaml} processor with additional object and tag +* definitions relevant to the Library plugin. +* +* (Ideally this would implement {@link java.io.ObjectInput} and {@link +* java.io.ObjectOutput} but they have too many methods to bother with...) +* +* @see Yaml +* @author infinity0 */ -public class YamlReaderWriter -implements ObjectStreamReader, ObjectStreamWriter { +public class YamlReaderWriter implements ObjectStreamReader, ObjectStreamWriter { final public static String MIME_TYPE = "text/yaml"; final public static String FILE_EXTENSION = ".yml"; @@ -60,10 +60,11 @@ public class YamlReaderWriter public YamlReaderWriter() { } - /*@Override**/ public Object readObject(InputStream is) throws IOException { + @Override + public Object readObject(InputStream is) throws IOException { parallelLimiter.acquireUninterruptibly(); try { - return makeYAML().load(new InputStreamReader(is, "UTF-8")); + return makeYAML().load(new InputStreamReader(is, StandardCharsets.UTF_8)); } catch (YAMLException e) { throw new DataFormatException("Yaml could not process the stream: " + is, e, is, null, null); } finally { @@ -71,10 +72,11 @@ public YamlReaderWriter() { } } - /*@Override**/ public void writeObject(Object o, OutputStream os) throws IOException { + @Override + public void writeObject(Object o, OutputStream os) throws IOException { parallelLimiter.acquireUninterruptibly(); try { - makeYAML().dump(o, new OutputStreamWriter(os, "UTF-8")); + makeYAML().dump(o, new OutputStreamWriter(os, StandardCharsets.UTF_8)); } catch (YAMLException e) { throw new DataFormatException("Yaml could not process the object", e, o, null, null); } finally { @@ -82,9 +84,11 @@ public YamlReaderWriter() { } } - /** We do NOT keep this thread-local, because the Composer is only cleared after + /** + * We do NOT keep this thread-local, because the Composer is only cleared after * the next call to load(), so it can persist with a lot of useless data if we - * then use a different thread. So lets just construct them as needed. */ + * then use a different thread. So lets just construct them as needed. + */ private Yaml makeYAML() { DumperOptions opt = new DumperOptions(); opt.setWidth(Integer.MAX_VALUE); @@ -97,37 +101,40 @@ private Yaml makeYAML() { final public static ObjectBlueprint tebp_page; static { try { - tebp_term = new ObjectBlueprint(TermTermEntry.class, Arrays.asList("subj", "rel", "term")); - tebp_index = new ObjectBlueprint(TermIndexEntry.class, Arrays.asList("subj", "rel", "index")); - tebp_page = new ObjectBlueprint(TermPageEntry.class, Arrays.asList("subj", "rel", "page", "title", "positions", "posFragments")); - } catch (NoSuchFieldException e) { - throw new AssertionError(e); - } catch (NoSuchMethodException e) { + tebp_term = new ObjectBlueprint<>(TermTermEntry.class, + Arrays.asList("subj", "rel", "term")); + tebp_index = new ObjectBlueprint<>(TermIndexEntry.class, + Arrays.asList("subj", "rel", "index")); + tebp_page = new ObjectBlueprint<>(TermPageEntry.class, + Arrays.asList("subj", "rel", "page", "title", "positions", "posFragments")); + } catch (NoSuchFieldException | NoSuchMethodException e) { throw new AssertionError(e); } } - /************************************************************************ - ** DOCUMENT + /** + * DOCUMENT */ public static class ExtendedRepresenter extends Representer { public ExtendedRepresenter() { this.representers.put(FreenetURI.class, new Represent() { - /*@Override**/ public Node representData(Object data) { - return representScalar(new Tag("!FreenetURI"), ((FreenetURI) data).toString()); + @Override + public Node representData(Object data) { + return representScalar(new Tag("!FreenetURI"), data.toString()); } }); this.representers.put(Packer.BinInfo.class, new Represent() { - /*@Override**/ public Node representData(Object data) { - Packer.BinInfo inf = (Packer.BinInfo)data; - Map map = Collections.singletonMap(inf.getID(), inf.getWeight()); + @Override + public Node representData(Object data) { + Packer.BinInfo inf = (Packer.BinInfo) data; + Map map = Collections.singletonMap(inf.getID(), inf.getWeight()); return representMapping(new Tag("!BinInfo"), map, DumperOptions.FlowStyle.FLOW); } }); - this.representers.put(TermTermEntry.class, new RepresentTermEntry(tebp_term)); - this.representers.put(TermIndexEntry.class, new RepresentTermEntry(tebp_index)); - this.representers.put(TermPageEntry.class, new RepresentTermEntry(tebp_page)); + this.representers.put(TermTermEntry.class, new RepresentTermEntry<>(tebp_term)); + this.representers.put(TermIndexEntry.class, new RepresentTermEntry<>(tebp_index)); + this.representers.put(TermPageEntry.class, new RepresentTermEntry<>(tebp_page)); } public class RepresentTermEntry implements Represent { @@ -140,45 +147,47 @@ public RepresentTermEntry(ObjectBlueprint bp) { tag = "!" + bp.getObjectClass().getSimpleName(); } - /*@Override**/ public Node representData(Object data) { - return representMapping(new Tag(tag), blueprint.objectAsMap((T)data), DumperOptions.FlowStyle.FLOW); + @Override + public Node representData(Object data) { + return representMapping(new Tag(tag), blueprint.objectAsMap((T) data), DumperOptions.FlowStyle.FLOW); } - } - } - - /************************************************************************ - ** DOCUMENT + /** + * DOCUMENT */ public static class ExtendedConstructor extends Constructor { public ExtendedConstructor() { this.yamlConstructors.put(new Tag("!FreenetURI"), new AbstractConstruct() { - /*@Override**/ public Object construct(Node node) { - String uri = (String) constructScalar((ScalarNode)node); + @Override + public Object construct(Node node) { + String uri = constructScalar((ScalarNode)node); try { return new FreenetURI(uri); } catch (java.net.MalformedURLException e) { - throw new ConstructorException("while constructing a FreenetURI", node.getStartMark(), "found malformed URI " + uri, null); + throw new ConstructorException("while constructing a FreenetURI", node.getStartMark(), + "found malformed URI " + uri, null); } } }); this.yamlConstructors.put(new Tag("!BinInfo"), new AbstractConstruct() { - /*@Override**/ public Object construct(Node node) { - Map map = (Map) constructMapping((MappingNode)node); + @Override + public Object construct(Node node) { + Map map = constructMapping((MappingNode)node); if (map.size() != 1) { - throw new ConstructorException("while constructing a Packer.BinInfo", node.getStartMark(), "found incorrectly sized map data " + map, null); + throw new ConstructorException("while constructing a Packer.BinInfo", node.getStartMark(), + "found incorrectly sized map data " + map, null); } - for (Map.Entry en: map.entrySet()) { - return new Packer.BinInfo(en.getKey(), (Integer)en.getValue()); + for (Map.Entry en: map.entrySet()) { + return new Packer.BinInfo(en.getKey(), (Integer) en.getValue()); } throw new AssertionError(); } }); - this.yamlConstructors.put(new Tag("!TermTermEntry"), new ConstructTermEntry(tebp_term)); - this.yamlConstructors.put(new Tag("!TermIndexEntry"), new ConstructTermEntry(tebp_index)); - this.yamlConstructors.put(new Tag("!TermPageEntry"), new ConstructTermEntry(tebp_page)); + this.yamlConstructors.put(new Tag("!TermTermEntry"), new ConstructTermEntry<>(tebp_term)); + this.yamlConstructors.put(new Tag("!TermIndexEntry"), new ConstructTermEntry<>(tebp_index)); + this.yamlConstructors.put(new Tag("!TermPageEntry"), new ConstructTermEntry<>(tebp_page)); } public class ConstructTermEntry extends AbstractConstruct { @@ -189,19 +198,17 @@ public ConstructTermEntry(ObjectBlueprint bp) { blueprint = bp; } - /*@Override**/ public Object construct(Node node) { - Map map = (Map)constructMapping((MappingNode)node); - map.put("rel", new Float(((Double)map.get("rel")).floatValue())); + @Override + public Object construct(Node node) { + Map map = constructMapping((MappingNode)node); + map.put("rel", ((Double) map.get("rel")).floatValue()); try { return blueprint.objectFromMap(map); - } catch (Exception e) { - //java.lang.InstantiationException - //java.lang.IllegalAccessException - //java.lang.reflect.InvocationTargetException - throw new ConstructorException("while constructing a " + blueprint.getObjectClass().getSimpleName(), node.getStartMark(), "could not instantiate map " + map, null, e); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + throw new ConstructorException("while constructing a " + blueprint.getObjectClass().getSimpleName(), + node.getStartMark(), "could not instantiate map " + map, null, e); } } - } } @@ -214,6 +221,4 @@ public ConstructorException(String context, Mark contextMark, String problem, Ma super(context, contextMark, problem, problemMark, cause); } } - - } diff --git a/test/plugins/Library/io/serial/YamlMapTest.java b/test/plugins/Library/io/serial/YamlMapTest.java index 70ea7f30..7ef8eccc 100644 --- a/test/plugins/Library/io/serial/YamlMapTest.java +++ b/test/plugins/Library/io/serial/YamlMapTest.java @@ -24,7 +24,7 @@ public class YamlMapTest extends TestCase { public void testYamlMap() throws IOException { - Map data = new TreeMap(); + Map data = new TreeMap<>(); data.put("key1", new Bean()); data.put("key2", new Bean()); data.put("key3", new Custom("test")); @@ -38,15 +38,13 @@ public void testYamlMap() throws IOException { os.close(); FileInputStream is = new FileInputStream(file); - Object o = yaml.load(new InputStreamReader(is)); + Map map = yaml.load(new InputStreamReader(is)); is.close(); - assertTrue(o instanceof Map); - Map m = (Map)o; - assertTrue(m.get("key1") instanceof Bean); - assertTrue(m.get("key2") instanceof Bean); // NOTE these tests fail in snakeYAML 1.2 and below, fixed in 1.3 - assertTrue(m.get("key3") instanceof Custom); - assertTrue(m.get("key4") instanceof Wrapper); + assertTrue(map.get("key1") instanceof Bean); + assertTrue(map.get("key2") instanceof Bean); // NOTE these tests fail in snakeYAML 1.2 and below, fixed in 1.3 + assertTrue(map.get("key3") instanceof Custom); + assertTrue(map.get("key4") instanceof Wrapper); } public static class Bean { @@ -54,7 +52,6 @@ public static class Bean { public Bean() { a = ""; } public String getA() { return a; } public void setA(String s) { a = s; } - } public static class Wrapper { @@ -86,7 +83,6 @@ public Custom(Custom c) { public String toString() { return str; } } - public static class ExtendedRepresenter extends Representer { public ExtendedRepresenter() { this.representers.put(Custom.class, new RepresentCustom()); @@ -94,12 +90,11 @@ public ExtendedRepresenter() { private class RepresentCustom implements Represent { public Node representData(Object data) { - return representScalar(new Tag("!Custom"), ((Custom) data).toString()); + return representScalar(new Tag("!Custom"), data.toString()); } } } - public static class ExtendedConstructor extends Constructor { public ExtendedConstructor() { this.yamlConstructors.put(new Tag("!Custom"), new ConstructCustom()); @@ -107,11 +102,10 @@ public ExtendedConstructor() { private class ConstructCustom implements Construct { public Object construct(Node node) { - String str = (String) constructScalar((ScalarNode)node); + String str = constructScalar((ScalarNode)node); return new Custom(str); } public void construct2ndStep(Node node, Object object) { } } } - } From 07d43f9a9c5d69b6abd6ce3e0b0481383da2f8c8 Mon Sep 17 00:00:00 2001 From: Oleh Shklyar Date: Tue, 10 Dec 2019 17:52:19 +0200 Subject: [PATCH 6/9] refactoring: bypass for same bug in second place --- .../Library/index/ProtoIndexSerialiser.java | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/plugins/Library/index/ProtoIndexSerialiser.java b/src/plugins/Library/index/ProtoIndexSerialiser.java index fc0e64d2..3485950c 100644 --- a/src/plugins/Library/index/ProtoIndexSerialiser.java +++ b/src/plugins/Library/index/ProtoIndexSerialiser.java @@ -9,7 +9,6 @@ import plugins.Library.util.SkeletonBTreeSet; import plugins.Library.util.exec.SimpleProgress; import plugins.Library.util.exec.TaskAbortException; -import plugins.Library.io.serial.Serialiser.*; import plugins.Library.io.serial.LiveArchiver; import plugins.Library.io.serial.Serialiser; import plugins.Library.io.serial.Translator; @@ -110,7 +109,7 @@ public static ProtoIndexSerialiser forIndex(File prefix) { serialisable.data.put("reqID", task.meta); } try { - task.data = trans.rev(serialisable.data); + task.data = trans.rev(cast(serialisable.data /* FIXME: there Double and String keys */)); } catch (DataFormatException e) { throw new TaskAbortException("Could not construct index from data", e); } @@ -123,6 +122,23 @@ public static ProtoIndexSerialiser forIndex(File prefix) { task.meta = serialisable.meta; } + // should be removed when the cause of the mixture of Double and String among the keys is eliminated + private static Map cast(Map map) { + List> wrongEntries = new ArrayList<>(); + for(Iterator> it = map.entrySet().iterator(); it.hasNext(); ) { + Map.Entry entry = it.next(); + if (entry.getKey() instanceof String) { + continue; + } + wrongEntries.add(entry); + it.remove(); + } + for (Map.Entry entry : wrongEntries) { + map.put(String.valueOf(entry.getKey()), entry.getValue()); + } + return (Map) map; + } + public static class IndexTranslator implements Translator> { @@ -200,20 +216,8 @@ public IndexTranslator(LiveArchiver, SimpleProgress> subsrl) SkeletonBTreeMap> utab = utrans.rev((Map)map.get("utab")); // FIXME: termTable has Double and String keys - Map termTable = (Map) map.get("ttab"); - List wrongEntries = new ArrayList<>(); - for(Iterator it = termTable.entrySet().iterator(); it.hasNext(); ) { - Map.Entry entry = it.next(); - if (entry.getKey() instanceof String) { - continue; - } - wrongEntries.add(entry); - it.remove(); - } - for (Map.Entry entry : wrongEntries) { - termTable.put(String.valueOf(entry.getKey()), entry.getValue()); - } - SkeletonBTreeMap> ttab = ttrans.rev((Map) termTable); + Map termTable = ProtoIndexSerialiser.cast((Map) map.get("ttab")); + SkeletonBTreeMap> ttab = ttrans.rev(termTable); return cmpsrl.setSerialiserFor(new ProtoIndex(reqID, name, ownerName, ownerEmail, totalPages, modified, extra, utab, ttab)); From bc7ab73b89a464207f95a0426a883fe5bdc3e813 Mon Sep 17 00:00:00 2001 From: Oleh Shklyar Date: Tue, 10 Dec 2019 22:08:45 +0200 Subject: [PATCH 7/9] disable snakeyaml implicit resolvers --- .../Library/index/ProtoIndexSerialiser.java | 40 +++++-------------- src/plugins/Library/io/YamlReaderWriter.java | 11 +++-- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/src/plugins/Library/index/ProtoIndexSerialiser.java b/src/plugins/Library/index/ProtoIndexSerialiser.java index 3485950c..e6206530 100644 --- a/src/plugins/Library/index/ProtoIndexSerialiser.java +++ b/src/plugins/Library/index/ProtoIndexSerialiser.java @@ -19,6 +19,8 @@ import freenet.keys.FreenetURI; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.*; import java.io.File; @@ -109,7 +111,7 @@ public static ProtoIndexSerialiser forIndex(File prefix) { serialisable.data.put("reqID", task.meta); } try { - task.data = trans.rev(cast(serialisable.data /* FIXME: there Double and String keys */)); + task.data = trans.rev(serialisable.data); } catch (DataFormatException e) { throw new TaskAbortException("Could not construct index from data", e); } @@ -122,23 +124,6 @@ public static ProtoIndexSerialiser forIndex(File prefix) { task.meta = serialisable.meta; } - // should be removed when the cause of the mixture of Double and String among the keys is eliminated - private static Map cast(Map map) { - List> wrongEntries = new ArrayList<>(); - for(Iterator> it = map.entrySet().iterator(); it.hasNext(); ) { - Map.Entry entry = it.next(); - if (entry.getKey() instanceof String) { - continue; - } - wrongEntries.add(entry); - it.remove(); - } - for (Map.Entry entry : wrongEntries) { - map.put(String.valueOf(entry.getKey()), entry.getValue()); - } - return (Map) map; - } - public static class IndexTranslator implements Translator> { @@ -204,20 +189,15 @@ public IndexTranslator(LiveArchiver, SimpleProgress> subsrl) String name = (String)map.get("name"); String ownerName = (String)map.get("ownerName"); String ownerEmail = (String)map.get("ownerEmail"); - // FIXME yaml idiocy??? It seems to give a Long if the number is big enough to need one, and an Integer otherwise. - long totalPages; - Object o = map.get("totalPages"); - if(o instanceof Long) - totalPages = (Long)o; - else // Integer - totalPages = (Integer)o; - Date modified = (Date)map.get("modified"); + long totalPages = Long.parseLong((String) map.get("totalPages")); + Date modified = null; + try { + modified = new SimpleDateFormat("yyyy-MM-dd").parse((String) map.get("modified")); + } catch (ParseException ignored) { + } Map extra = (Map)map.get("extra"); SkeletonBTreeMap> utab = utrans.rev((Map)map.get("utab")); - - // FIXME: termTable has Double and String keys - Map termTable = ProtoIndexSerialiser.cast((Map) map.get("ttab")); - SkeletonBTreeMap> ttab = ttrans.rev(termTable); + SkeletonBTreeMap> ttab = ttrans.rev((Map) map.get("ttab")); return cmpsrl.setSerialiserFor(new ProtoIndex(reqID, name, ownerName, ownerEmail, totalPages, modified, extra, utab, ttab)); diff --git a/src/plugins/Library/io/YamlReaderWriter.java b/src/plugins/Library/io/YamlReaderWriter.java index 1d99dc6a..4279f6fd 100644 --- a/src/plugins/Library/io/YamlReaderWriter.java +++ b/src/plugins/Library/io/YamlReaderWriter.java @@ -30,6 +30,7 @@ import java.io.IOException; /* class definitions added to the extended Yaml processor */ +import org.yaml.snakeyaml.resolver.Resolver; import plugins.Library.io.serial.Packer; import plugins.Library.index.TermEntry; import plugins.Library.index.TermPageEntry; @@ -93,7 +94,11 @@ private Yaml makeYAML() { DumperOptions opt = new DumperOptions(); opt.setWidth(Integer.MAX_VALUE); opt.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - return new Yaml(new ExtendedConstructor(), new ExtendedRepresenter(), opt); + return new Yaml(new ExtendedConstructor(), new ExtendedRepresenter(), opt, new Resolver() { + @Override + protected void addImplicitResolvers() { // disable implicit resolvers + } + }); } final public static ObjectBlueprint tebp_term; @@ -180,7 +185,7 @@ public Object construct(Node node) { "found incorrectly sized map data " + map, null); } for (Map.Entry en: map.entrySet()) { - return new Packer.BinInfo(en.getKey(), (Integer) en.getValue()); + return new Packer.BinInfo(en.getKey(), Integer.parseInt((String) en.getValue())); } throw new AssertionError(); } @@ -201,7 +206,7 @@ public ConstructTermEntry(ObjectBlueprint bp) { @Override public Object construct(Node node) { Map map = constructMapping((MappingNode)node); - map.put("rel", ((Double) map.get("rel")).floatValue()); + map.put("rel", Float.valueOf((String) map.get("rel"))); try { return blueprint.objectFromMap(map); } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { From 10d6ea2eb6962ae9eb0b964f5f1193e065c55e10 Mon Sep 17 00:00:00 2001 From: Oleh Shklyar Date: Thu, 12 Dec 2019 17:30:47 +0200 Subject: [PATCH 8/9] kludge related to bc7ab73 --- src/plugins/Library/SpiderIndexUploader.java | 2 +- .../Library/index/ProtoIndexSerialiser.java | 41 +++++++++--- src/plugins/Library/index/TermEntry.java | 3 +- src/plugins/Library/io/ObjectBlueprint.java | 3 + src/plugins/Library/io/YamlReaderWriter.java | 61 ++++++++++++++++-- .../Library/util/SkeletonBTreeMap.java | 62 ++++++++++++++----- 6 files changed, 141 insertions(+), 31 deletions(-) diff --git a/src/plugins/Library/SpiderIndexUploader.java b/src/plugins/Library/SpiderIndexUploader.java index 9b853337..29fc76dd 100644 --- a/src/plugins/Library/SpiderIndexUploader.java +++ b/src/plugins/Library/SpiderIndexUploader.java @@ -389,7 +389,7 @@ private Closure>, TaskAbortException> /*@Override**/ public void invoke(Map.Entry> entry) throws TaskAbortException { String key = entry.getKey(); SkeletonBTreeSet tree = entry.getValue(); - if(logMINOR) Logger.minor(this, "Processing: "+key+" : "+tree); + Logger.minor(this, "Processing: " + key + " : " + (tree != null ? tree : "new")); if(tree != null) Logger.debug(this, "Merging data (on disk) in term "+key); else diff --git a/src/plugins/Library/index/ProtoIndexSerialiser.java b/src/plugins/Library/index/ProtoIndexSerialiser.java index e6206530..0051da5a 100644 --- a/src/plugins/Library/index/ProtoIndexSerialiser.java +++ b/src/plugins/Library/index/ProtoIndexSerialiser.java @@ -179,21 +179,48 @@ public IndexTranslator(LiveArchiver, SimpleProgress> subsrl) } /*@Override**/ public ProtoIndex rev(Map map) throws DataFormatException { - long magic = (Long)map.get("serialVersionUID"); + Object serialVersionUID = map.get("serialVersionUID"); + long magic; + if (serialVersionUID instanceof String) { // FIXME + magic = Long.parseLong((String) map.get("serialVersionUID")); + } else { + magic = (Long) serialVersionUID; + } if (magic == ProtoIndex.serialVersionUID) { try { // FIXME yet more hacks related to the lack of proper asynchronous FreenetArchiver... - ProtoIndexComponentSerialiser cmpsrl = ProtoIndexComponentSerialiser.get((Integer)map.get("serialFormatUID"), subsrl); + Object serialFormatUIDObj = map.get("serialFormatUID"); + int serialFormatUID; + if (serialFormatUIDObj instanceof String) { // FIXME + serialFormatUID = Integer.parseInt((String) map.get("serialFormatUID")); + } else { + serialFormatUID = (Integer) serialFormatUIDObj; + } + ProtoIndexComponentSerialiser cmpsrl = ProtoIndexComponentSerialiser.get(serialFormatUID, subsrl); FreenetURI reqID = (FreenetURI)map.get("reqID"); String name = (String)map.get("name"); String ownerName = (String)map.get("ownerName"); String ownerEmail = (String)map.get("ownerEmail"); - long totalPages = Long.parseLong((String) map.get("totalPages")); - Date modified = null; - try { - modified = new SimpleDateFormat("yyyy-MM-dd").parse((String) map.get("modified")); - } catch (ParseException ignored) { + Object totalPagesObj = map.get("totalPages"); + long totalPages; + if (totalPagesObj instanceof String) { // FIXME + totalPages = Long.parseLong((String) totalPagesObj); + } else if (totalPagesObj instanceof Long) { // FIXME yaml??? It seems to give a Long if the number + totalPages = (Long) totalPagesObj; // is big enough to need one, and an Integer otherwise. + } else { + totalPages = (Integer) totalPagesObj; + } + Object modifiedObj = map.get("modified"); + Date modified; + if (modifiedObj instanceof String) { // FIXME + try { + modified = new SimpleDateFormat("yyyy-MM-dd").parse((String) modifiedObj); + } catch (ParseException ignored) { + modified = null; + } + } else { + modified = (Date) modifiedObj; } Map extra = (Map)map.get("extra"); SkeletonBTreeMap> utab = utrans.rev((Map)map.get("utab")); diff --git a/src/plugins/Library/index/TermEntry.java b/src/plugins/Library/index/TermEntry.java index e5832b5a..5e86727a 100644 --- a/src/plugins/Library/index/TermEntry.java +++ b/src/plugins/Library/index/TermEntry.java @@ -37,7 +37,8 @@ public enum EntryType { public TermEntry(String s, float r) { if (s == null) { - throw new IllegalArgumentException("can't have a null subject!"); +// throw new IllegalArgumentException("can't have a null subject!"); + s = "null"; // FIXME } if (r < 0/* || r > 1*/) { // FIXME: I don't see how our relevance algorithm can be guaranteed to produce relevance <1. throw new IllegalArgumentException("Relevance must be in the half-closed interval (0,1]. Supplied: " + r); diff --git a/src/plugins/Library/io/ObjectBlueprint.java b/src/plugins/Library/io/ObjectBlueprint.java index 36969847..771ad312 100644 --- a/src/plugins/Library/io/ObjectBlueprint.java +++ b/src/plugins/Library/io/ObjectBlueprint.java @@ -299,6 +299,9 @@ public T objectFromMap(Map map) throws InstantiationException, IllegalAcce String property = en.getKey(); Class type = en.getValue(); Object value = map.get(property); + if (value != null && value.equals("null")) { // FIXME: case when type Map and value String "null" + value = null; + } try { if (type.isPrimitive()) { value = boxCast(type, value); diff --git a/src/plugins/Library/io/YamlReaderWriter.java b/src/plugins/Library/io/YamlReaderWriter.java index 4279f6fd..f0a9eeae 100644 --- a/src/plugins/Library/io/YamlReaderWriter.java +++ b/src/plugins/Library/io/YamlReaderWriter.java @@ -3,6 +3,7 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.io; +import freenet.support.SortedIntSet; import org.yaml.snakeyaml.nodes.Tag; import org.yaml.snakeyaml.Yaml; @@ -19,9 +20,7 @@ import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.Arrays; -import java.util.Map; +import java.util.*; import java.util.concurrent.Semaphore; import java.io.OutputStream; import java.io.OutputStreamWriter; @@ -185,7 +184,13 @@ public Object construct(Node node) { "found incorrectly sized map data " + map, null); } for (Map.Entry en: map.entrySet()) { - return new Packer.BinInfo(en.getKey(), Integer.parseInt((String) en.getValue())); + int w; // FIXME + if (en.getValue() instanceof String) { + w = Integer.parseInt((String) en.getValue()); + } else { + w = (Integer) en.getValue(); + } + return new Packer.BinInfo(en.getKey(), w); } throw new AssertionError(); } @@ -206,7 +211,53 @@ public ConstructTermEntry(ObjectBlueprint bp) { @Override public Object construct(Node node) { Map map = constructMapping((MappingNode)node); - map.put("rel", Float.valueOf((String) map.get("rel"))); + Object relObj = map.get("rel"); + float rel; + if (relObj instanceof Double) { // FIXME + rel = ((Double) relObj).floatValue(); + } else { + rel = Float.parseFloat((String) relObj); + } + map.put("rel", rel); + + // FIXME + Object posObj = map.get("positions"); + if (posObj != null) { + if ("null".equals(posObj)) { + map.put("positions", null); + } else { + Set pos = new SortedIntSet(); + for (Object p : (Set) posObj) { + if (p instanceof String) { + pos.add("null".equals(p) ? null : Integer.parseInt((String) p)); + } else { + pos.add((Integer) p); + } + } + map.put("positions", pos); + } + } + + // FIXME + Object posFragmentsObj = map.get("posFragments"); + if (posFragmentsObj != null) { + if ("null".equals(posFragmentsObj)) { + map.put("posFragments", null); + } else { + Map frags = new HashMap<>(); + for (Map.Entry entry : ((Map) posFragmentsObj).entrySet()) { + Integer key; + if (entry.getKey() instanceof String) { + key = "null".equals(entry.getKey()) ? null : Integer.parseInt((String) entry.getKey()); + } else { + key = (Integer) entry.getKey(); + } + frags.put(key, "null".equals(entry.getValue()) ? null : (String) entry.getValue()); + } + map.put("posFragments", frags); + } + } + try { return blueprint.objectFromMap(map); } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { diff --git a/src/plugins/Library/util/SkeletonBTreeMap.java b/src/plugins/Library/util/SkeletonBTreeMap.java index aa326304..450ee336 100644 --- a/src/plugins/Library/util/SkeletonBTreeMap.java +++ b/src/plugins/Library/util/SkeletonBTreeMap.java @@ -1333,19 +1333,17 @@ private void handleLocalRemove(SkeletonNode n, K key, TrackingSweeper 10)) { count = 0; -// if(ccount++ > 10) { - System.out.println(/*System.identityHashCode(this) + " " + */proc_val + " " + proc_pull + " " + proc_push+ " "+proc_deflate); -// ccount = 0; -// } + if (proc_val == null) { + Logger.debug(this, /*System.identityHashCode(this) + " " + */ + proc_val + " " + proc_pull + " " + proc_push + " " + proc_deflate); + } else { + Logger.minor(this, proc_val + " " + proc_pull + " " + proc_push + " " + proc_deflate); + } notifier.waitUpdate(1000); } progress = false; @@ -1508,17 +1506,35 @@ public NodeTranslator(Translator k, Translator, R> m boolean notleaf = map.containsKey("subnodes"); List gh = null; if (notleaf) { - Map subnodes = (Map)map.get("subnodes"); - gh = new ArrayList(subnodes.size()); - for (Map.Entry en: subnodes.entrySet()) { - GhostNode ghost = new GhostNode(null, null, null, en.getValue()); + Map subnodes = (Map) map.get("subnodes"); + gh = new ArrayList<>(subnodes.size()); + for (Map.Entry en: subnodes.entrySet()) { + Object sObj = en.getValue(); + int s; + if (sObj instanceof String) { // FIXME + s = Integer.parseInt((String) sObj); + } else { + s = (Integer) sObj; + } + GhostNode ghost = new GhostNode(null, null, null, s); ghost.setMeta(en.getKey()); gh.add(ghost); } } + + // FIXME + Object lkey = map.get("lkey"); + if ("null".equals(lkey)) { + lkey = null; + } + Object rkey = map.get("rkey"); + if ("null".equals(rkey)) { + rkey = null; + } + SkeletonNode node = new SkeletonNode( - (ktr == null)? (K)map.get("lkey"): ktr.rev((Q)map.get("lkey")), - (ktr == null)? (K)map.get("rkey"): ktr.rev((Q)map.get("rkey")), + (ktr == null) ? (K) lkey : ktr.rev((Q) lkey), + (ktr == null) ? (K) rkey : ktr.rev((Q) rkey), !notleaf, (mtr == null)? (SkeletonTreeMap)map.get("entries") : mtr.rev((R)map.get("entries")), @@ -1575,8 +1591,20 @@ public TreeTranslator(Translator k, Translator, ?> m /*@Override**/ public SkeletonBTreeMap rev(Map map) throws DataFormatException { try { - SkeletonBTreeMap tree = new SkeletonBTreeMap((Integer)map.get("node_min")); - tree.size = (Integer)map.get("size"); + Object nodeMinObj = map.get("node_min"); + int nodeMin; + if (nodeMinObj instanceof String) { // FIXME + nodeMin = Integer.parseInt((String) nodeMinObj); + } else { + nodeMin = (Integer) map.get("node_min"); + } + SkeletonBTreeMap tree = new SkeletonBTreeMap<>(nodeMin); + Object sizeObj = map.get("size"); + if (sizeObj instanceof String) { + tree.size = Integer.parseInt((String) sizeObj); + } else { + tree.size = (Integer) sizeObj; + } // map.put("lkey", null); // NULLNOTICE: get() gives null which matches // map.put("rkey", null); // NULLNOTICE: get() gives null which matches tree.root = tree.makeNodeTranslator(ktr, mtr).rev(map); From afee80553f175e5e4f15b979bb787c70c9acc107 Mon Sep 17 00:00:00 2001 From: Oleh Shklyar Date: Thu, 12 Dec 2019 21:31:26 +0200 Subject: [PATCH 9/9] clean --- src/plugins/Library/io/YamlReaderWriter.java | 21 ++++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/plugins/Library/io/YamlReaderWriter.java b/src/plugins/Library/io/YamlReaderWriter.java index f0a9eeae..1fd6b07d 100644 --- a/src/plugins/Library/io/YamlReaderWriter.java +++ b/src/plugins/Library/io/YamlReaderWriter.java @@ -166,7 +166,7 @@ public ExtendedConstructor() { this.yamlConstructors.put(new Tag("!FreenetURI"), new AbstractConstruct() { @Override public Object construct(Node node) { - String uri = constructScalar((ScalarNode)node); + String uri = constructScalar((ScalarNode) node); try { return new FreenetURI(uri); } catch (java.net.MalformedURLException e) { @@ -178,21 +178,20 @@ public Object construct(Node node) { this.yamlConstructors.put(new Tag("!BinInfo"), new AbstractConstruct() { @Override public Object construct(Node node) { - Map map = constructMapping((MappingNode)node); + Map map = constructMapping((MappingNode) node); if (map.size() != 1) { throw new ConstructorException("while constructing a Packer.BinInfo", node.getStartMark(), "found incorrectly sized map data " + map, null); } - for (Map.Entry en: map.entrySet()) { - int w; // FIXME - if (en.getValue() instanceof String) { - w = Integer.parseInt((String) en.getValue()); - } else { - w = (Integer) en.getValue(); - } - return new Packer.BinInfo(en.getKey(), w); + + Map.Entry entry = map.entrySet().iterator().next(); + int w; // FIXME + if (entry.getValue() instanceof String) { + w = Integer.parseInt((String) entry.getValue()); + } else { + w = (Integer) entry.getValue(); } - throw new AssertionError(); + return new Packer.BinInfo(entry.getKey(), w); } }); this.yamlConstructors.put(new Tag("!TermTermEntry"), new ConstructTermEntry<>(tebp_term));