diff --git a/.classpath b/.classpath deleted file mode 100644 index 97403d05..00000000 --- a/.classpath +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - diff --git a/.hgignore b/.hgignore new file mode 100644 index 00000000..de8451a4 --- /dev/null +++ b/.hgignore @@ -0,0 +1,7 @@ +build +shared/bin +uploader/bin +lib/snakeyaml-1.5.jar +dist/Library.jar +dist/uploader.jar +shared/TermEntryTest/test.yml diff --git a/.project b/.project deleted file mode 100644 index b40f21ce..00000000 --- a/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - Library - - - - - - org.eclipse.jdt.core.javabuilder - - - - - - org.eclipse.jdt.core.javanature - - diff --git a/README b/README index 6da0a190..8681abfb 100644 --- a/README +++ b/README @@ -4,6 +4,7 @@ Build: plugin-Library$ ant + == Javadoc == If you want to generate Javadocs, download bliki-doclet, which is a little @@ -56,3 +57,9 @@ this may change soon : + +== Ongoing work to split == + +The plugin is in src, test (for historical reasons). + +The uploader (standalone program) is in uploader/src and uploader/test depending on fcp and plugin. diff --git a/README.clean-build b/README.clean-build index 756e210d..ca465d38 100644 --- a/README.clean-build +++ b/README.clean-build @@ -1 +1,2 @@ -Library depends on SnakeYAML. It will fetch it from a known URL with a known version and checksums by default. If you want a completely clean build, fetch it yourself and build it yourself, and put it in lib/SnakeYAML-1.3.jar. +Library depends on SnakeYAML. It will fetch it from a known URL with a known version and checksums by default. +If you want a completely clean build, fetch it yourself and build it yourself, and put it in lib/SnakeYAML-1.3.jar. diff --git a/build.xml b/build.xml index fa7a6670..e5fad5ef 100644 --- a/build.xml +++ b/build.xml @@ -2,10 +2,10 @@ - + - - + + @@ -64,14 +64,14 @@ - - - + + + - - - + + + @@ -80,7 +80,7 @@ - + - + @@ -146,7 +146,7 @@ - + @@ -173,7 +173,7 @@ - + diff --git a/loop.sh b/loop.sh new file mode 100755 index 00000000..c8b87772 --- /dev/null +++ b/loop.sh @@ -0,0 +1,7 @@ +#!/bin/sh -x + +while test -f library.continue.loop +do + java -jar `dirname $0`/dist/uploader.jar + sleep 60 +done diff --git a/src/plugins/Library/ArchiverFactory.java b/src/plugins/Library/ArchiverFactory.java new file mode 100644 index 00000000..e881f6e9 --- /dev/null +++ b/src/plugins/Library/ArchiverFactory.java @@ -0,0 +1,18 @@ +package plugins.Library; + +import plugins.Library.io.ObjectStreamReader; +import plugins.Library.io.ObjectStreamWriter; +import plugins.Library.io.serial.LiveArchiver; +import plugins.Library.util.exec.SimpleProgress; + +public interface ArchiverFactory { + + LiveArchiver + newArchiver(S rw, String mime, int size, + Priority priorityLevel); + + + LiveArchiver + newArchiver(S rw, String mime, int size, + LiveArchiver archiver); +} diff --git a/src/plugins/Library/FactoryRegister.java b/src/plugins/Library/FactoryRegister.java new file mode 100644 index 00000000..47ca38e7 --- /dev/null +++ b/src/plugins/Library/FactoryRegister.java @@ -0,0 +1,14 @@ +package plugins.Library; + +public class FactoryRegister { + private static ArchiverFactory archiver = null; + + public static void register(ArchiverFactory factory) { + archiver = factory; + } + + public static ArchiverFactory getArchiverFactory() { + assert archiver != null; + return archiver; + } +} diff --git a/src/plugins/Library/Library.java b/src/plugins/Library/Library.java index 9a35b099..04d361d8 100644 --- a/src/plugins/Library/Library.java +++ b/src/plugins/Library/Library.java @@ -13,20 +13,15 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import plugins.Library.client.FreenetArchiver; -import plugins.Library.index.ProtoIndex; -import plugins.Library.index.ProtoIndexSerialiser; import plugins.Library.index.xml.URLUpdateHook; import plugins.Library.index.xml.XMLIndex; -import plugins.Library.io.ObjectStreamReader; -import plugins.Library.io.ObjectStreamWriter; -import plugins.Library.io.serial.Serialiser.PullTask; import plugins.Library.search.InvalidSearchException; -import plugins.Library.util.exec.TaskAbortException; import freenet.client.FetchContext; import freenet.client.FetchException; @@ -46,6 +41,18 @@ import freenet.client.events.ExpectedMIMEEvent; import freenet.keys.FreenetURI; import freenet.keys.USK; +import plugins.Library.ArchiverFactory; +import plugins.Library.FactoryRegister; +import plugins.Library.index.Index; +import plugins.Library.index.ProtoIndex; +import plugins.Library.index.ProtoIndexSerialiser; +import plugins.Library.io.ObjectStreamReader; +import plugins.Library.io.ObjectStreamWriter; +import plugins.Library.io.serial.LiveArchiver; +import plugins.Library.io.serial.Serialiser.PullTask; +import plugins.Library.Priority; +import plugins.Library.util.exec.SimpleProgress; +import plugins.Library.util.exec.TaskAbortException; import freenet.node.NodeClientCore; import freenet.node.RequestClient; import freenet.node.RequestStarter; @@ -55,12 +62,11 @@ import freenet.support.Logger; import freenet.support.io.FileUtil; - /** * Library class is the api for others to use search facilities, it is used by the interfaces * @author MikeB */ -final public class Library implements URLUpdateHook { +final public class Library implements URLUpdateHook, ArchiverFactory { public static final String BOOKMARK_PREFIX = "bookmark:"; public static final String DEFAULT_INDEX_SITE = BOOKMARK_PREFIX + "liberty-of-information" + " " + BOOKMARK_PREFIX + "free-market-free-people" + " " + @@ -68,6 +74,8 @@ final public class Library implements URLUpdateHook { private static int version = 36; public static final String plugName = "Library " + getVersion(); + + public static String getPlugName() { return plugName; } @@ -121,8 +129,9 @@ public boolean realTimeFlag() { * Method to setup Library class so it has access to PluginRespirator, and load bookmarks * TODO pull bookmarks from disk */ - private Library(PluginRespirator pr){ + private Library(PluginRespirator pr) { this.pr = pr; + FactoryRegister.register(this); PluginStore ps; if(pr!=null) { this.exec = pr.getNode().executor; @@ -184,7 +193,7 @@ private Library(PluginRespirator pr){ bookmarkCallbacks.put(name, callback); USK u; try { - u = USK.create(uri); + u = USK.create(new freenet.keys.FreenetURI(uri.toString())); } catch (MalformedURLException e) { Logger.error(this, "Invalid bookmark USK: "+target+" for "+name, e); continue; @@ -193,6 +202,12 @@ private Library(PluginRespirator pr){ callback.ret = uskManager.subscribeContent(u, callback, false, pr.getHLSimpleClient().getFetchContext(), RequestStarter.IMMEDIATE_SPLITFILE_PRIORITY_CLASS, rcBulk); } } + if (!bookmarks.containsKey("debbies-library-development-index")) { + addBookmark("debbies-library-development-index", + "USK@E0jWjfYUfJqESuiM~5ZklhTZXKCWapxl~CRj1jmZ-~I,gl48QSprqZC1mASLbE9EOhQoBa~PheO8r-q9Lqj~uXA,AQACAAE/index.yml/966"); + migrated = true; + Logger.normal(this, "Added new default index"); + } if(bookmarks.isEmpty() || needNewWanna || !bookmarks.containsKey("gotcha") || !bookmarks.containsKey("liberty-of-information") || !bookmarks.containsKey("free-market-free-people")) { @@ -241,16 +256,6 @@ public synchronized void saveState(){ * search for multiple terms in the same btree, but for now, turning off caching is the only viable option. */ -// /** -// ** Holds all the read-indexes. -// */ -// private Map rtab = new HashMap(); -// -// /** -// ** Holds all the writeable indexes. -// */ -// private Map wtab = new HashMap(); -// /** ** Holds all the bookmarks (aliases into the rtab). */ @@ -258,9 +263,13 @@ public synchronized void saveState(){ private Map bookmarkCallbacks = new HashMap(); + /** Set of all the enabled indices */ + public Set selectedIndices = new HashSet(); + /** ** Get the index type giving a {@code FreenetURI}. This must not contain ** a metastring (end with "/") or be a USK. + * @throws MalformedURLException */ public Class getIndexType(FreenetURI indexuri) throws FetchException { if(indexuri.lastMetaString()!=null && indexuri.lastMetaString().equals(XMLIndex.DEFAULT_FILE)) @@ -329,10 +338,10 @@ public Class getIndexType(FreenetURI indexuri) throws FetchException { public Class getIndexTypeFromMIME(String mime) { if (mime.equals(ProtoIndex.MIME_TYPE)) { - //return "YAML index"; + // YAML index return ProtoIndex.class; } else if (mime.equals(XMLIndex.MIME_TYPE)) { - //return "XML index"; + // XML index return XMLIndex.class; } else { throw new UnsupportedOperationException("Unknown mime-type for index: "+mime); @@ -340,103 +349,6 @@ public Class getIndexTypeFromMIME(String mime) { } -/* - KEYEXPLORER slightly more efficient version that depends on KeyExplorer - - /** - ** Get the index type giving a {@code FreenetURI}. This should have been - ** passed through {@link KeyExplorerUtils#sanitizeURI(List, String)} at - ** some point - ie. it must not contain a metastring (end with "/") or be - ** a USK. - * / - public Class getIndexType(FreenetURI uri) - throws FetchException, IOException, MetadataParseException, LowLevelGetException, KeyListenerConstructionException { - GetResult getresult = KeyExplorerUtils.simpleGet(pr, uri); - byte[] data = BucketTools.toByteArray(getresult.getData()); - - if (getresult.isMetaData()) { - try { - Metadata md = Metadata.construct(data); - - if (md.isArchiveManifest()) { - if (md.getArchiveType() == ARCHIVE_TYPE.TAR) { - return getIndexTypeFromManifest(uri, false, true); - - } else if (md.getArchiveType() == ARCHIVE_TYPE.ZIP) { - return getIndexTypeFromManifest(uri, true, false); - - } else { - throw new UnsupportedOperationException("not implemented - unknown archive manifest"); - } - - } else if (md.isSimpleManifest()) { - return getIndexTypeFromManifest(uri, false, false); - } - - return getIndexTypeFromSimpleMetadata(md); - - } catch (MetadataParseException e) { - throw new RuntimeException(e); - } - } else { - throw new UnsupportedOperationException("Found data instead of metadata; I do not have enough intelligence to decode this."); - } - } - - public Class getIndexTypeFromSimpleMetadata(Metadata md) { - String mime = md.getMIMEType(); - if (mime.equals(ProtoIndex.MIME_TYPE)) { - //return "YAML index"; - return ProtoIndex.class; - } else if (mime.equals(XMLIndex.MIME_TYPE)) { - //return "XML index"; - return XMLIndex.class; - } else { - throw new UnsupportedOperationException("Unknown mime-type for index"); - } - } - - public Class getIndexTypeFromManifest(FreenetURI furi, boolean zip, boolean tar) - throws FetchException, IOException, MetadataParseException, LowLevelGetException, KeyListenerConstructionException { - - boolean automf = true, deep = true, ml = true; - Metadata md = null; - - if (zip) - md = KeyExplorerUtils.zipManifestGet(pr, furi); - else if (tar) - md = KeyExplorerUtils.tarManifestGet(pr, furi, ".metadata"); - else { - md = KeyExplorerUtils.simpleManifestGet(pr, furi); - if (ml) { - md = KeyExplorerUtils.splitManifestGet(pr, md); - } - } - - if (md.isSimpleManifest()) { - // a subdir - HashMap docs = md.getDocuments(); - Metadata defaultDoc = md.getDefaultDocument(); - - if (defaultDoc != null) { - //return "(default doc method) " + getIndexTypeFromSimpleMetadata(defaultDoc); - return getIndexTypeFromSimpleMetadata(defaultDoc); - } - - if (docs.containsKey(ProtoIndex.DEFAULT_FILE)) { - //return "(doclist method) YAML index"; - return ProtoIndex.class; - } else if (docs.containsKey(XMLIndex.DEFAULT_FILE)) { - //return "(doclist method) XML index"; - return XMLIndex.class; - } else { - throw new UnsupportedOperationException("Could not find a supported index in the document-listings for " + furi.toString()); - } - } - - throw new UnsupportedOperationException("Parsed metadata but did not reach a simple manifest: " + furi.toString()); - } -*/ public Class getIndexType(File f) { if (f.getName().endsWith(ProtoIndexSerialiser.FILE_EXTENSION)) return ProtoIndex.class; @@ -451,9 +363,7 @@ public Object getAddressTypeFromString(String indexuri) { // return KeyExplorerUtils.sanitizeURI(new ArrayList(), indexuri); KEYEXPLORER // OPT HIGH if it already ends with eg. *Index.DEFAULT_FILE, don't strip // the MetaString, and have getIndexType behave accordingly - FreenetURI tempURI = new FreenetURI(indexuri); -// if (tempURI.hasMetaStrings()) { tempURI = tempURI.setMetaString(null); } -// if (tempURI.isUSK()) { tempURI = tempURI.sskForUSK(); } + plugins.Library.io.FreenetURI tempURI = new plugins.Library.io.FreenetURI(indexuri); return tempURI; } catch (MalformedURLException e) { File file = new File(indexuri); @@ -480,7 +390,7 @@ public String addBookmark(String name, String uri) { try { u = new FreenetURI(uri); if(u.isUSK()) { - uskNew = USK.create(u); + uskNew = USK.create(new freenet.keys.FreenetURI(u.toString())); edition = uskNew.suggestedEdition; } } catch (MalformedURLException e) { @@ -503,7 +413,7 @@ public String addBookmark(String name, String uri) { try { FreenetURI uold = new FreenetURI(old); if(uold.isUSK()) { - USK usk = USK.create(uold); + USK usk = USK.create(new freenet.keys.FreenetURI(uold.toString())); if(!(uskNew != null && usk.equals(uskNew, false))) { uskManager.unsubscribe(usk, callback); uskManager.unsubscribeContent(usk, callback.ret, true); @@ -606,14 +516,6 @@ public final ArrayList getIndices(String indexuris) throws InvalidSearchE return indices; } - // See comments near rtab. Can't use in parallel so not acceptable. -// /** -// * Method to get all of the instatiated Indexes -// */ -// public final Iterable getAllIndices() { -// return rtab.values(); -// } -// public final Index getIndex(String indexuri) throws InvalidSearchException, TaskAbortException { return getIndex(indexuri, null); } @@ -644,10 +546,6 @@ public final Index getIndex(String indexuri, String origIndexName) throws Invali throw new InvalidSearchException("Index bookmark '"+indexuri+" does not exist"); } - // See comments near rtab. Can't use in parallel so caching is dangerous. -// if (rtab.containsKey(indexuri)) -// return rtab.get(indexuri); -// Class indextype; Index index; Object indexkey; @@ -663,9 +561,9 @@ public final Index getIndex(String indexuri, String origIndexName) throws Invali try { if (indexkey instanceof File) { indextype = getIndexType((File)indexkey); - } else if (indexkey instanceof FreenetURI) { + } else if (indexkey instanceof plugins.Library.io.FreenetURI) { // TODO HIGH make this non-blocking - FreenetURI uri = (FreenetURI)indexkey; + FreenetURI uri = new FreenetURI(indexkey.toString()); if(uri.isUSK()) edition = uri.getEdition(); indextype = getIndexType(uri); @@ -676,7 +574,7 @@ public final Index getIndex(String indexuri, String origIndexName) throws Invali if (indextype == ProtoIndex.class) { // TODO HIGH this *must* be non-blocking as it fetches the whole index root PullTask task = new PullTask(indexkey); - ProtoIndexSerialiser.forIndex(indexkey, RequestStarter.INTERACTIVE_PRIORITY_CLASS).pull(task); + ProtoIndexSerialiser.forIndex(indexkey, Priority.Interactive).pull(task); index = task.data; } else if (indextype == XMLIndex.class) { @@ -686,33 +584,22 @@ public final Index getIndex(String indexuri, String origIndexName) throws Invali throw new AssertionError(); } - // See comments near rtab. Can't use in parallel so caching is dangerous. - //rtab.put(indexuri, index); Logger.normal(this, "Loaded index type " + indextype.getName() + " at " + indexuri); return index; + } catch (MalformedURLException e) { + Logger.warning(this, "Failed to find index type", e); + throw new TaskAbortException("Failed to find index type " + indexuri+" : "+e, e, true); } catch (FetchException e) { - throw new TaskAbortException("Failed to fetch index " + indexuri+" : "+e, e, true); // can retry -/* KEYEXPLORER - } catch (IOException e) { - throw new TaskAbortException("Failed to fetch index " + indexuri, e, true); // can retry - - } catch (LowLevelGetException e) { - throw new TaskAbortException("Failed to fetch index " + indexuri, e, true); // can retry - - } catch (KeyListenerConstructionException e) { - throw new TaskAbortException("Failed to fetch index " + indexuri, e, true); // can retry - - } catch (MetadataParseException e) { - throw new TaskAbortException("Failed to parse index " + indexuri, e); -*/ + Logger.warning(this, "Failed to find fetch index", e); + throw new TaskAbortException("Failed to fetch index " + indexuri+" : "+e, e, true); } catch (UnsupportedOperationException e) { + Logger.warning(this, "Failed to find parse index", e); throw new TaskAbortException("Failed to parse index " + indexuri+" : "+e, e); - } catch (RuntimeException e) { + Logger.warning(this, "Failed to find load index", e); throw new TaskAbortException("Failed to load index " + indexuri+" : "+e, e); - } } @@ -724,7 +611,7 @@ public final Index getIndex(String indexuri, String origIndexName) throws Invali ** @throws IllegalStateException if the singleton has not been initialised ** or if it does not have a respirator. */ - public static FreenetArchiver + public static LiveArchiver makeArchiver(ObjectStreamReader r, ObjectStreamWriter w, String mime, int size, short priorityClass) { if (lib == null || lib.pr == null) { throw new IllegalStateException("Cannot archive to freenet without a fully live Library plugin connected to a freenet node."); @@ -732,7 +619,7 @@ public final Index getIndex(String indexuri, String origIndexName) throws Invali return new FreenetArchiver(lib.pr.getNode().clientCore, r, w, mime, size, priorityClass); } } - + /** ** Create a {@link FreenetArchiver} connected to the core of the ** singleton's {@link PluginRespirator}. @@ -740,11 +627,35 @@ public final Index getIndex(String indexuri, String origIndexName) throws Invali ** @throws IllegalStateException if the singleton has not been initialised ** or if it does not have a respirator. */ - public static FreenetArchiver + public static LiveArchiver makeArchiver(S rw, String mime, int size, short priorityClass) { return Library.makeArchiver(rw, rw, mime, size, priorityClass); } + public LiveArchiver + newArchiver(S rw, String mime, int size, Priority priorityLevel) { + short priorityClass = 0; + switch (priorityLevel) { + case Interactive: + priorityClass = RequestStarter.INTERACTIVE_PRIORITY_CLASS; + break; + case Bulk: + priorityClass = RequestStarter.BULK_SPLITFILE_PRIORITY_CLASS; + break; + } + return makeArchiver(rw, mime, size, priorityClass); + } + + public LiveArchiver + newArchiver(S rw, String mime, int size, LiveArchiver archiver) { + short priorityClass = RequestStarter.BULK_SPLITFILE_PRIORITY_CLASS; + if (archiver != null && + archiver instanceof FreenetArchiver) + priorityClass = ((FreenetArchiver) archiver).priorityClass; + + return makeArchiver(rw, mime, size, priorityClass); + } + public static String convertToHex(byte[] data) { StringBuilder buf = new StringBuilder(); for (int i = 0; i < data.length; i++) { diff --git a/src/plugins/Library/Main.java b/src/plugins/Library/Main.java index 600cb330..f45a1e69 100644 --- a/src/plugins/Library/Main.java +++ b/src/plugins/Library/Main.java @@ -3,76 +3,25 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library; -import freenet.node.RequestStarter; import freenet.pluginmanager.PluginReplySender; -import freenet.support.MutableBoolean; import freenet.support.SimpleFieldSet; import freenet.support.api.Bucket; -import freenet.support.io.BucketTools; -import freenet.support.io.Closer; -import freenet.support.io.FileBucket; -import freenet.support.io.FileUtil; -import freenet.support.io.LineReadingInputStream; -import freenet.support.io.NativeThread; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.logging.Level; - -import plugins.Library.client.FreenetArchiver; -import plugins.Library.index.ProtoIndex; -import plugins.Library.index.ProtoIndexComponentSerialiser; -import plugins.Library.index.ProtoIndexSerialiser; -import plugins.Library.index.TermEntry; -import plugins.Library.index.TermPageEntry; + import plugins.Library.search.Search; import plugins.Library.ui.WebInterface; -import plugins.Library.util.SkeletonBTreeMap; -import plugins.Library.util.SkeletonBTreeSet; -import plugins.Library.util.TaskAbortExceptionConvertor; -import plugins.Library.util.concurrent.Executors; -import plugins.Library.util.exec.SimpleProgress; -import plugins.Library.util.exec.TaskAbortException; -import plugins.Library.util.func.Closure; import freenet.pluginmanager.FredPlugin; import freenet.pluginmanager.FredPluginL10n; import freenet.pluginmanager.FredPluginRealVersioned; import freenet.pluginmanager.FredPluginThreadless; import freenet.pluginmanager.FredPluginVersioned; -import freenet.pluginmanager.PluginNotFoundException; import freenet.pluginmanager.PluginRespirator; import freenet.support.Executor; -import freenet.client.InsertException; -import freenet.keys.FreenetURI; -import freenet.keys.InsertableClientSSK; -import freenet.l10n.BaseL10n.LANGUAGE; +import plugins.Library.util.concurrent.Executors; import freenet.pluginmanager.FredPluginFCP; import freenet.support.Logger; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.EOFException; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.FileWriter; -import java.io.FilenameFilter; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; import java.security.MessageDigest; -import plugins.Library.index.TermEntryReaderWriter; -import plugins.Library.index.xml.LibrarianHandler; -import plugins.Library.io.serial.LiveArchiver; -import plugins.Library.io.serial.Serialiser.PullTask; -import plugins.Library.io.serial.Serialiser.PushTask; /** * Library class is the api for others to use search facilities, it is used by the interfaces @@ -85,10 +34,7 @@ public class Main implements FredPlugin, FredPluginVersioned, freenet.pluginmana private Library library; private WebInterface webinterface; private SpiderIndexUploader uploader; - - static volatile boolean logMINOR; - static volatile boolean logDEBUG; - + static { Logger.registerClass(Main.class); } @@ -195,5 +141,4 @@ public void handle(PluginReplySender replysender, SimpleFieldSet params, final B Logger.error(this, "Unknown command : \""+params.get("command")); } } - } diff --git a/src/plugins/Library/Priority.java b/src/plugins/Library/Priority.java new file mode 100644 index 00000000..0f702d26 --- /dev/null +++ b/src/plugins/Library/Priority.java @@ -0,0 +1,6 @@ +package plugins.Library; + +public enum Priority { + Interactive, + Bulk; +} diff --git a/src/plugins/Library/SpiderIndexURIs.java b/src/plugins/Library/SpiderIndexURIs.java index 46ce8edd..e6ee89e6 100644 --- a/src/plugins/Library/SpiderIndexURIs.java +++ b/src/plugins/Library/SpiderIndexURIs.java @@ -24,11 +24,6 @@ class SpiderIndexURIs { this.pr = pr; } - synchronized long setEdition(long newEdition) { - if(newEdition < edition) return edition; - else return edition = newEdition; - } - synchronized FreenetURI loadSSKURIs() { if(privURI == null) { File f = new File(SpiderIndexUploader.PRIV_URI_FILENAME); @@ -82,29 +77,32 @@ synchronized FreenetURI loadSSKURIs() { } finally { Closer.close(fos); } - try { - fis = new FileInputStream(new File(SpiderIndexUploader.EDITION_FILENAME)); - BufferedReader br = new BufferedReader(new InputStreamReader(fis, "UTF-8")); - try { - edition = Long.parseLong(br.readLine()); - } catch (NumberFormatException e) { - edition = 0; - } - Logger.debug(this, "Edition: "+edition); - fis.close(); - fis = null; - } catch (IOException e) { - // Ignore - edition = 0; - } finally { - Closer.close(fis); - } +//<<<<<<< HEAD +// try { +// fis = new FileInputStream(new File(SpiderIndexUploader.EDITION_FILENAME)); +// BufferedReader br = new BufferedReader(new InputStreamReader(fis, "UTF-8")); +// try { +// edition = Long.parseLong(br.readLine()); +// } catch (NumberFormatException e) { +// edition = 0; +// } +// Logger.debug(this, "Edition: "+edition); +// fis.close(); +// fis = null; +// } catch (IOException e) { +// // Ignore +// edition = 0; +// } finally { +// Closer.close(fis); +// } +//======= +//>>>>>>> debbiedub/fcp-uploader } return privURI; } synchronized FreenetURI getPrivateUSK() { - return loadSSKURIs().setKeyType("USK").setDocName(SpiderIndexUploader.INDEX_DOCNAME).setSuggestedEdition(edition); + return loadSSKURIs().setKeyType("USK").setDocName(SpiderIndexUploader.INDEX_DOCNAME).setSuggestedEdition(getLastUploadedEdition()); } /** Will return edition -1 if no successful uploads so far, otherwise the correct edition. */ @@ -114,8 +112,23 @@ synchronized FreenetURI getPublicUSK() { } private synchronized long getLastUploadedEdition() { - /** If none uploaded, return -1, otherwise return the last uploaded version. */ - return edition-1; + FileInputStream fis = null; + try { + fis = new FileInputStream(new File(SpiderIndexUploader.EDITION_FILENAME)); + BufferedReader br = new BufferedReader(new InputStreamReader(fis, "UTF-8")); + try { + edition = Long.parseLong(br.readLine()); + } catch (NumberFormatException e) { + Logger.error(this, "Failed to parse edition", e); + } + fis.close(); + fis = null; + } catch (IOException e) { + Logger.error(this, "Failed to read edition", e); + } finally { + Closer.close(fis); + } + return edition; } } \ No newline at end of file diff --git a/src/plugins/Library/SpiderIndexUploader.java b/src/plugins/Library/SpiderIndexUploader.java index 9b853337..8dbca853 100644 --- a/src/plugins/Library/SpiderIndexUploader.java +++ b/src/plugins/Library/SpiderIndexUploader.java @@ -1,57 +1,19 @@ package plugins.Library; -import java.io.BufferedReader; -import java.io.EOFException; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.FilenameFilter; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.MalformedURLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.logging.Level; -import plugins.Library.client.FreenetArchiver; -import plugins.Library.index.ProtoIndex; -import plugins.Library.index.ProtoIndexComponentSerialiser; -import plugins.Library.index.ProtoIndexSerialiser; -import plugins.Library.index.TermEntry; -import plugins.Library.index.TermEntryReaderWriter; -import plugins.Library.io.serial.LiveArchiver; -import plugins.Library.io.serial.Serialiser.PullTask; -import plugins.Library.io.serial.Serialiser.PushTask; -import plugins.Library.util.SkeletonBTreeMap; -import plugins.Library.util.SkeletonBTreeSet; -import plugins.Library.util.TaskAbortExceptionConvertor; -import plugins.Library.util.exec.SimpleProgress; -import plugins.Library.util.exec.TaskAbortException; -import plugins.Library.util.func.Closure; -import freenet.client.InsertException; import freenet.keys.FreenetURI; -import freenet.node.RequestStarter; import freenet.pluginmanager.PluginNotFoundException; import freenet.pluginmanager.PluginReplySender; import freenet.pluginmanager.PluginRespirator; import freenet.support.Logger; -import freenet.support.MutableBoolean; import freenet.support.SimpleFieldSet; -import freenet.support.TimeUtil; import freenet.support.api.Bucket; import freenet.support.io.BucketTools; -import freenet.support.io.Closer; import freenet.support.io.FileBucket; -import freenet.support.io.FileUtil; -import freenet.support.io.LineReadingInputStream; + public class SpiderIndexUploader { @@ -66,828 +28,835 @@ public class SpiderIndexUploader { } private final PluginRespirator pr; - private Object freenetMergeSync = new Object(); - private boolean freenetMergeRunning = false; - private boolean diskMergeRunning = false; - - private final ArrayList toMergeToDisk = new ArrayList(); - static final int MAX_HANDLING_COUNT = 5; - // When pushing is broken, allow max handling to reach this level before stalling forever to prevent running out of disk space. - private int PUSH_BROKEN_MAX_HANDLING_COUNT = 10; - // Don't use too much disk space, take into account fact that Spider slows down over time. - - private boolean pushBroken; - - /** The temporary on-disk index. We merge stuff into this until it exceeds a threshold size, then - * we create a new diskIdx and merge the old one into the idxFreenet. */ - ProtoIndex idxDisk; - /** idxDisk gets merged into idxFreenet this long after the last merge completed. */ - static final long MAX_TIME = 24*60*60*1000L; - /** idxDisk gets merged into idxFreenet after this many incoming updates from Spider. */ - static final int MAX_UPDATES = 16; - /** idxDisk gets merged into idxFreenet after it has grown to this many terms. - * Note that the entire main tree of terms (not the sub-trees with the positions and urls in) must - * fit into memory during the merge process. */ - static final int MAX_TERMS = 100*1000; - /** idxDisk gets merged into idxFreenet after it has grown to this many terms. - * Note that the entire main tree of terms (not the sub-trees with the positions and urls in) must - * fit into memory during the merge process. */ - static final int MAX_TERMS_NOT_UPLOADED = 10*1000; - /** Maximum size of a single entry, in TermPageEntry count, on disk. If we exceed this we force an - * insert-to-freenet and move on to a new disk index. The problem is that the merge to Freenet has - * to keep the whole of each entry in RAM. This is only true for the data being merged in - the - * on-disk index - and not for the data on Freenet, which is pulled on demand. SCALABILITY */ - static final int MAX_DISK_ENTRY_SIZE = 10000; - /** Like pushNumber, the number of the current disk dir, used to create idxDiskDir. */ - private int dirNumber; - static final String DISK_DIR_PREFIX = "library-temp-index-"; - /** Directory the current idxDisk is saved in. */ - File idxDiskDir; - private int mergedToDisk; - - ProtoIndexSerialiser srl = null; - FreenetURI lastUploadURI = null; - String lastDiskIndexName; - /** The uploaded index on Freenet. This never changes, it just gets updated. */ - ProtoIndex idxFreenet; - +//<<<<<<< HEAD +// private final Object freenetMergeSync = new Object(); +// private boolean freenetMergeRunning = false; +// private boolean diskMergeRunning = false; +// +// private final ArrayList toMergeToDisk = new ArrayList(); +// static final int MAX_HANDLING_COUNT = 5; +// // When pushing is broken, allow max handling to reach this level before stalling forever to prevent running out of disk space. +// private int PUSH_BROKEN_MAX_HANDLING_COUNT = 10; +// // Don't use too much disk space, take into account fact that Spider slows down over time. +// +// // Flag for the Spider plugin to hang +// private boolean pushBroken; +// +// /** The temporary on-disk index. We merge stuff into this until it exceeds a threshold size, then +// * we create a new diskIdx and merge the old one into the idxFreenet. */ +// ProtoIndex idxDisk; +// /** idxDisk gets merged into idxFreenet this long after the last merge completed. */ +// static final long MAX_TIME = 24*60*60*1000L; +// /** idxDisk gets merged into idxFreenet after this many incoming updates from Spider. */ +// static final int MAX_UPDATES = 16; +// /** idxDisk gets merged into idxFreenet after it has grown to this many terms. +// * Note that the entire main tree of terms (not the sub-trees with the positions and urls in) must +// * fit into memory during the merge process. */ +// static final int MAX_TERMS = 100*1000; +// /** idxDisk gets merged into idxFreenet after it has grown to this many terms. +// * Note that the entire main tree of terms (not the sub-trees with the positions and urls in) must +// * fit into memory during the merge process. */ +// static final int MAX_TERMS_NOT_UPLOADED = 10*1000; +// /** Maximum size of a single entry, in TermPageEntry count, on disk. If we exceed this we force an +// * insert-to-freenet and move on to a new disk index. The problem is that the merge to Freenet has +// * to keep the whole of each entry in RAM. This is only true for the data being merged in - the +// * on-disk index - and not for the data on Freenet, which is pulled on demand. SCALABILITY */ +// static final int MAX_DISK_ENTRY_SIZE = 10000; +// /** Like pushNumber, the number of the current disk dir, used to create idxDiskDir. */ +// private int dirNumber; +// static final String DISK_DIR_PREFIX = "library-temp-index-"; +// /** Directory the current idxDisk is saved in. */ +// File idxDiskDir; +// private int mergedToDisk; +// +// ProtoIndexSerialiser srl = null; +// FreenetURI lastUploadURI = null; +// String lastDiskIndexName; +// /** The uploaded index on Freenet. This never changes, it just gets updated. */ +// ProtoIndex idxFreenet; +// +//======= +//>>>>>>> debbiedub/fcp-uploader private final SpiderIndexURIs spiderIndexURIs; - long pushNumber; - static final String LAST_URL_FILENAME = "library.index.lastpushed.chk"; + private long pushNumber; static final String PRIV_URI_FILENAME = "library.index.privkey"; static final String PUB_URI_FILENAME = "library.index.pubkey"; - static final String EDITION_FILENAME = "library.index.next-edition"; - - static final String LAST_DISK_FILENAME = "library.index.lastpushed.disk"; + static final String EDITION_FILENAME = "library.index.last-edition"; static final String BASE_FILENAME_PUSH_DATA = "library.index.data."; - /** Merge from the Bucket chain to the on-disk idxDisk. */ - protected void wrapMergeToDisk() { - spiderIndexURIs.loadSSKURIs(); - boolean first = true; - while(true) { - final Bucket data; - synchronized(freenetMergeSync) { - if(pushBroken) { - Logger.error(this, "Pushing broken"); - return; - } - if(first && diskMergeRunning) { - Logger.error(this, "Already running a handler!"); - return; - } else if((!first) && (!diskMergeRunning)) { - Logger.error(this, "Already running yet runningHandler is false?!"); - return; - } - first = false; - if(toMergeToDisk.size() == 0) { - if(logMINOR) Logger.minor(this, "Nothing to handle"); - diskMergeRunning = false; - freenetMergeSync.notifyAll(); - return; - } - data = toMergeToDisk.remove(0); - freenetMergeSync.notifyAll(); - diskMergeRunning = true; - } - try { - mergeToDisk(data); - } catch (Throwable t) { - // Failed. - synchronized(freenetMergeSync) { - diskMergeRunning = false; - pushBroken = true; - freenetMergeSync.notifyAll(); - } - if(t instanceof RuntimeException) - throw (RuntimeException)t; - if(t instanceof Error) - throw (Error)t; - } - } - } - - // This is a member variable because it is huge, and having huge stuff in local variables seems to upset the default garbage collector. - // It doesn't need to be synchronized because it's always used from mergeToDisk, which never runs in parallel. - private Map> newtrees; - // Ditto - private SortedSet terms; - - ProtoIndexSerialiser srlDisk = null; - private ProtoIndexComponentSerialiser leafsrlDisk; - - private long lastMergedToFreenet = -1; - - /** Merge a bucket of TermEntry's into an on-disk index. */ - private void mergeToDisk(Bucket data) { - - boolean newIndex = false; - - if(idxDiskDir == null) { - newIndex = true; - if(!createDiskDir()) return; - } - - if(!makeDiskDirSerialiser()) return; - - // Read data into newtrees and trees. - long entriesAdded = readTermsFrom(data); - - if(terms.size() == 0) { - Logger.debug(this, "Nothing to merge"); - synchronized(this) { - newtrees = null; - terms = null; - } - return; - } - - // Merge the new data to the disk index. - - try { - final MutableBoolean maxDiskEntrySizeExceeded = new MutableBoolean(); - maxDiskEntrySizeExceeded.value = false; - long mergeStartTime = System.currentTimeMillis(); - if(newIndex) { - if(createDiskIndex()) - maxDiskEntrySizeExceeded.value = true; - } else { - // async merge - Closure>, TaskAbortException> clo = - createMergeFromNewtreesClosure(maxDiskEntrySizeExceeded); - assert(idxDisk.ttab.isBare()); - Logger.debug(this, "Merging "+terms.size()+" terms, tree.size = "+idxDisk.ttab.size()+" from "+data+"..."); - idxDisk.ttab.update(terms, null, clo, new TaskAbortExceptionConvertor()); - - } - // Synchronize anyway so garbage collector knows about it. - synchronized(this) { - newtrees = null; - terms = null; - } - assert(idxDisk.ttab.isBare()); - PushTask task4 = new PushTask(idxDisk); - srlDisk.push(task4); - - long mergeEndTime = System.currentTimeMillis(); - Logger.debug(this, entriesAdded + " entries merged to disk in " + (mergeEndTime-mergeStartTime) + " ms, root at " + task4.meta + ", "); - // FileArchiver produces a String, which is a filename not including the prefix or suffix. - String uri = (String)task4.meta; - lastDiskIndexName = uri; - Logger.normal(this, "Pushed new index to file "+uri); - if(writeStringTo(new File(LAST_DISK_FILENAME), uri) && - writeStringTo(new File(idxDiskDir, LAST_DISK_FILENAME), uri)) { - // Successfully uploaded and written new status. Can delete the incoming data. - data.free(); - } - - maybeMergeToFreenet(maxDiskEntrySizeExceeded); - } catch (TaskAbortException e) { - Logger.error(this, "Failed to upload index for spider: ", e); - e.printStackTrace(); - synchronized(freenetMergeSync) { - pushBroken = true; - } - } - } - - /** We have just written a Bucket of new data to an on-disk index. We may or may not want to - * upload to an on-Freenet index, depending on how big the data is etc. If we do, we will need - * to create a new on-disk index. - * @param maxDiskEntrySizeExceeded A flag object which is set (off-thread) if any single term - * in the index is very large. - */ - private void maybeMergeToFreenet(MutableBoolean maxDiskEntrySizeExceeded) { - // Maybe chain to mergeToFreenet ??? - - boolean termTooBig = false; - synchronized(maxDiskEntrySizeExceeded) { - termTooBig = maxDiskEntrySizeExceeded.value; - } - - mergedToDisk++; - if((lastMergedToFreenet > 0 && idxDisk.ttab.size() > MAX_TERMS) || - (idxDisk.ttab.size() > MAX_TERMS_NOT_UPLOADED) - || (mergedToDisk > MAX_UPDATES) || termTooBig || - (lastMergedToFreenet > 0 && (System.currentTimeMillis() - lastMergedToFreenet) > MAX_TIME)) { - - final ProtoIndex diskToMerge = idxDisk; - final File dir = idxDiskDir; - Logger.debug(this, "Exceeded threshold, starting new disk index and starting merge from disk to Freenet..."); - mergedToDisk = 0; - lastMergedToFreenet = -1; - idxDisk = null; - srlDisk = null; - leafsrlDisk = null; - idxDiskDir = null; - lastDiskIndexName = null; - - synchronized(freenetMergeSync) { - while(freenetMergeRunning) { - if(pushBroken) return; - Logger.normal(this, "Need to merge to Freenet, but last merge not finished yet. Waiting..."); - try { - freenetMergeSync.wait(); - } catch (InterruptedException e) { - // Ignore - } - } - if(pushBroken) return; - freenetMergeRunning = true; - } - - Runnable r = new Runnable() { - - public void run() { - try { - mergeToFreenet(diskToMerge, dir); - } catch (Throwable t) { - Logger.error(this, "Merge to Freenet failed: ", t); - t.printStackTrace(); - synchronized(freenetMergeSync) { - pushBroken = true; - } - } finally { - synchronized(freenetMergeSync) { - freenetMergeRunning = false; - if(!pushBroken) - lastMergedToFreenet = System.currentTimeMillis(); - freenetMergeSync.notifyAll(); - } - } - } - - }; - pr.getNode().executor.execute(r, "Library: Merge data from disk to Freenet"); - } else { - Logger.debug(this, "Not merging to Freenet yet: "+idxDisk.ttab.size()+" terms in index, "+mergedToDisk+" merges, "+(lastMergedToFreenet <= 0 ? "never merged to Freenet" : ("last merged to Freenet "+TimeUtil.formatTime(System.currentTimeMillis() - lastMergedToFreenet))+"ago")); - } - } - - private boolean writeURITo(File filename, FreenetURI uri) { - return writeStringTo(filename, uri.toString()); - } - - private boolean writeStringTo(File filename, String uri) { - FileOutputStream fos = null; - try { - fos = new FileOutputStream(filename); - OutputStreamWriter osw = new OutputStreamWriter(fos, "UTF-8"); - osw.write(uri.toString()); - osw.close(); - fos = null; - return true; - } catch (IOException e) { - Logger.error(this, "Failed to write to "+filename+" : "+uri, e); - return false; - } finally { - Closer.close(fos); - } - } - - private String readStringFrom(File file) { - String ret; - FileInputStream fis = null; - try { - fis = new FileInputStream(file); - BufferedReader br = new BufferedReader(new InputStreamReader(fis, "UTF-8")); - ret = br.readLine(); - fis.close(); - fis = null; - return ret; - } catch (IOException e) { - // Ignore - return null; - } finally { - Closer.close(fis); - } - } - - private FreenetURI readURIFrom(File file) { - String s = readStringFrom(file); - if(s != null) { - try { - return new FreenetURI(s); - } catch (MalformedURLException e) { - // Ignore. - } - } - return null; - } - - /** Create a callback object which will do the merging of individual terms. This will be called - * for each term as it is unpacked from the existing on-disk index. It then merges in new data - * from newtrees and writes the subtree for the term back to disk. Most of the work is done in - * update() below. - * @param maxDiskEntrySizeExceeded Will be set if any single term is so large that we need to - * upload to Freenet immediately. */ - private Closure>, TaskAbortException> createMergeFromNewtreesClosure(final MutableBoolean maxDiskEntrySizeExceeded) { - return new - Closure>, TaskAbortException>() { - /*@Override**/ public void invoke(Map.Entry> entry) throws TaskAbortException { - String key = entry.getKey(); - SkeletonBTreeSet tree = entry.getValue(); - if(logMINOR) Logger.minor(this, "Processing: "+key+" : "+tree); - if(tree != null) - Logger.debug(this, "Merging data (on disk) in term "+key); - else - Logger.debug(this, "Adding new term to disk index: "+key); - //System.out.println("handling " + key + ((tree == null)? " (new)":" (old)")); - if (tree == null) { - entry.setValue(tree = makeEntryTree(leafsrlDisk)); - } - assert(tree.isBare()); - SortedSet toMerge = newtrees.get(key); - tree.update(toMerge, null); - if(toMerge.size() > MAX_DISK_ENTRY_SIZE) - synchronized(maxDiskEntrySizeExceeded) { - maxDiskEntrySizeExceeded.value = true; - } - toMerge = null; - newtrees.remove(key); - assert(tree.isBare()); - if(logMINOR) Logger.minor(this, "Updated: "+key+" : "+tree); - //System.out.println("handled " + key); - } - }; - } - - /** Create a new on-disk index from terms and newtrees. - * @return True if the size of any one item in the index is so large that we must upload - * immediately to Freenet. - * @throws TaskAbortException If something broke catastrophically. */ - private boolean createDiskIndex() throws TaskAbortException { - boolean tooBig = false; - // created a new index, fill it with data. - // DON'T MERGE, merge with a lot of data will deadlock. - // FIXME throw in update() if it will deadlock. - for(String key : terms) { - SkeletonBTreeSet tree = makeEntryTree(leafsrlDisk); - SortedSet toMerge = newtrees.get(key); - tree.addAll(toMerge); - if(toMerge.size() > MAX_DISK_ENTRY_SIZE) - tooBig = true; - toMerge = null; - tree.deflate(); - assert(tree.isBare()); - idxDisk.ttab.put(key, tree); - } - idxDisk.ttab.deflate(); - return tooBig; - } - - /** Read the TermEntry's from the Bucket into newtrees and terms, and set up the index - * properties. - * @param data The Bucket containing TermPageEntry's etc serialised with TermEntryReaderWriter. - */ - private long readTermsFrom(Bucket data) { - FileWriter w = null; - newtrees = new HashMap>(); - terms = new TreeSet(); - int entriesAdded = 0; - InputStream is = null; - try { - Logger.normal(this, "Bucket of buffer received, "+data.size()+" bytes"); - is = data.getInputStream(); - SimpleFieldSet fs = new SimpleFieldSet(new LineReadingInputStream(is), 1024, 512, true, true, true); - idxDisk.setName(fs.get("index.title")); - idxDisk.setOwnerEmail(fs.get("index.owner.email")); - idxDisk.setOwner(fs.get("index.owner.name")); - idxDisk.setTotalPages(fs.getLong("totalPages", -1)); - try{ - while(true){ // Keep going til an EOFExcepiton is thrown - TermEntry readObject = TermEntryReaderWriter.getInstance().readObject(is); - SortedSet set = newtrees.get(readObject.subj); - if(set == null) - newtrees.put(readObject.subj, set = new TreeSet()); - set.add(readObject); - terms.add(readObject.subj); - entriesAdded++; - } - }catch(EOFException e){ - // EOF, do nothing - } - } catch (IOException ex) { - java.util.logging.Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex); - } finally { - Closer.close(is); - } - return entriesAdded; - } - - /** Create a directory for an on-disk index. - * @return False if something broke and we can't continue. */ - private boolean createDiskDir() { - dirNumber++; - idxDiskDir = new File(DISK_DIR_PREFIX + Integer.toString(dirNumber)); - Logger.normal(this, "Created new disk dir for merging: "+idxDiskDir); - if(!(idxDiskDir.mkdir() || idxDiskDir.isDirectory())) { - Logger.error(this, "Unable to create new disk dir: "+idxDiskDir); - synchronized(this) { - pushBroken = true; - return false; - } - } - return true; - } - - /** Set up the serialisers for an on-disk index. - * @return False if something broke and we can't continue. */ - private boolean makeDiskDirSerialiser() { - if(srlDisk == null) { - srlDisk = ProtoIndexSerialiser.forIndex(idxDiskDir); - LiveArchiver,SimpleProgress> archiver = - (LiveArchiver,SimpleProgress>)(srlDisk.getChildSerialiser()); - leafsrlDisk = ProtoIndexComponentSerialiser.get(ProtoIndexComponentSerialiser.FMT_FILE_LOCAL, archiver); - if(lastDiskIndexName == null) { - try { - idxDisk = new ProtoIndex(new FreenetURI("CHK@"), "test", null, null, 0L); - } catch (java.net.MalformedURLException e) { - throw new AssertionError(e); - } - // FIXME more hacks: It's essential that we use the same FileArchiver instance here. - leafsrlDisk.setSerialiserFor(idxDisk); - } else { - try { - PullTask pull = new PullTask(lastDiskIndexName); - Logger.debug(this, "Pulling previous index "+lastDiskIndexName+" from disk so can update it."); - srlDisk.pull(pull); - Logger.debug(this, "Pulled previous index "+lastDiskIndexName+" from disk - updating..."); - idxDisk = pull.data; - if(idxDisk.getSerialiser().getLeafSerialiser() != archiver) - throw new IllegalStateException("Different serialiser: "+idxFreenet.getSerialiser()+" should be "+leafsrl); - } catch (TaskAbortException e) { - Logger.error(this, "Failed to download previous index for spider update: "+e, e); - e.printStackTrace(); - synchronized(freenetMergeSync) { - pushBroken = true; - } - return false; - } - } - } - return true; - } - +//<<<<<<< HEAD +// /** Merge from the Bucket chain to the on-disk idxDisk. */ +// protected void wrapMergeToDisk() { +// spiderIndexURIs.loadSSKURIs(); +// boolean first = true; +// while(true) { +// final Bucket data; +// synchronized(freenetMergeSync) { +// if(pushBroken) { +// Logger.error(this, "Pushing broken"); +// return; +// } +// if(first && diskMergeRunning) { +// Logger.error(this, "Already running a handler!"); +// return; +// } else if((!first) && (!diskMergeRunning)) { +// Logger.error(this, "Already running yet runningHandler is false?!"); +// return; +// } +// first = false; +// if(toMergeToDisk.size() == 0) { +// if(logMINOR) Logger.minor(this, "Nothing to handle"); +// diskMergeRunning = false; +// freenetMergeSync.notifyAll(); +// return; +// } +// data = toMergeToDisk.remove(0); +// freenetMergeSync.notifyAll(); +// diskMergeRunning = true; +// } +// try { +// mergeToDisk(data); +// } catch (Throwable t) { +// // Failed. +// synchronized(freenetMergeSync) { +// diskMergeRunning = false; +// pushBroken = true; +// freenetMergeSync.notifyAll(); +// } +// if(t instanceof RuntimeException) +// throw (RuntimeException)t; +// if(t instanceof Error) +// throw (Error)t; +// } +// } +// } +// +// // This is a member variable because it is huge, and having huge stuff in local variables seems to upset the default garbage collector. +// // It doesn't need to be synchronized because it's always used from mergeToDisk, which never runs in parallel. +// private Map> newtrees; +// // Ditto +// private SortedSet terms; +// +// ProtoIndexSerialiser srlDisk = null; +// private ProtoIndexComponentSerialiser leafsrlDisk; +// +// private long lastMergedToFreenet = -1; +// +// /** Merge a bucket of TermEntry's into an on-disk index. */ +// private void mergeToDisk(Bucket data) { +// +// boolean newIndex = false; +// +// if(idxDiskDir == null) { +// newIndex = true; +// if(!createDiskDir()) return; +// } +// +// if(!makeDiskDirSerialiser()) return; +// +// // Read data into newtrees and trees. +// long entriesAdded = readTermsFrom(data); +// +// if(terms.size() == 0) { +// Logger.debug(this, "Nothing to merge"); +// synchronized(this) { +// newtrees = null; +// terms = null; +// } +// return; +// } +// +// // Merge the new data to the disk index. +// +// try { +// final MutableBoolean maxDiskEntrySizeExceeded = new MutableBoolean(); +// maxDiskEntrySizeExceeded.value = false; +// long mergeStartTime = System.currentTimeMillis(); +// if(newIndex) { +// if(createDiskIndex()) +// maxDiskEntrySizeExceeded.value = true; +// } else { +// // async merge +// Closure>, TaskAbortException> clo = +// createMergeFromNewtreesClosure(maxDiskEntrySizeExceeded); +// assert(idxDisk.ttab.isBare()); +// Logger.debug(this, "Merging "+terms.size()+" terms, tree.size = "+idxDisk.ttab.size()+" from "+data+"..."); +// idxDisk.ttab.update(terms, null, clo, new TaskAbortExceptionConvertor()); +// +// } +// // Synchronize anyway so garbage collector knows about it. +// synchronized(this) { +// newtrees = null; +// terms = null; +// } +// assert(idxDisk.ttab.isBare()); +// PushTask task4 = new PushTask(idxDisk); +// srlDisk.push(task4); +// +// long mergeEndTime = System.currentTimeMillis(); +// Logger.debug(this, entriesAdded + " entries merged to disk in " + (mergeEndTime-mergeStartTime) + " ms, root at " + task4.meta + ", "); +// // FileArchiver produces a String, which is a filename not including the prefix or suffix. +// String uri = (String)task4.meta; +// lastDiskIndexName = uri; +// Logger.normal(this, "Pushed new index to file "+uri); +// if(writeStringTo(new File(LAST_DISK_FILENAME), uri) && +// writeStringTo(new File(idxDiskDir, LAST_DISK_FILENAME), uri)) { +// // Successfully uploaded and written new status. Can delete the incoming data. +// data.free(); +// } +// +// maybeMergeToFreenet(maxDiskEntrySizeExceeded); +// } catch (TaskAbortException e) { +// Logger.error(this, "Failed to upload index for spider: ", e); +// e.printStackTrace(); +// synchronized(freenetMergeSync) { +// pushBroken = true; +// } +// } +// } +// +// /** We have just written a Bucket of new data to an on-disk index. We may or may not want to +// * upload to an on-Freenet index, depending on how big the data is etc. If we do, we will need +// * to create a new on-disk index. +// * @param maxDiskEntrySizeExceeded A flag object which is set (off-thread) if any single term +// * in the index is very large. +// */ +// private void maybeMergeToFreenet(MutableBoolean maxDiskEntrySizeExceeded) { +// // Maybe chain to mergeToFreenet ??? +// +// boolean termTooBig = false; +// synchronized(maxDiskEntrySizeExceeded) { +// termTooBig = maxDiskEntrySizeExceeded.value; +// } +// +// mergedToDisk++; +// if((lastMergedToFreenet > 0 && idxDisk.ttab.size() > MAX_TERMS) || +// (idxDisk.ttab.size() > MAX_TERMS_NOT_UPLOADED) +// || (mergedToDisk > MAX_UPDATES) || termTooBig || +// (lastMergedToFreenet > 0 && (System.currentTimeMillis() - lastMergedToFreenet) > MAX_TIME)) { +// +// final ProtoIndex diskToMerge = idxDisk; +// final File dir = idxDiskDir; +// Logger.debug(this, "Exceeded threshold, starting new disk index and starting merge from disk to Freenet..."); +// mergedToDisk = 0; +// lastMergedToFreenet = -1; +// idxDisk = null; +// srlDisk = null; +// leafsrlDisk = null; +// idxDiskDir = null; +// lastDiskIndexName = null; +// +// synchronized(freenetMergeSync) { +// while(freenetMergeRunning) { +// if(pushBroken) return; +// Logger.normal(this, "Need to merge to Freenet, but last merge not finished yet. Waiting..."); +// try { +// freenetMergeSync.wait(); +// } catch (InterruptedException e) { +// // Ignore +// } +// } +// if(pushBroken) return; +// freenetMergeRunning = true; +// } +// +// Runnable r = new Runnable() { +// +// public void run() { +// try { +// mergeToFreenet(diskToMerge, dir); +// } catch (Throwable t) { +// Logger.error(this, "Merge to Freenet failed: ", t); +// t.printStackTrace(); +// synchronized(freenetMergeSync) { +// pushBroken = true; +// } +// } finally { +// synchronized(freenetMergeSync) { +// freenetMergeRunning = false; +// if(!pushBroken) +// lastMergedToFreenet = System.currentTimeMillis(); +// freenetMergeSync.notifyAll(); +// } +// } +// } +// +// }; +// pr.getNode().executor.execute(r, "Library: Merge data from disk to Freenet"); +// } else { +// Logger.debug(this, "Not merging to Freenet yet: "+idxDisk.ttab.size()+" terms in index, "+mergedToDisk+" merges, "+(lastMergedToFreenet <= 0 ? "never merged to Freenet" : ("last merged to Freenet "+TimeUtil.formatTime(System.currentTimeMillis() - lastMergedToFreenet))+"ago")); +// } +// } +// +// private boolean writeURITo(File filename, FreenetURI uri) { +// return writeStringTo(filename, uri.toString()); +// } +// +// private boolean writeStringTo(File filename, String uri) { +// FileOutputStream fos = null; +// try { +// fos = new FileOutputStream(filename); +// OutputStreamWriter osw = new OutputStreamWriter(fos, "UTF-8"); +// osw.write(uri.toString()); +// osw.close(); +// fos = null; +// return true; +// } catch (IOException e) { +// Logger.error(this, "Failed to write to "+filename+" : "+uri, e); +// return false; +// } finally { +// Closer.close(fos); +// } +// } +// +// private String readStringFrom(File file) { +// String ret; +// FileInputStream fis = null; +// try { +// fis = new FileInputStream(file); +// BufferedReader br = new BufferedReader(new InputStreamReader(fis, "UTF-8")); +// ret = br.readLine(); +// fis.close(); +// fis = null; +// return ret; +// } catch (IOException e) { +// // Ignore +// return null; +// } finally { +// Closer.close(fis); +// } +// } +// +// private FreenetURI readURIFrom(File file) { +// String s = readStringFrom(file); +// if(s != null) { +// try { +// return new FreenetURI(s); +// } catch (MalformedURLException e) { +// // Ignore. +// } +// } +// return null; +// } +// +// /** Create a callback object which will do the merging of individual terms. This will be called +// * for each term as it is unpacked from the existing on-disk index. It then merges in new data +// * from newtrees and writes the subtree for the term back to disk. Most of the work is done in +// * update() below. +// * @param maxDiskEntrySizeExceeded Will be set if any single term is so large that we need to +// * upload to Freenet immediately. */ +// private Closure>, TaskAbortException> createMergeFromNewtreesClosure(final MutableBoolean maxDiskEntrySizeExceeded) { +// return new +// Closure>, TaskAbortException>() { +// /*@Override**/ public void invoke(Map.Entry> entry) throws TaskAbortException { +// String key = entry.getKey(); +// SkeletonBTreeSet tree = entry.getValue(); +// Logger.minor(this, "Processing: " + key + " : " + (tree != null ? tree : "new")); +// if(tree != null) +// Logger.debug(this, "Merging data (on disk) in term "+key); +// else +// Logger.debug(this, "Adding new term to disk index: "+key); +// //System.out.println("handling " + key + ((tree == null)? " (new)":" (old)")); +// if (tree == null) { +// entry.setValue(tree = makeEntryTree(leafsrlDisk)); +// } +// assert(tree.isBare()); +// SortedSet toMerge = newtrees.get(key); +// tree.update(toMerge, null); +// if(toMerge.size() > MAX_DISK_ENTRY_SIZE) +// synchronized(maxDiskEntrySizeExceeded) { +// maxDiskEntrySizeExceeded.value = true; +// } +// toMerge = null; +// newtrees.remove(key); +// assert(tree.isBare()); +// if(logMINOR) Logger.minor(this, "Updated: "+key+" : "+tree); +// //System.out.println("handled " + key); +// } +// }; +// } +// +// /** Create a new on-disk index from terms and newtrees. +// * @return True if the size of any one item in the index is so large that we must upload +// * immediately to Freenet. +// * @throws TaskAbortException If something broke catastrophically. */ +// private boolean createDiskIndex() throws TaskAbortException { +// boolean tooBig = false; +// // created a new index, fill it with data. +// // DON'T MERGE, merge with a lot of data will deadlock. +// // FIXME throw in update() if it will deadlock. +// for(String key : terms) { +// SkeletonBTreeSet tree = makeEntryTree(leafsrlDisk); +// SortedSet toMerge = newtrees.get(key); +// tree.addAll(toMerge); +// if(toMerge.size() > MAX_DISK_ENTRY_SIZE) +// tooBig = true; +// toMerge = null; +// tree.deflate(); +// assert(tree.isBare()); +// idxDisk.ttab.put(key, tree); +// } +// idxDisk.ttab.deflate(); +// return tooBig; +// } +// +// /** Read the TermEntry's from the Bucket into newtrees and terms, and set up the index +// * properties. +// * @param data The Bucket containing TermPageEntry's etc serialised with TermEntryReaderWriter. +// */ +// private long readTermsFrom(Bucket data) { +// FileWriter w = null; +// newtrees = new HashMap>(); +// terms = new TreeSet(); +// int entriesAdded = 0; +// InputStream is = null; +// try { +// Logger.normal(this, "Bucket of buffer received, "+data.size()+" bytes"); +// is = data.getInputStream(); +// SimpleFieldSet fs = new SimpleFieldSet(new LineReadingInputStream(is), 1024, 512, true, true, true); +// idxDisk.setName(fs.get("index.title")); +// idxDisk.setOwnerEmail(fs.get("index.owner.email")); +// idxDisk.setOwner(fs.get("index.owner.name")); +// idxDisk.setTotalPages(fs.getLong("totalPages", -1)); +// try{ +// while(true){ // Keep going til an EOFExcepiton is thrown +// TermEntry readObject = TermEntryReaderWriter.getInstance().readObject(is); +// SortedSet set = newtrees.get(readObject.subj); +// if(set == null) +// newtrees.put(readObject.subj, set = new TreeSet()); +// set.add(readObject); +// terms.add(readObject.subj); +// entriesAdded++; +// } +// }catch(EOFException e){ +// // EOF, do nothing +// } +// } catch (IOException ex) { +// java.util.logging.Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex); +// } finally { +// Closer.close(is); +// } +// return entriesAdded; +// } +// +// /** Create a directory for an on-disk index. +// * @return False if something broke and we can't continue. */ +// private boolean createDiskDir() { +// dirNumber++; +// idxDiskDir = new File(DISK_DIR_PREFIX + Integer.toString(dirNumber)); +// Logger.normal(this, "Created new disk dir for merging: "+idxDiskDir); +// if(!(idxDiskDir.mkdir() || idxDiskDir.isDirectory())) { +// Logger.error(this, "Unable to create new disk dir: "+idxDiskDir); +// synchronized(this) { +// pushBroken = true; +// return false; +// } +// } +// return true; +// } +// +// /** Set up the serialisers for an on-disk index. +// * @return False if something broke and we can't continue. */ +// private boolean makeDiskDirSerialiser() { +// if(srlDisk == null) { +// srlDisk = ProtoIndexSerialiser.forIndex(idxDiskDir); +// LiveArchiver,SimpleProgress> archiver = +// (LiveArchiver,SimpleProgress>)(srlDisk.getChildSerialiser()); +// leafsrlDisk = ProtoIndexComponentSerialiser.get(ProtoIndexComponentSerialiser.FMT_FILE_LOCAL, archiver); +// if(lastDiskIndexName == null) { +// try { +// idxDisk = new ProtoIndex(new FreenetURI("CHK@"), "test", null, null, 0L); +// } catch (java.net.MalformedURLException e) { +// throw new AssertionError(e); +// } +// // FIXME more hacks: It's essential that we use the same FileArchiver instance here. +// leafsrlDisk.setSerialiserFor(idxDisk); +// } else { +// try { +// PullTask pull = new PullTask(lastDiskIndexName); +// Logger.debug(this, "Pulling previous index "+lastDiskIndexName+" from disk so can update it."); +// srlDisk.pull(pull); +// Logger.debug(this, "Pulled previous index "+lastDiskIndexName+" from disk - updating..."); +// idxDisk = pull.data; +// if(idxDisk.getSerialiser().getLeafSerialiser() != archiver) +// throw new IllegalStateException("Different serialiser: "+idxFreenet.getSerialiser()+" should be "+leafsrl); +// } catch (TaskAbortException e) { +// Logger.error(this, "Failed to download previous index for spider update: "+e, e); +// e.printStackTrace(); +// synchronized(freenetMergeSync) { +// pushBroken = true; +// } +// return false; +// } +// } +// } +// return true; +// } +// +// static final String INDEX_DOCNAME = "index.yml"; +// +// private ProtoIndexComponentSerialiser leafsrl; +// +// /** Merge a disk dir to an on-Freenet index. Usually called on startup, i.e. we haven't just +// * created the on-disk index so we need to setup the ProtoIndex etc. */ +// protected void mergeToFreenet(File diskDir) { +// ProtoIndexSerialiser s = ProtoIndexSerialiser.forIndex(diskDir); +// LiveArchiver,SimpleProgress> archiver = +// (LiveArchiver,SimpleProgress>)(s.getChildSerialiser()); +// ProtoIndexComponentSerialiser leaf = ProtoIndexComponentSerialiser.get(ProtoIndexComponentSerialiser.FMT_FILE_LOCAL, archiver); +// String f = this.readStringFrom(new File(diskDir, LAST_DISK_FILENAME)); +// if(f == null) { +// if(diskDir.list().length == 0) { +// Logger.debug(this, "Directory "+diskDir+" is empty. Nothing to merge."); +// diskDir.delete(); +// return; +// } +// // Ignore +// Logger.error(this, "Unable to merge old data "+diskDir); +// return; +// } else { +// Logger.debug(this, "Continuing old bucket: "+f); +// } +// +// ProtoIndex idxDisk = null; +// try { +// PullTask pull = new PullTask(f); +// Logger.debug(this, "Pulling previous index "+f+" from disk so can update it."); +// s.pull(pull); +// Logger.debug(this, "Pulled previous index "+f+" from disk - updating..."); +// idxDisk = pull.data; +// if(idxDisk.getSerialiser().getLeafSerialiser() != archiver) +// throw new IllegalStateException("Different serialiser: "+idxDisk.getSerialiser()+" should be "+archiver); +// } catch (TaskAbortException e) { +// Logger.error(this, "Failed to download previous index for spider update: "+e, e); +// e.printStackTrace(); +// synchronized(freenetMergeSync) { +// pushBroken = true; +// } +// return; +// } +// mergeToFreenet(idxDisk, diskDir); +// } +// +// private final Object inflateSync = new Object(); +// +// /** Merge from an on-disk index to an on-Freenet index. +// * @param diskToMerge The on-disk index. +// * @param diskDir The folder the on-disk index is stored in. +// */ +// protected void mergeToFreenet(ProtoIndex diskToMerge, File diskDir) { +// Logger.debug(this, "Merging on-disk index to Freenet: "+diskDir); +// if(lastUploadURI == null) { +// lastUploadURI = readURIFrom(new File(LAST_URL_FILENAME)); +// } +// setupFreenetCacheDir(); +// +// makeFreenetSerialisers(); +// +// updateOverallMetadata(diskToMerge); +// +// final SkeletonBTreeMap> newtrees = diskToMerge.ttab; +// +// // Do the upload +// +// // async merge +// Closure>, TaskAbortException> clo = +// createMergeFromTreeClosure(newtrees); +// try { +// long mergeStartTime = System.currentTimeMillis(); +// assert(idxFreenet.ttab.isBare()); +// Iterator it = +// diskToMerge.ttab.keySetAutoDeflate().iterator(); +// TreeSet terms = new TreeSet(); +// while(it.hasNext()) terms.add(it.next()); +// Logger.debug(this, "Merging "+terms.size()+" terms from disk to Freenet..."); +// assert(terms.size() == diskToMerge.ttab.size()); +// assert(idxFreenet.ttab.isBare()); +// assert(diskToMerge.ttab.isBare()); +// long entriesAdded = terms.size(); +// // Run the actual merge. +// idxFreenet.ttab.update(terms, null, clo, new TaskAbortExceptionConvertor()); +// assert(idxFreenet.ttab.isBare()); +// // Deflate the main tree. +// newtrees.deflate(); +// assert(diskToMerge.ttab.isBare()); +// +// // Push the top node to a CHK. +// PushTask task4 = new PushTask(idxFreenet); +// task4.meta = FreenetURI.EMPTY_CHK_URI; +// srl.push(task4); +// +// // Now wait for the inserts to finish. They are started asynchronously in the above merge. +// FreenetArchiver> arch = +// (FreenetArchiver>) srl.getChildSerialiser(); +// arch.waitForAsyncInserts(); +// +// long mergeEndTime = System.currentTimeMillis(); +// Logger.debug(this, entriesAdded + " entries merged in " + (mergeEndTime-mergeStartTime) + " ms, root at " + task4.meta + ", "); +// FreenetURI uri = (FreenetURI)task4.meta; +// lastUploadURI = uri; +// Logger.debug(this, "Uploaded new index to "+uri); +// if(writeURITo(new File(LAST_URL_FILENAME), uri)) { +// newtrees.deflate(); +// diskToMerge = null; +// terms = null; +// Logger.debug(this, "Finished with disk index "+diskDir); +// FileUtil.removeAll(diskDir); +// } +// +// // Create the USK to redirect to the CHK at the top of the index. +// uploadUSKForFreenetIndex(uri); +// +// } catch (TaskAbortException e) { +// Logger.error(this, "Failed to upload index for spider: "+e, e); +// e.printStackTrace(); +// synchronized(freenetMergeSync) { +// pushBroken = true; +// } +// } +// } +// +// private void uploadUSKForFreenetIndex(FreenetURI uri) { +// FreenetURI privUSK = spiderIndexURIs.getPrivateUSK(); +// try { +// FreenetURI tmp = pr.getHLSimpleClient().insertRedirect(privUSK, uri); +// long ed; +// synchronized(freenetMergeSync) { +// ed = spiderIndexURIs.setEdition(tmp.getEdition()+1); +// } +// Logger.debug(this, "Uploaded index as USK to "+tmp); +// +// writeStringTo(new File(EDITION_FILENAME), Long.toString(ed)); +// +// } catch (InsertException e) { +// e.printStackTrace(); +// Logger.error(this, "Failed to upload USK for index update", e); +// } +// } +// +// /** Create a Closure which will merge the subtrees from one index (on disk) into the subtrees +// * of another index (on Freenet). It will be called with each subtree from the on-Freenet +// * index, and will merge data from the relevant on-disk subtree. Both subtrees are initially +// * deflated, and should be deflated when we leave the method, to avoid running out of memory. +// * @param newtrees The on-disk tree of trees to get data from. +// * @return +// */ +// private Closure>, TaskAbortException> createMergeFromTreeClosure(final SkeletonBTreeMap> newtrees) { +// return new +// Closure>, TaskAbortException>() { +// /*@Override**/ public void invoke(Map.Entry> entry) throws TaskAbortException { +// String key = entry.getKey(); +// SkeletonBTreeSet tree = entry.getValue(); +// if(logMINOR) Logger.minor(this, "Processing: "+key+" : "+tree); +// //System.out.println("handling " + key + ((tree == null)? " (new)":" (old)")); +// boolean newTree = false; +// if (tree == null) { +// entry.setValue(tree = makeEntryTree(leafsrl)); +// newTree = true; +// } +// assert(tree.isBare()); +// SortedSet data; +// // Can't be run in parallel. +// synchronized(inflateSync) { +// newtrees.inflate(key, true); +// SkeletonBTreeSet entries; +// entries = newtrees.get(key); +// // CONCURRENCY: Because the lower-level trees are packed by the top tree, the bottom +// // trees (SkeletonBTreeSet's) are not independant of each other. When the newtrees +// // inflate above runs, it can deflate a tree that is still in use by another instance +// // of this callback. Therefore we must COPY IT AND DEFLATE IT INSIDE THE LOCK. +// entries.inflate(); +// data = new TreeSet(entries); +// entries.deflate(); +// assert(entries.isBare()); +// } +// if(tree != null) +// +// if(newTree) { +// tree.addAll(data); +// assert(tree.size() == data.size()); +// Logger.debug(this, "Added data to Freenet for term "+key+" : "+data.size()); +// } else { +// int oldSize = tree.size(); +// tree.update(data, null); +// // Note that it is possible for data.size() + oldSize != tree.size(), because we might be merging data we've already merged. +// // But most of the time it will add up. +// Logger.debug(this, "Merged data to Freenet in term "+key+" : "+data.size()+" + "+oldSize+" -> "+tree.size()); +// } +// tree.deflate(); +// assert(tree.isBare()); +// if(logMINOR) Logger.minor(this, "Updated: "+key+" : "+tree); +// } +// }; +// } +// +// /** Update the overall metadata for the on-Freenet index from the on-disk index. */ +// private void updateOverallMetadata(ProtoIndex diskToMerge) { +// idxFreenet.setName(diskToMerge.getName()); +// idxFreenet.setOwnerEmail(diskToMerge.getOwnerEmail()); +// idxFreenet.setOwner(diskToMerge.getOwner()); +// // This is roughly accurate, it might not be exactly so if we process a bit out of order. +// idxFreenet.setTotalPages(diskToMerge.getTotalPages() + Math.max(0,idxFreenet.getTotalPages())); +// } +// +// /** Setup the serialisers for uploading to Freenet. These convert tree nodes to and from blocks +// * on Freenet, essentially. */ +// private void makeFreenetSerialisers() { +// if(srl == null) { +// srl = ProtoIndexSerialiser.forIndex(lastUploadURI, RequestStarter.BULK_SPLITFILE_PRIORITY_CLASS); +// LiveArchiver,SimpleProgress> archiver = +// (LiveArchiver,SimpleProgress>)(srl.getChildSerialiser()); +// leafsrl = ProtoIndexComponentSerialiser.get(ProtoIndexComponentSerialiser.FMT_DEFAULT, archiver); +// if(lastUploadURI == null) { +// try { +// idxFreenet = new ProtoIndex(new FreenetURI("CHK@"), "test", null, null, 0L); +// } catch (java.net.MalformedURLException e) { +// throw new AssertionError(e); +// } +// // FIXME more hacks: It's essential that we use the same FreenetArchiver instance here. +// leafsrl.setSerialiserFor(idxFreenet); +// } else { +// try { +// PullTask pull = new PullTask(lastUploadURI); +// Logger.debug(this, "Pulling previous index "+lastUploadURI+" so can update it."); +// srl.pull(pull); +// Logger.debug(this, "Pulled previous index "+lastUploadURI+" - updating..."); +// idxFreenet = pull.data; +// if(idxFreenet.getSerialiser().getLeafSerialiser() != archiver) +// throw new IllegalStateException("Different serialiser: "+idxFreenet.getSerialiser()+" should be "+leafsrl); +// } catch (TaskAbortException e) { +// Logger.error(this, "Failed to download previous index for spider update: "+e, e); +// e.printStackTrace(); +// synchronized(freenetMergeSync) { +// pushBroken = true; +// } +// return; +// } +// } +// } +// } +// +// /** Set up the on-disk cache, which keeps a copy of everything we upload to Freenet, so we +// * won't need to re-download it, which can be very slow and doesn't always succeed. */ +// private void setupFreenetCacheDir() { +// if(FreenetArchiver.getCacheDir() == null) { +// File dir = new File("library-spider-pushed-data-cache"); +// dir.mkdir(); +// FreenetArchiver.setCacheDir(dir); +// } +// } +// +// protected static SkeletonBTreeSet makeEntryTree(ProtoIndexComponentSerialiser leafsrl) { +// SkeletonBTreeSet tree = new SkeletonBTreeSet(ProtoIndex.BTREE_NODE_MIN); +// leafsrl.setSerialiserFor(tree); +// return tree; +// } +// +// public void start() { +// final String[] oldToMerge; +// synchronized(freenetMergeSync) { +// oldToMerge = new File(".").list(new FilenameFilter() { +// +// public boolean accept(File arg0, String arg1) { +// if(!(arg1.toLowerCase().startsWith(BASE_FILENAME_PUSH_DATA))) return false; +// File f = new File(arg0, arg1); +// if(!f.isFile()) return false; +// if(f.length() == 0) { f.delete(); return false; } +// String s = f.getName().substring(BASE_FILENAME_PUSH_DATA.length()); +// pushNumber = Math.max(pushNumber, Long.parseLong(s)+1); +// return true; +// } +// +// }); +// } +// final String[] dirsToMerge; +// synchronized(freenetMergeSync) { +// dirsToMerge = new File(".").list(new FilenameFilter() { +// +// public boolean accept(File arg0, String arg1) { +// if(!(arg1.toLowerCase().startsWith(DISK_DIR_PREFIX))) return false; +// File f = new File(arg0, arg1); +// String s = f.getName().substring(DISK_DIR_PREFIX.length()); +// dirNumber = Math.max(dirNumber, Integer.parseInt(s)+1); +// return true; +// } +// +// }); +// } +// if(oldToMerge != null && oldToMerge.length > 0) { +// Logger.debug(this, "Found "+oldToMerge.length+" buckets of old index data to merge..."); +// Runnable r = new Runnable() { +// +// public void run() { +// synchronized(freenetMergeSync) { +// for(String filename : oldToMerge) { +// File f = new File(filename); +// toMergeToDisk.add(new FileBucket(f, true, false, false, true)); +// } +// } +// wrapMergeToDisk(); +// } +// +// }; +// pr.getNode().executor.execute(r, "Library: handle index data from previous run"); +// } +// if(dirsToMerge != null && dirsToMerge.length > 0) { +// Logger.debug(this, "Found "+dirsToMerge.length+" disk trees of old index data to merge..."); +// Runnable r = new Runnable() { +// +// public void run() { +// synchronized(freenetMergeSync) { +// while(freenetMergeRunning) { +// if(pushBroken) return; +// Logger.normal(this, "Need to merge to Freenet, but last merge not finished yet. Waiting..."); +// try { +// freenetMergeSync.wait(); +// } catch (InterruptedException e) { +// // Ignore +// } +// } +// if(pushBroken) return; +// freenetMergeRunning = true; +// } +// try { +// for(String filename : dirsToMerge) { +// File f = new File(filename); +// mergeToFreenet(f); +// } +// } finally { +// synchronized(freenetMergeSync) { +// freenetMergeRunning = false; +// if(!pushBroken) +// lastMergedToFreenet = System.currentTimeMillis(); +// freenetMergeSync.notifyAll(); +// } +// } +// +// } +// +// }; +// pr.getNode().executor.execute(r, "Library: handle trees from previous run"); +// } +//======= static final String INDEX_DOCNAME = "index.yml"; - - private ProtoIndexComponentSerialiser leafsrl; - - /** Merge a disk dir to an on-Freenet index. Usually called on startup, i.e. we haven't just - * created the on-disk index so we need to setup the ProtoIndex etc. */ - protected void mergeToFreenet(File diskDir) { - ProtoIndexSerialiser s = ProtoIndexSerialiser.forIndex(diskDir); - LiveArchiver,SimpleProgress> archiver = - (LiveArchiver,SimpleProgress>)(s.getChildSerialiser()); - ProtoIndexComponentSerialiser leaf = ProtoIndexComponentSerialiser.get(ProtoIndexComponentSerialiser.FMT_FILE_LOCAL, archiver); - String f = this.readStringFrom(new File(diskDir, LAST_DISK_FILENAME)); - if(f == null) { - if(diskDir.list().length == 0) { - Logger.debug(this, "Directory "+diskDir+" is empty. Nothing to merge."); - diskDir.delete(); - return; - } - // Ignore - Logger.error(this, "Unable to merge old data "+diskDir); - return; - } else { - Logger.debug(this, "Continuing old bucket: "+f); - } - - ProtoIndex idxDisk = null; - try { - PullTask pull = new PullTask(f); - Logger.debug(this, "Pulling previous index "+f+" from disk so can update it."); - s.pull(pull); - Logger.debug(this, "Pulled previous index "+f+" from disk - updating..."); - idxDisk = pull.data; - if(idxDisk.getSerialiser().getLeafSerialiser() != archiver) - throw new IllegalStateException("Different serialiser: "+idxDisk.getSerialiser()+" should be "+archiver); - } catch (TaskAbortException e) { - Logger.error(this, "Failed to download previous index for spider update: "+e, e); - e.printStackTrace(); - synchronized(freenetMergeSync) { - pushBroken = true; - } - return; - } - mergeToFreenet(idxDisk, diskDir); - } - - private final Object inflateSync = new Object(); - - /** Merge from an on-disk index to an on-Freenet index. - * @param diskToMerge The on-disk index. - * @param diskDir The folder the on-disk index is stored in. - */ - protected void mergeToFreenet(ProtoIndex diskToMerge, File diskDir) { - Logger.debug(this, "Merging on-disk index to Freenet: "+diskDir); - if(lastUploadURI == null) { - lastUploadURI = readURIFrom(new File(LAST_URL_FILENAME)); - } - setupFreenetCacheDir(); - - makeFreenetSerialisers(); - - updateOverallMetadata(diskToMerge); - - final SkeletonBTreeMap> newtrees = diskToMerge.ttab; - - // Do the upload - - // async merge - Closure>, TaskAbortException> clo = - createMergeFromTreeClosure(newtrees); - try { - long mergeStartTime = System.currentTimeMillis(); - assert(idxFreenet.ttab.isBare()); - Iterator it = - diskToMerge.ttab.keySetAutoDeflate().iterator(); - TreeSet terms = new TreeSet(); - while(it.hasNext()) terms.add(it.next()); - Logger.debug(this, "Merging "+terms.size()+" terms from disk to Freenet..."); - assert(terms.size() == diskToMerge.ttab.size()); - assert(idxFreenet.ttab.isBare()); - assert(diskToMerge.ttab.isBare()); - long entriesAdded = terms.size(); - // Run the actual merge. - idxFreenet.ttab.update(terms, null, clo, new TaskAbortExceptionConvertor()); - assert(idxFreenet.ttab.isBare()); - // Deflate the main tree. - newtrees.deflate(); - assert(diskToMerge.ttab.isBare()); - - // Push the top node to a CHK. - PushTask task4 = new PushTask(idxFreenet); - task4.meta = FreenetURI.EMPTY_CHK_URI; - srl.push(task4); - - // Now wait for the inserts to finish. They are started asynchronously in the above merge. - FreenetArchiver> arch = - (FreenetArchiver>) srl.getChildSerialiser(); - arch.waitForAsyncInserts(); - - long mergeEndTime = System.currentTimeMillis(); - Logger.debug(this, entriesAdded + " entries merged in " + (mergeEndTime-mergeStartTime) + " ms, root at " + task4.meta + ", "); - FreenetURI uri = (FreenetURI)task4.meta; - lastUploadURI = uri; - Logger.debug(this, "Uploaded new index to "+uri); - if(writeURITo(new File(LAST_URL_FILENAME), uri)) { - newtrees.deflate(); - diskToMerge = null; - terms = null; - Logger.debug(this, "Finished with disk index "+diskDir); - FileUtil.removeAll(diskDir); - } - - // Create the USK to redirect to the CHK at the top of the index. - uploadUSKForFreenetIndex(uri); - - } catch (TaskAbortException e) { - Logger.error(this, "Failed to upload index for spider: "+e, e); - e.printStackTrace(); - synchronized(freenetMergeSync) { - pushBroken = true; - } - } - } - - private void uploadUSKForFreenetIndex(FreenetURI uri) { - FreenetURI privUSK = spiderIndexURIs.getPrivateUSK(); - try { - FreenetURI tmp = pr.getHLSimpleClient().insertRedirect(privUSK, uri); - long ed; - synchronized(freenetMergeSync) { - ed = spiderIndexURIs.setEdition(tmp.getEdition()+1); - } - Logger.debug(this, "Uploaded index as USK to "+tmp); - - writeStringTo(new File(EDITION_FILENAME), Long.toString(ed)); - - } catch (InsertException e) { - e.printStackTrace(); - Logger.error(this, "Failed to upload USK for index update", e); - } - } - - /** Create a Closure which will merge the subtrees from one index (on disk) into the subtrees - * of another index (on Freenet). It will be called with each subtree from the on-Freenet - * index, and will merge data from the relevant on-disk subtree. Both subtrees are initially - * deflated, and should be deflated when we leave the method, to avoid running out of memory. - * @param newtrees The on-disk tree of trees to get data from. - * @return - */ - private Closure>, TaskAbortException> createMergeFromTreeClosure(final SkeletonBTreeMap> newtrees) { - return new - Closure>, TaskAbortException>() { - /*@Override**/ public void invoke(Map.Entry> entry) throws TaskAbortException { - String key = entry.getKey(); - SkeletonBTreeSet tree = entry.getValue(); - if(logMINOR) Logger.minor(this, "Processing: "+key+" : "+tree); - //System.out.println("handling " + key + ((tree == null)? " (new)":" (old)")); - boolean newTree = false; - if (tree == null) { - entry.setValue(tree = makeEntryTree(leafsrl)); - newTree = true; - } - assert(tree.isBare()); - SortedSet data; - // Can't be run in parallel. - synchronized(inflateSync) { - newtrees.inflate(key, true); - SkeletonBTreeSet entries; - entries = newtrees.get(key); - // CONCURRENCY: Because the lower-level trees are packed by the top tree, the bottom - // trees (SkeletonBTreeSet's) are not independant of each other. When the newtrees - // inflate above runs, it can deflate a tree that is still in use by another instance - // of this callback. Therefore we must COPY IT AND DEFLATE IT INSIDE THE LOCK. - entries.inflate(); - data = new TreeSet(entries); - entries.deflate(); - assert(entries.isBare()); - } - if(tree != null) - - if(newTree) { - tree.addAll(data); - assert(tree.size() == data.size()); - Logger.debug(this, "Added data to Freenet for term "+key+" : "+data.size()); - } else { - int oldSize = tree.size(); - tree.update(data, null); - // Note that it is possible for data.size() + oldSize != tree.size(), because we might be merging data we've already merged. - // But most of the time it will add up. - Logger.debug(this, "Merged data to Freenet in term "+key+" : "+data.size()+" + "+oldSize+" -> "+tree.size()); - } - tree.deflate(); - assert(tree.isBare()); - if(logMINOR) Logger.minor(this, "Updated: "+key+" : "+tree); - } - }; - } - - /** Update the overall metadata for the on-Freenet index from the on-disk index. */ - private void updateOverallMetadata(ProtoIndex diskToMerge) { - idxFreenet.setName(diskToMerge.getName()); - idxFreenet.setOwnerEmail(diskToMerge.getOwnerEmail()); - idxFreenet.setOwner(diskToMerge.getOwner()); - // This is roughly accurate, it might not be exactly so if we process a bit out of order. - idxFreenet.setTotalPages(diskToMerge.getTotalPages() + Math.max(0,idxFreenet.getTotalPages())); - } - - /** Setup the serialisers for uploading to Freenet. These convert tree nodes to and from blocks - * on Freenet, essentially. */ - private void makeFreenetSerialisers() { - if(srl == null) { - srl = ProtoIndexSerialiser.forIndex(lastUploadURI, RequestStarter.BULK_SPLITFILE_PRIORITY_CLASS); - LiveArchiver,SimpleProgress> archiver = - (LiveArchiver,SimpleProgress>)(srl.getChildSerialiser()); - leafsrl = ProtoIndexComponentSerialiser.get(ProtoIndexComponentSerialiser.FMT_DEFAULT, archiver); - if(lastUploadURI == null) { - try { - idxFreenet = new ProtoIndex(new FreenetURI("CHK@"), "test", null, null, 0L); - } catch (java.net.MalformedURLException e) { - throw new AssertionError(e); - } - // FIXME more hacks: It's essential that we use the same FreenetArchiver instance here. - leafsrl.setSerialiserFor(idxFreenet); - } else { - try { - PullTask pull = new PullTask(lastUploadURI); - Logger.debug(this, "Pulling previous index "+lastUploadURI+" so can update it."); - srl.pull(pull); - Logger.debug(this, "Pulled previous index "+lastUploadURI+" - updating..."); - idxFreenet = pull.data; - if(idxFreenet.getSerialiser().getLeafSerialiser() != archiver) - throw new IllegalStateException("Different serialiser: "+idxFreenet.getSerialiser()+" should be "+leafsrl); - } catch (TaskAbortException e) { - Logger.error(this, "Failed to download previous index for spider update: "+e, e); - e.printStackTrace(); - synchronized(freenetMergeSync) { - pushBroken = true; - } - return; - } - } - } - } - - /** Set up the on-disk cache, which keeps a copy of everything we upload to Freenet, so we - * won't need to re-download it, which can be very slow and doesn't always succeed. */ - private void setupFreenetCacheDir() { - if(FreenetArchiver.getCacheDir() == null) { - File dir = new File("library-spider-pushed-data-cache"); - dir.mkdir(); - FreenetArchiver.setCacheDir(dir); - } - } - - protected static SkeletonBTreeSet makeEntryTree(ProtoIndexComponentSerialiser leafsrl) { - SkeletonBTreeSet tree = new SkeletonBTreeSet(ProtoIndex.BTREE_NODE_MIN); - leafsrl.setSerialiserFor(tree); - return tree; - } public void start() { - final String[] oldToMerge; - synchronized(freenetMergeSync) { - oldToMerge = new File(".").list(new FilenameFilter() { - - public boolean accept(File arg0, String arg1) { - if(!(arg1.toLowerCase().startsWith(BASE_FILENAME_PUSH_DATA))) return false; - File f = new File(arg0, arg1); - if(!f.isFile()) return false; - if(f.length() == 0) { f.delete(); return false; } - String s = f.getName().substring(BASE_FILENAME_PUSH_DATA.length()); - pushNumber = Math.max(pushNumber, Long.parseLong(s)+1); - return true; - } - - }); - } - final String[] dirsToMerge; - synchronized(freenetMergeSync) { - dirsToMerge = new File(".").list(new FilenameFilter() { - - public boolean accept(File arg0, String arg1) { - if(!(arg1.toLowerCase().startsWith(DISK_DIR_PREFIX))) return false; - File f = new File(arg0, arg1); - String s = f.getName().substring(DISK_DIR_PREFIX.length()); - dirNumber = Math.max(dirNumber, Integer.parseInt(s)+1); - return true; - } - - }); - } - if(oldToMerge != null && oldToMerge.length > 0) { - Logger.debug(this, "Found "+oldToMerge.length+" buckets of old index data to merge..."); - Runnable r = new Runnable() { - - public void run() { - synchronized(freenetMergeSync) { - for(String filename : oldToMerge) { - File f = new File(filename); - toMergeToDisk.add(new FileBucket(f, true, false, false, true)); - } - } - wrapMergeToDisk(); - } - - }; - pr.getNode().executor.execute(r, "Library: handle index data from previous run"); - } - if(dirsToMerge != null && dirsToMerge.length > 0) { - Logger.debug(this, "Found "+dirsToMerge.length+" disk trees of old index data to merge..."); - Runnable r = new Runnable() { - - public void run() { - synchronized(freenetMergeSync) { - while(freenetMergeRunning) { - if(pushBroken) return; - Logger.normal(this, "Need to merge to Freenet, but last merge not finished yet. Waiting..."); - try { - freenetMergeSync.wait(); - } catch (InterruptedException e) { - // Ignore - } - } - if(pushBroken) return; - freenetMergeRunning = true; - } - try { - for(String filename : dirsToMerge) { - File f = new File(filename); - mergeToFreenet(f); - } - } finally { - synchronized(freenetMergeSync) { - freenetMergeRunning = false; - if(!pushBroken) - lastMergedToFreenet = System.currentTimeMillis(); - freenetMergeSync.notifyAll(); - } - } - - } - - }; - pr.getNode().executor.execute(r, "Library: handle trees from previous run"); - } + System.out.println("Started pass-though spider uploader."); +//>>>>>>> debbiedub/fcp-uploader } + /** + * Retrieving data from Spider + */ public void handlePushBuffer(SimpleFieldSet params, Bucket data) { - if(data.size() == 0) { Logger.error(this, "Bucket of data ("+data+") to push is empty", new Exception("error")); data.free(); return; } - // Process data off-thread, but only one load at a time. - // Hence it won't stall Spider unless we get behind. - long pn; synchronized(this) { pn = pushNumber++; @@ -901,54 +870,41 @@ public void handlePushBuffer(SimpleFieldSet params, Bucket data) { Logger.debug(this, "Written data to "+pushFile); } catch (IOException e1) { e1.printStackTrace(); - Logger.error(this, "Unable to back up push data #"+pn, e1); - output = data; } - synchronized(freenetMergeSync) { - boolean waited = false; - while(toMergeToDisk.size() > MAX_HANDLING_COUNT && !pushBroken) { - Logger.error(this, "Spider feeding us data too fast, waiting for background process to finish. Ahead of us in the queue: "+toMergeToDisk.size()); + // Stall Spider if we get behind. + int countFilesToMerge = 0; + int tooManyFilesToMerge = 0; + do { + if (tooManyFilesToMerge > 0) { + System.out.println("There are " + countFilesToMerge + " files to merge...stalling spider."); try { - waited = true; - freenetMergeSync.wait(); + Thread.sleep( + tooManyFilesToMerge * + tooManyFilesToMerge * + 100 * + 1000); } catch (InterruptedException e) { - // Ignore - } - } - toMergeToDisk.add(output); - if(pushBroken) { - if(toMergeToDisk.size() < PUSH_BROKEN_MAX_HANDLING_COUNT) - // We have written the data, it will be recovered after restart. - Logger.error(this, "Pushing is broken, failing"); - else { - // Wait forever to prevent running out of disk space. - // Spider is single threaded. - // FIXME: Use an error return or a throwable to shut down Spider. - while(true) { - try { - freenetMergeSync.wait(); - } catch (InterruptedException e) { - // Ignore - } - } + // TODO Auto-generated catch block + e.printStackTrace(); } - return; } - if(waited) - Logger.error(this, "Waited for previous handler to go away, moving on..."); - //if(freenetMergeRunning) return; // Already running, no need to restart it. - if(diskMergeRunning) return; // Already running, no need to restart it. - } - Runnable r = new Runnable() { - - public void run() { -// wrapMergeToFreenet(); - wrapMergeToDisk(); - } - - }; - pr.getNode().executor.execute(r, "Library: Handle data from Spider"); + String[] filesToMerge = new File(".").list(new FilenameFilter() { + + public boolean accept(File arg0, String arg1) { + if(!(arg1.toLowerCase().startsWith(BASE_FILENAME_PUSH_DATA))) return false; + File f = new File(arg0, arg1); + if(!f.isFile()) return false; + if(f.length() == 0) { f.delete(); return false; } + return true; + } + + }); + + countFilesToMerge = filesToMerge.length; + tooManyFilesToMerge = countFilesToMerge - 20; + } while (tooManyFilesToMerge > 0); + System.out.println("There are " + countFilesToMerge + " files to merge."); } public FreenetURI getPublicUSKURI() { @@ -966,4 +922,4 @@ public void handleGetSpiderURI(PluginReplySender replysender) { // Race condition, ignore. } } -} \ No newline at end of file +} diff --git a/src/plugins/Library/Version.java b/src/plugins/Library/Version.java index 042ee3cf..41f8c309 100644 --- a/src/plugins/Library/Version.java +++ b/src/plugins/Library/Version.java @@ -13,5 +13,4 @@ public class Version { public static String vcsRevision() { return vcsRevision; } - } diff --git a/src/plugins/Library/VirtualIndex.java b/src/plugins/Library/VirtualIndex.java deleted file mode 100644 index 9fae7c91..00000000 --- a/src/plugins/Library/VirtualIndex.java +++ /dev/null @@ -1,20 +0,0 @@ -/* This code is part of Freenet. It is distributed under the GNU General - * Public License, version 2 (or at your option any later version). See - * http://www.gnu.org/ for further details of the GPL. */ -package plugins.Library; - -import plugins.Library.index.TermEntry; -import plugins.Library.index.URIEntry; -import plugins.Library.util.exec.Execution; - -/** -** Represents a virtual index that gets its data from another plugin. -** -** @author infinity0 -*/ -public interface VirtualIndex extends Index { - - - - -} diff --git a/src/plugins/Library/WriteableIndex.java b/src/plugins/Library/WriteableIndex.java deleted file mode 100644 index 5debe962..00000000 --- a/src/plugins/Library/WriteableIndex.java +++ /dev/null @@ -1,32 +0,0 @@ -/* This code is part of Freenet. It is distributed under the GNU General - * Public License, version 2 (or at your option any later version). See - * http://www.gnu.org/ for further details of the GPL. */ -package plugins.Library; - -import plugins.Library.index.TermEntry; -import plugins.Library.index.URIEntry; -import plugins.Library.util.exec.Execution; - -import freenet.keys.FreenetURI; - -import java.util.Collection; -import java.util.Set; - -/** -** Represents a writable Index. -** -** TODO -** -** @author infinity0 -*/ -public interface WriteableIndex extends Index { - - public Execution putTermEntries(Collection entries); - - public Execution remTermEntries(Collection entries); - - public Execution putURIEntries(Collection entries); - - public Execution remURIEntries(Collection entries); - -} diff --git a/src/plugins/Library/client/FreenetArchiver.java b/src/plugins/Library/client/FreenetArchiver.java index 4401bbd6..a8c3545e 100644 --- a/src/plugins/Library/client/FreenetArchiver.java +++ b/src/plugins/Library/client/FreenetArchiver.java @@ -6,50 +6,34 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; +import java.net.MalformedURLException; import java.util.ArrayList; -import java.util.HashSet; -import plugins.Library.Library; -import plugins.Library.io.ObjectStreamReader; -import plugins.Library.io.ObjectStreamWriter; -import plugins.Library.io.serial.LiveArchiver; -import plugins.Library.util.exec.ProgressParts; -import plugins.Library.util.exec.SimpleProgress; -import plugins.Library.util.exec.TaskAbortException; - -import freenet.client.ClientMetadata; import freenet.client.FetchException; import freenet.client.FetchException.FetchExceptionMode; import freenet.client.FetchResult; import freenet.client.HighLevelSimpleClient; -import freenet.client.InsertBlock; -import freenet.client.InsertContext; import freenet.client.InsertException; -import freenet.client.async.BaseClientPutter; import freenet.client.async.ClientContext; -import freenet.client.async.ClientPutCallback; -import freenet.client.async.ClientPutter; -import freenet.client.async.PersistenceDisabledException; import freenet.client.events.ClientEvent; import freenet.client.events.ClientEventListener; import freenet.client.events.SplitfileProgressEvent; import freenet.crypt.SHA256; -import freenet.keys.CHKBlock; -import freenet.keys.FreenetURI; +import plugins.Library.io.FreenetURI; +import plugins.Library.io.ObjectStreamReader; +import plugins.Library.io.ObjectStreamWriter; +import plugins.Library.io.serial.LiveArchiver; +import plugins.Library.util.exec.ProgressParts; +import plugins.Library.util.exec.SimpleProgress; +import plugins.Library.util.exec.TaskAbortException; import freenet.node.NodeClientCore; -import freenet.node.RequestClient; import freenet.node.RequestStarter; import freenet.support.Base64; import freenet.support.Logger; import freenet.support.SimpleReadOnlyArrayBucket; -import freenet.support.SizeUtil; import freenet.support.api.Bucket; -import freenet.support.api.RandomAccessBucket; -import freenet.support.io.BucketTools; import freenet.support.io.Closer; import freenet.support.io.FileBucket; -import freenet.support.io.ResumeFailedException; /** ** Converts between a map of {@link String} to {@link Object}, and a freenet @@ -62,8 +46,7 @@ ** ** @author infinity0 */ -public class FreenetArchiver -implements LiveArchiver { +public class FreenetArchiver implements LiveArchiver { final protected NodeClientCore core; final protected ObjectStreamReader reader; @@ -90,10 +73,6 @@ public class FreenetArchiver * long time. */ static final boolean SEMI_ASYNC_PUSH = true; - private final HashSet semiAsyncPushes = new HashSet(); - private final ArrayList pushesFailed = new ArrayList(); - private long totalBytesPushing; - public static void setCacheDir(File dir) { cacheDir = dir; } @@ -118,11 +97,20 @@ public FreenetArchiver(NodeClientCore c, ObjectStreamReader r, ObjectStreamWrite default_mime = mime; expected_bytes = size; } - + + private freenet.keys.FreenetURI toFreenetURI(FreenetURI u) { + try { + return new freenet.keys.FreenetURI(u.toString()); + } catch (MalformedURLException e) { + Logger.error(this, "Failed to create URI", e); + throw new RuntimeException("Failed to complete task: ", e); + } + } + public FreenetArchiver(NodeClientCore c, S rw, String mime, int size, short priority) { this(c, rw, rw, mime, size, priority); } - + /** ** {@inheritDoc} ** @@ -146,10 +134,18 @@ public FreenetArchiver(NodeC byte[] initialMetadata; String cacheKey; - if(task.meta instanceof FreenetURI) { + if(task.meta instanceof String) { + try { + task.meta = new FreenetURI((String) task.meta); + } catch (MalformedURLException e) { + // This wasn't an URI after all. + } + } + + if (task.meta instanceof FreenetURI) { u = (FreenetURI) task.meta; initialMetadata = null; - cacheKey = u.toString(false, true); + cacheKey = u.toString(); } else { initialMetadata = (byte[]) task.meta; u = FreenetURI.EMPTY_CHK_URI; @@ -192,23 +188,24 @@ public FreenetArchiver(NodeC FetchResult res; // bookkeeping. detects bugs in the SplitfileProgressEvent handler + ProgressParts prog_old = null; + if (progress != null) { + prog_old = progress.getParts(); + } + + if(initialMetadata != null) + res = hlsc.fetchFromMetadata(new SimpleReadOnlyArrayBucket(initialMetadata)); + else + res = hlsc.fetch(toFreenetURI(u)); + + ProgressParts prog_new; if (progress != null) { - ProgressParts prog_old = progress.getParts(); - if(initialMetadata != null) - res = hlsc.fetchFromMetadata(new SimpleReadOnlyArrayBucket(initialMetadata)); - else - res = hlsc.fetch(u); - ProgressParts prog_new = progress.getParts(); + prog_new = progress.getParts(); if (prog_old.known - prog_old.done != prog_new.known - prog_new.done) { Logger.error(this, "Inconsistency when tracking split file progress (pulling): "+prog_old.known+" of "+prog_old.done+" -> "+prog_new.known+" of "+prog_new.done); System.err.println("Inconsistency when tracking split file progress (pulling): "+prog_old.known+" of "+prog_old.done+" -> "+prog_new.known+" of "+prog_new.done); } progress.addPartKnown(0, true); - } else { - if(initialMetadata != null) - res = hlsc.fetchFromMetadata(new SimpleReadOnlyArrayBucket(initialMetadata)); - else - res = hlsc.fetch(u); } tempB = res.asBucket(); @@ -219,23 +216,33 @@ public FreenetArchiver(NodeC progress.addPartDone(); } } - long endTime = System.currentTimeMillis(); - Logger.debug(this, "Fetched block for FreenetArchiver in "+(endTime-startTime)+"ms."); is = tempB.getInputStream(); task.data = (T)reader.readObject(is); is.close(); + long endTime = System.currentTimeMillis(); + System.out.println("Fetched block for FreenetArchiver in "+(endTime-startTime)+"ms."); } catch (FetchException e) { if(e.mode == FetchExceptionMode.PERMANENT_REDIRECT && e.newURI != null) { - u = e.newURI; - continue; + try { + u = new FreenetURI(e.newURI.toString()); + continue; + } catch (MalformedURLException e1) { + System.out.println("Cannot convert " + e.newURI + "."); + } } + System.out.println("FetchException:"); + e.printStackTrace(); throw new TaskAbortException("Failed to fetch content", e, true); } catch (IOException e) { + System.out.println("IOException:"); + e.printStackTrace(); throw new TaskAbortException("Failed to read content from local tempbucket", e, true); } catch (RuntimeException e) { + System.out.println("RuntimeException:"); + e.printStackTrace(); throw new TaskAbortException("Failed to complete task: ", e); } @@ -251,284 +258,284 @@ public FreenetArchiver(NodeC } } - /** - ** {@inheritDoc} - ** - ** This implementation produces metdata of type {@link FreenetURI}. - ** - ** If the input metadata is an insert URI (SSK or USK), it will be replaced - ** by its corresponding request URI. Otherwise, the data will be inserted - ** as a CHK. Note that since {@link FreenetURI} is immutable, the {@link - ** FreenetURI#suggestedEdition} of a USK is '''not''' automatically - ** incremented. - */ - /*@Override**/ public void pushLive(PushTask task, final SimpleProgress progress) throws TaskAbortException { - HighLevelSimpleClient hlsc = core.makeClient(priorityClass, false, false); - RandomAccessBucket tempB = null; OutputStream os = null; - - - try { - ClientPutter putter = null; - PushCallback cb = null; - try { - tempB = core.tempBucketFactory.makeBucket(expected_bytes, 2); - os = tempB.getOutputStream(); - writer.writeObject(task.data, os); - os.close(); os = null; - tempB.setReadOnly(); - - boolean insertAsMetadata; - FreenetURI target; - - if(task.meta instanceof FreenetURI) { - insertAsMetadata = false; - target = (FreenetURI) task.meta; - } else { - insertAsMetadata = true; - target = FreenetURI.EMPTY_CHK_URI; - } - InsertBlock ib = new InsertBlock(tempB, new ClientMetadata(default_mime), target); - - Logger.debug(this, "Inserting block for FreenetArchiver..."); - long startTime = System.currentTimeMillis(); - - // code for async insert - maybe be useful elsewhere - //ClientContext cctx = core.clientContext; - //InsertContext ictx = hlsc.getInsertContext(true); - //PutWaiter pw = new PutWaiter(); - //ClientPutter pu = hlsc.insert(ib, false, null, false, ictx, pw); - //pu.setPriorityClass(RequestStarter.INTERACTIVE_PRIORITY_CLASS, cctx, null); - //FreenetURI uri = pw.waitForCompletion(); - - // bookkeeping. detects bugs in the SplitfileProgressEvent handler - ProgressParts prog_old = null; - if(progress != null) - prog_old = progress.getParts(); - - // FIXME make retry count configgable by client metadata somehow - // unlimited for push/merge - InsertContext ctx = hlsc.getInsertContext(false); - ctx.maxInsertRetries = -1; - // Early encode is normally a security risk. - // Hopefully it isn't here. - ctx.earlyEncode = true; - - String cacheKey = null; - -// if(!SEMI_ASYNC_PUSH) { -// // Actually report progress. -// if (progress != null) { -// hlsc.addEventHook(new SimpleProgressUpdater(progress)); -// } -// uri = hlsc.insert(ib, false, null, priorityClass, ctx); -// if (progress != null) -// progress.addPartKnown(0, true); +//<<<<<<< HEAD +// /** +// ** {@inheritDoc} +// ** +// ** This implementation produces metdata of type {@link FreenetURI}. +// ** +// ** If the input metadata is an insert URI (SSK or USK), it will be replaced +// ** by its corresponding request URI. Otherwise, the data will be inserted +// ** as a CHK. Note that since {@link FreenetURI} is immutable, the {@link +// ** FreenetURI#suggestedEdition} of a USK is '''not''' automatically +// ** incremented. +// */ +// /*@Override**/ public void pushLive(PushTask task, final SimpleProgress progress) throws TaskAbortException { +// HighLevelSimpleClient hlsc = core.makeClient(priorityClass, false, false); +// RandomAccessBucket tempB = null; OutputStream os = null; +// +// +// try { +// ClientPutter putter = null; +// PushCallback cb = null; +// try { +// tempB = core.tempBucketFactory.makeBucket(expected_bytes, 2); +// os = tempB.getOutputStream(); +// writer.writeObject(task.data, os); +// os.close(); os = null; +// tempB.setReadOnly(); +// +// boolean insertAsMetadata; +// FreenetURI target; +// +// if(task.meta instanceof FreenetURI) { +// insertAsMetadata = false; +// target = (FreenetURI) task.meta; // } else { - // Do NOT report progress. Pretend we are done as soon as - // we have the URI. This allows us to minimise memory usage - // without yet splitting up IterableSerialiser.push() and - // doing it properly. FIXME - if(progress != null) - progress.addPartKnown(1, true); - cb = new PushCallback(progress, ib); - putter = new ClientPutter(cb, ib.getData(), FreenetURI.EMPTY_CHK_URI, ib.clientMetadata, - ctx, priorityClass, - false, null, false, core.clientContext, null, insertAsMetadata ? CHKBlock.DATA_LENGTH : -1); - cb.setPutter(putter); - long tStart = System.currentTimeMillis(); - try { - core.clientContext.start(putter); - } catch (PersistenceDisabledException e) { - // Impossible - } - WAIT_STATUS status = cb.waitFor(); - if(status == WAIT_STATUS.FAILED) { - cb.throwError(); - } else if(status == WAIT_STATUS.GENERATED_URI) { - FreenetURI uri = cb.getURI(); - task.meta = uri; - cacheKey = uri.toString(false, true); - Logger.debug(this, "Got URI for asynchronous insert: "+uri+" size "+tempB.size()+" in "+(System.currentTimeMillis() - cb.startTime)); - } else { - Bucket data = cb.getGeneratedMetadata(); - byte[] buf = BucketTools.toByteArray(data); - data.free(); - task.meta = buf; - cacheKey = Base64.encode(SHA256.digest(buf)); - Logger.debug(this, "Got generated metadata ("+buf.length+" bytes) for asynchronous insert size "+tempB.size()+" in "+(System.currentTimeMillis() - cb.startTime)); - } - if(progress != null) - progress.addPartDone(); +// insertAsMetadata = true; +// target = FreenetURI.EMPTY_CHK_URI; // } - - if(progress != null) { - ProgressParts prog_new = progress.getParts(); - if (prog_old.known - prog_old.done != prog_new.known - prog_new.done) { - Logger.error(this, "Inconsistency when tracking split file progress (pushing): "+prog_old.known+" of "+prog_old.done+" -> "+prog_new.known+" of "+prog_new.done); - System.err.println("Inconsistency when tracking split file progress (pushing): "+prog_old.known+" of "+prog_old.done+" -> "+prog_new.known+" of "+prog_new.done); - } - } - - task.data = null; - - if(cacheKey != null && cacheDir != null && cacheDir.exists() && cacheDir.canRead()) { - File cached = new File(cacheDir, cacheKey); - Bucket cachedBucket = new FileBucket(cached, false, false, false, false); - BucketTools.copy(tempB, cachedBucket); - } - - if(SEMI_ASYNC_PUSH) - tempB = null; // Don't free it here. - - } catch (InsertException e) { - if(cb != null) { - synchronized(this) { - if(semiAsyncPushes.remove(cb)) - totalBytesPushing -= cb.size(); - } - } - throw new TaskAbortException("Failed to insert content", e, true); - - } catch (IOException e) { - throw new TaskAbortException("Failed to write content to local tempbucket", e, true); - - } catch (RuntimeException e) { - throw new TaskAbortException("Failed to complete task: ", e); - - } - } catch (TaskAbortException e) { - if (progress != null) { progress.abort(e); } - throw e; +// InsertBlock ib = new InsertBlock(tempB, new ClientMetadata(default_mime), target); +// +// Logger.debug(this, "Inserting block for FreenetArchiver..."); +// long startTime = System.currentTimeMillis(); +// +// // code for async insert - maybe be useful elsewhere +// //ClientContext cctx = core.clientContext; +// //InsertContext ictx = hlsc.getInsertContext(true); +// //PutWaiter pw = new PutWaiter(); +// //ClientPutter pu = hlsc.insert(ib, false, null, false, ictx, pw); +// //pu.setPriorityClass(RequestStarter.INTERACTIVE_PRIORITY_CLASS, cctx, null); +// //FreenetURI uri = pw.waitForCompletion(); +// +// // bookkeeping. detects bugs in the SplitfileProgressEvent handler +// ProgressParts prog_old = null; +// if(progress != null) +// prog_old = progress.getParts(); +// +// // FIXME make retry count configgable by client metadata somehow +// // unlimited for push/merge +// InsertContext ctx = hlsc.getInsertContext(false); +// ctx.maxInsertRetries = -1; +// // Early encode is normally a security risk. +// // Hopefully it isn't here. +// ctx.earlyEncode = true; +// +// String cacheKey = null; +// +//// if(!SEMI_ASYNC_PUSH) { +//// // Actually report progress. +//// if (progress != null) { +//// hlsc.addEventHook(new SimpleProgressUpdater(progress)); +//// } +//// uri = hlsc.insert(ib, false, null, priorityClass, ctx); +//// if (progress != null) +//// progress.addPartKnown(0, true); +//// } else { +// // Do NOT report progress. Pretend we are done as soon as +// // we have the URI. This allows us to minimise memory usage +// // without yet splitting up IterableSerialiser.push() and +// // doing it properly. FIXME +// if(progress != null) +// progress.addPartKnown(1, true); +// cb = new PushCallback(progress, ib); +// putter = new ClientPutter(cb, ib.getData(), FreenetURI.EMPTY_CHK_URI, ib.clientMetadata, +// ctx, priorityClass, +// false, null, false, core.clientContext, null, insertAsMetadata ? CHKBlock.DATA_LENGTH : -1); +// cb.setPutter(putter); +// long tStart = System.currentTimeMillis(); +// try { +// core.clientContext.start(putter); +// } catch (PersistenceDisabledException e) { +// // Impossible +// } +// WAIT_STATUS status = cb.waitFor(); +// if(status == WAIT_STATUS.FAILED) { +// cb.throwError(); +// } else if(status == WAIT_STATUS.GENERATED_URI) { +// FreenetURI uri = cb.getURI(); +// task.meta = uri; +// cacheKey = uri.toString(false, true); +// Logger.debug(this, "Got URI for asynchronous insert: "+uri+" size "+tempB.size()+" in "+(System.currentTimeMillis() - cb.startTime)); +// } else { +// Bucket data = cb.getGeneratedMetadata(); +// byte[] buf = BucketTools.toByteArray(data); +// data.free(); +// task.meta = buf; +// cacheKey = Base64.encode(SHA256.digest(buf)); +// Logger.debug(this, "Got generated metadata ("+buf.length+" bytes) for asynchronous insert size "+tempB.size()+" in "+(System.currentTimeMillis() - cb.startTime)); +// } +// if(progress != null) +// progress.addPartDone(); +//// } +// +// if(progress != null) { +// ProgressParts prog_new = progress.getParts(); +// if (prog_old.known - prog_old.done != prog_new.known - prog_new.done) { +// Logger.error(this, "Inconsistency when tracking split file progress (pushing): "+prog_old.known+" of "+prog_old.done+" -> "+prog_new.known+" of "+prog_new.done); +// System.err.println("Inconsistency when tracking split file progress (pushing): "+prog_old.known+" of "+prog_old.done+" -> "+prog_new.known+" of "+prog_new.done); +// } +// } +// +// task.data = null; +// +// if(cacheKey != null && cacheDir != null && cacheDir.exists() && cacheDir.canRead()) { +// File cached = new File(cacheDir, cacheKey); +// Bucket cachedBucket = new FileBucket(cached, false, false, false, false); +// BucketTools.copy(tempB, cachedBucket); +// } +// +// if(SEMI_ASYNC_PUSH) +// tempB = null; // Don't free it here. +// +// } catch (InsertException e) { +// if(cb != null) { +// synchronized(this) { +// if(semiAsyncPushes.remove(cb)) +// totalBytesPushing -= cb.size(); +// } +// } +// throw new TaskAbortException("Failed to insert content", e, true); +// +// } catch (IOException e) { +// throw new TaskAbortException("Failed to write content to local tempbucket", e, true); +// +// } catch (RuntimeException e) { +// throw new TaskAbortException("Failed to complete task: ", e); +// +// } +// } catch (TaskAbortException e) { +// if (progress != null) { progress.abort(e); } +// throw e; +//======= +//>>>>>>> debbiedub/fcp-uploader - } finally { - Closer.close(os); - Closer.close(tempB); - } - } - enum WAIT_STATUS { FAILED, GENERATED_URI, GENERATED_METADATA; } - - public class PushCallback implements ClientPutCallback { - - public final long startTime = System.currentTimeMillis(); - private ClientPutter putter; - private FreenetURI generatedURI; - private Bucket generatedMetadata; - private InsertException failed; - // See FIXME's in push(), IterableSerialiser. - // We don't do real progress, we pretend we're done when push() returns. -// private final SimpleProgress progress; - private final long size; - private final InsertBlock ib; - - public PushCallback(SimpleProgress progress, InsertBlock ib) { -// this.progress = progress; - this.ib = ib; - size = ib.getData().size(); - } - - public long size() { - return size; - } - - public synchronized void setPutter(ClientPutter put) { - putter = put; - synchronized(FreenetArchiver.this) { - if(semiAsyncPushes.add(this)) - totalBytesPushing += size; - Logger.debug(this, "Pushing "+totalBytesPushing+" bytes on "+semiAsyncPushes.size()+" inserters"); - } - } - - public synchronized WAIT_STATUS waitFor() { - while(generatedURI == null && generatedMetadata == null && failed == null) { - try { - wait(); - } catch (InterruptedException e) { - // Ignore - } - } - if(failed != null) return WAIT_STATUS.FAILED; - if(generatedURI != null) return WAIT_STATUS.GENERATED_URI; - return WAIT_STATUS.GENERATED_METADATA; - } - - public synchronized void throwError() throws InsertException { - if(failed != null) throw failed; - } - - public synchronized FreenetURI getURI() { - return generatedURI; - } - - public synchronized Bucket getGeneratedMetadata() { - return generatedMetadata; - } - - @Override - public void onFailure(InsertException e, BaseClientPutter state) { - Logger.error(this, "Failed background insert ("+generatedURI+"), now running: "+semiAsyncPushes.size()+" ("+SizeUtil.formatSize(totalBytesPushing)+")."); - synchronized(this) { - failed = e; - notifyAll(); - } - synchronized(FreenetArchiver.this) { - if(semiAsyncPushes.remove(this)) - totalBytesPushing -= size; - pushesFailed.add(e); - FreenetArchiver.this.notifyAll(); - } - if(ib != null) - ib.free(); - } - - @Override - public void onFetchable(BaseClientPutter state) { - // Ignore - } - - @Override - public synchronized void onGeneratedURI(FreenetURI uri, BaseClientPutter state) { - generatedURI = uri; - notifyAll(); - } - - @Override - public void onSuccess(BaseClientPutter state) { - synchronized(FreenetArchiver.this) { - if(semiAsyncPushes.remove(this)) - totalBytesPushing -= size; - Logger.debug(this, "Completed background insert ("+generatedURI+") in "+(System.currentTimeMillis()-startTime)+"ms, now running: "+semiAsyncPushes.size()+" ("+SizeUtil.formatSize(totalBytesPushing)+")."); - FreenetArchiver.this.notifyAll(); - } - if(ib != null) - ib.free(); -// if(progress != null) progress.addPartKnown(0, true); - - } - - @Override - public synchronized void onGeneratedMetadata(Bucket metadata, - BaseClientPutter state) { - generatedMetadata = metadata; - notifyAll(); - } - - @Override - public void onResume(ClientContext context) throws ResumeFailedException { - // Ignore. - } - - @Override - public RequestClient getRequestClient() { - return Library.REQUEST_CLIENT; - } - - } +//<<<<<<< HEAD +// +// public class PushCallback implements ClientPutCallback { +// +// public final long startTime = System.currentTimeMillis(); +// private ClientPutter putter; +// private FreenetURI generatedURI; +// private Bucket generatedMetadata; +// private InsertException failed; +// // See FIXME's in push(), IterableSerialiser. +// // We don't do real progress, we pretend we're done when push() returns. +//// private final SimpleProgress progress; +// private final long size; +// private final InsertBlock ib; +// +// public PushCallback(SimpleProgress progress, InsertBlock ib) { +//// this.progress = progress; +// this.ib = ib; +// size = ib.getData().size(); +// } +// +// public long size() { +// return size; +// } +// +// public synchronized void setPutter(ClientPutter put) { +// putter = put; +// synchronized(FreenetArchiver.this) { +// if(semiAsyncPushes.add(this)) +// totalBytesPushing += size; +// Logger.debug(this, "Pushing "+totalBytesPushing+" bytes on "+semiAsyncPushes.size()+" inserters"); +// } +// } +// +// public synchronized WAIT_STATUS waitFor() { +// while(generatedURI == null && generatedMetadata == null && failed == null) { +// try { +// wait(); +// } catch (InterruptedException e) { +// // Ignore +// } +// } +// if(failed != null) return WAIT_STATUS.FAILED; +// if(generatedURI != null) return WAIT_STATUS.GENERATED_URI; +// return WAIT_STATUS.GENERATED_METADATA; +// } +// +// public synchronized void throwError() throws InsertException { +// if(failed != null) throw failed; +// } +// +// public synchronized FreenetURI getURI() { +// return generatedURI; +// } +// +// public synchronized Bucket getGeneratedMetadata() { +// return generatedMetadata; +// } +// +// @Override +// public void onFailure(InsertException e, BaseClientPutter state) { +// Logger.error(this, "Failed background insert ("+generatedURI+"), now running: "+semiAsyncPushes.size()+" ("+SizeUtil.formatSize(totalBytesPushing)+")."); +// synchronized(this) { +// failed = e; +// notifyAll(); +// } +// synchronized(FreenetArchiver.this) { +// if(semiAsyncPushes.remove(this)) +// totalBytesPushing -= size; +// pushesFailed.add(e); +// FreenetArchiver.this.notifyAll(); +// } +// if(ib != null) +// ib.free(); +// } +// +// @Override +// public void onFetchable(BaseClientPutter state) { +// // Ignore +// } +// +// @Override +// public synchronized void onGeneratedURI(FreenetURI uri, BaseClientPutter state) { +// generatedURI = uri; +// notifyAll(); +// } +// +// @Override +// public void onSuccess(BaseClientPutter state) { +// synchronized(FreenetArchiver.this) { +// if(semiAsyncPushes.remove(this)) +// totalBytesPushing -= size; +// Logger.debug(this, "Completed background insert ("+generatedURI+") in "+(System.currentTimeMillis()-startTime)+"ms, now running: "+semiAsyncPushes.size()+" ("+SizeUtil.formatSize(totalBytesPushing)+")."); +// FreenetArchiver.this.notifyAll(); +// } +// if(ib != null) +// ib.free(); +//// if(progress != null) progress.addPartKnown(0, true); +// +// } +// +// @Override +// public synchronized void onGeneratedMetadata(Bucket metadata, +// BaseClientPutter state) { +// generatedMetadata = metadata; +// notifyAll(); +// } +// +// @Override +// public void onResume(ClientContext context) throws ResumeFailedException { +// // Ignore. +// } +// +// @Override +// public RequestClient getRequestClient() { +// return Library.REQUEST_CLIENT; +// } +// +// } +//======= +//>>>>>>> debbiedub/fcp-uploader @@ -585,24 +592,34 @@ public SimpleProgressUpdater(SimpleProgress prog) { } } + @Override + public void pushLive(plugins.Library.io.serial.Serialiser.PushTask task, + SimpleProgress p) throws TaskAbortException { + throw new RuntimeException("Not implemented."); + } + + @Override public void waitForAsyncInserts() throws TaskAbortException { - synchronized(this) { - while(true) { - if(!pushesFailed.isEmpty()) { - throw new TaskAbortException("Failed to insert content", pushesFailed.remove(0), true); - } - if(semiAsyncPushes.isEmpty()) { - Logger.debug(this, "Asynchronous inserts completed."); - return; // Completed all pushes. - } - Logger.debug(this, "Waiting for "+semiAsyncPushes.size()+" asynchronous inserts ("+SizeUtil.formatSize(totalBytesPushing)+")..."); - try { - wait(); - } catch (InterruptedException e) { - // Ignore - } - } - } +//<<<<<<< HEAD +// synchronized(this) { +// while(true) { +// if(!pushesFailed.isEmpty()) { +// throw new TaskAbortException("Failed to insert content", pushesFailed.remove(0), true); +// } +// if(semiAsyncPushes.isEmpty()) { +// Logger.debug(this, "Asynchronous inserts completed."); +// return; // Completed all pushes. +// } +// Logger.debug(this, "Waiting for "+semiAsyncPushes.size()+" asynchronous inserts ("+SizeUtil.formatSize(totalBytesPushing)+")..."); +// try { +// wait(); +// } catch (InterruptedException e) { +// // Ignore +// } +// } +// } +//======= + throw new RuntimeException("Not implemented."); +//>>>>>>> debbiedub/fcp-uploader } - } diff --git a/src/plugins/Library/Index.java b/src/plugins/Library/index/Index.java similarity index 64% rename from src/plugins/Library/Index.java rename to src/plugins/Library/index/Index.java index 196ffb1a..fb076d4d 100644 --- a/src/plugins/Library/Index.java +++ b/src/plugins/Library/index/Index.java @@ -1,14 +1,10 @@ /* This code is part of Freenet. It is distributed under the GNU General * Public License, version 2 (or at your option any later version). See * http://www.gnu.org/ for further details of the GPL. */ -package plugins.Library; +package plugins.Library.index; -import plugins.Library.index.TermEntry; -import plugins.Library.index.URIEntry; import plugins.Library.util.exec.Execution; -import freenet.keys.FreenetURI; - import java.util.Set; /** @@ -23,8 +19,5 @@ public interface Index { ** ** DOCUMENT */ - public Execution> getTermEntries(String term); - - public Execution getURIEntry(FreenetURI uri); - + Execution> getTermEntries(String term); } diff --git a/src/plugins/Library/index/ProtoIndex.java b/src/plugins/Library/index/ProtoIndex.java index 002e03bc..6998d476 100644 --- a/src/plugins/Library/index/ProtoIndex.java +++ b/src/plugins/Library/index/ProtoIndex.java @@ -3,38 +3,30 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.index; -import plugins.Library.Index; -import plugins.Library.io.serial.Serialiser; +import java.util.AbstractSet; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executor; + +import plugins.Library.io.FreenetURI; import plugins.Library.io.serial.ProgressTracker; +import plugins.Library.io.serial.Serialiser; +import plugins.Library.util.DataNotLoadedException; import plugins.Library.util.Skeleton; -import plugins.Library.util.SkeletonTreeMap; import plugins.Library.util.SkeletonBTreeMap; import plugins.Library.util.SkeletonBTreeSet; -import plugins.Library.util.DataNotLoadedException; -import plugins.Library.util.exec.Progress; -import plugins.Library.util.exec.ProgressParts; +import plugins.Library.util.concurrent.Executors; +import plugins.Library.util.exec.AbstractExecution; import plugins.Library.util.exec.ChainedProgress; import plugins.Library.util.exec.Execution; -import plugins.Library.util.exec.AbstractExecution; +import plugins.Library.util.exec.Progress; +import plugins.Library.util.exec.ProgressParts; import plugins.Library.util.exec.TaskAbortException; -import plugins.Library.util.concurrent.Executors; - -import freenet.keys.FreenetURI; -import freenet.support.Logger; - -import java.util.AbstractSet; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.Set; -import java.util.Map; -import java.util.SortedSet; -import java.util.HashMap; -import java.util.TreeSet; -import java.util.LinkedHashMap; -import java.util.Date; - -import java.util.concurrent.Executor; /** ** Prototype B-tree based index. DOCUMENT @@ -158,14 +150,6 @@ public Execution> getTermEntries(String term) { return request; } - - - - public Execution getURIEntry(FreenetURI uri) { - throw new UnsupportedOperationException("not implemented"); - } - - public class getTermEntriesHandler extends AbstractExecution> implements Runnable, ChainedProgress { // TODO NORM have a Runnable field instead of extending Runnable // basically, redesign this entire class and series of classes @@ -232,10 +216,12 @@ protected getTermEntriesHandler(String t) { long specific = root.size(); // Number of pages in this entry multiplier = Math.log(((double)total) / ((double)specific)); if(multiplier < 0) { - Logger.error(this, "Negative multiplier!: "+multiplier+" total = "+total+" specific = "+root.size()); + // Logger.error(this, "Negative multiplier!: "+multiplier+" total = "+total+" specific = "+root.size()); + System.out.println("Negative multiplier!: "+multiplier+" total = "+total+" specific = "+root.size()); multiplier = 1.0; - } else - Logger.normal(this, "Correcting results: "+multiplier); + } else { + // Logger.normal(this, "Correcting results: "+multiplier); + } } Set entries = wrapper(root, multiplier); diff --git a/src/plugins/Library/index/ProtoIndexComponentSerialiser.java b/src/plugins/Library/index/ProtoIndexComponentSerialiser.java index 73d6f4f4..3f765976 100644 --- a/src/plugins/Library/index/ProtoIndexComponentSerialiser.java +++ b/src/plugins/Library/index/ProtoIndexComponentSerialiser.java @@ -3,46 +3,35 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.index; -import plugins.Library.Library; -import plugins.Library.client.FreenetArchiver; -import plugins.Library.util.SkeletonTreeMap; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +import plugins.Library.FactoryRegister; +import plugins.Library.io.DataFormatException; +import plugins.Library.io.YamlReaderWriter; +import plugins.Library.io.serial.Archiver; +import plugins.Library.io.serial.FileArchiver; +import plugins.Library.io.serial.IterableSerialiser; +import plugins.Library.io.serial.LiveArchiver; +import plugins.Library.io.serial.MapSerialiser; +import plugins.Library.io.serial.Packer; +import plugins.Library.io.serial.Packer.Scale; +import plugins.Library.io.serial.ParallelSerialiser; +import plugins.Library.io.serial.ProgressTracker; +import plugins.Library.io.serial.Serialiser; +import plugins.Library.io.serial.Translator; import plugins.Library.util.SkeletonBTreeMap; import plugins.Library.util.SkeletonBTreeSet; -import plugins.Library.util.exec.ProgressParts; +import plugins.Library.util.SkeletonTreeMap; +import plugins.Library.util.exec.BaseCompositeProgress; import plugins.Library.util.exec.Progress; +import plugins.Library.util.exec.ProgressParts; import plugins.Library.util.exec.SimpleProgress; -import plugins.Library.util.exec.BaseCompositeProgress; import plugins.Library.util.exec.TaskAbortException; import plugins.Library.util.exec.TaskInProgressException; -import plugins.Library.io.serial.Serialiser.*; -import plugins.Library.io.serial.Serialiser; -import plugins.Library.io.serial.Translator; -import plugins.Library.io.serial.ProgressTracker; -import plugins.Library.io.serial.Archiver; -import plugins.Library.io.serial.IterableSerialiser; -import plugins.Library.io.serial.MapSerialiser; -import plugins.Library.io.serial.LiveArchiver; -import plugins.Library.io.serial.ParallelSerialiser; -import plugins.Library.io.serial.Packer; -import plugins.Library.io.serial.Packer.Scale; // WORKAROUND javadoc bug #4464323 -import plugins.Library.io.serial.FileArchiver; -import plugins.Library.io.DataFormatException; -import plugins.Library.io.YamlReaderWriter; - -import freenet.keys.FreenetURI; -import freenet.node.RequestStarter; - -import java.io.File; -import java.util.Collection; -import java.util.Set; -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.LinkedHashMap; -import java.util.HashMap; -import java.util.TreeSet; -import java.util.TreeMap; -import java.util.Date; +import plugins.Library.io.FreenetURI; /** ** Serialiser for the components of a ProtoIndex. @@ -205,10 +194,11 @@ protected ProtoIndexComponentSerialiser(int fmtid, LiveArchiver>(yamlrw, true, YamlReaderWriter.FILE_EXTENSION, "", "", null); @@ -380,8 +370,14 @@ public BTreeNodeSerialiser(String n, LiveArchiver, SimplePro ghost.setMeta(serialisable.meta); task.data = trans.rev(serialisable.data); p.exitingSerialiser(); } catch (RuntimeException e) { + System.out.println("RuntimeException"); + System.out.println(e); + e.printStackTrace(); p.abort(new TaskAbortException("Could not pull B-tree node", e)); } catch (DataFormatException e) { + System.out.println("DataFormatException"); + System.out.println(e); + e.printStackTrace(); p.abort(new TaskAbortException("Could not pull B-tree node", e)); } } @@ -400,6 +396,11 @@ public BTreeNodeSerialiser(String n, LiveArchiver, SimplePro } } + @Override + public void waitForAsyncInserts() throws TaskAbortException { + subsrl.waitForAsyncInserts(); + } + } @@ -524,6 +525,11 @@ public EntryGroupSerialiser(LiveArchiver, SimpleProgress> s, } } + @Override + public void waitForAsyncInserts() throws TaskAbortException { + subsrl.waitForAsyncInserts(); + } + } diff --git a/src/plugins/Library/index/ProtoIndexSerialiser.java b/src/plugins/Library/index/ProtoIndexSerialiser.java index 09c1a10a..1405c65b 100644 --- a/src/plugins/Library/index/ProtoIndexSerialiser.java +++ b/src/plugins/Library/index/ProtoIndexSerialiser.java @@ -3,13 +3,10 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.index; -import plugins.Library.Library; -import plugins.Library.client.FreenetArchiver; import plugins.Library.util.SkeletonBTreeMap; import plugins.Library.util.SkeletonBTreeSet; import plugins.Library.util.exec.SimpleProgress; import plugins.Library.util.exec.TaskAbortException; -import plugins.Library.io.serial.Serialiser.*; import plugins.Library.io.serial.LiveArchiver; import plugins.Library.io.serial.Serialiser; import plugins.Library.io.serial.Translator; @@ -18,19 +15,17 @@ import plugins.Library.io.YamlReaderWriter; import plugins.Library.io.DataFormatException; -import freenet.keys.FreenetURI; +import java.text.ParseException; +import java.text.SimpleDateFormat; -import java.util.Collection; -import java.util.Set; -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.LinkedHashMap; -import java.util.HashMap; -import java.util.TreeSet; -import java.util.TreeMap; -import java.util.Date; import java.io.File; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.Map; + +import plugins.Library.FactoryRegister; +import plugins.Library.io.FreenetURI; +import plugins.Library.Priority; /** ** Serialiser for ProtoIndex @@ -67,17 +62,18 @@ public ProtoIndexSerialiser(LiveArchiver, SimpleProgress> s) // final protected static HashMap, ProtoIndexSerialiser> // srl_cls = new HashMap, ProtoIndexSerialiser>(); - public static ProtoIndexSerialiser forIndex(Object o, short priorityClass) { + public static ProtoIndexSerialiser forIndex(Object o, Priority priorityLevel) { if (o instanceof FreenetURI) { - return forIndex((FreenetURI)o, priorityClass); + return forIndex((FreenetURI)o, priorityLevel); } else if (o instanceof File) { return forIndex((File)o); } else { - throw new UnsupportedOperationException("Don't know how to retrieve index for object " + o); + throw new UnsupportedOperationException("Don't know how to retrieve index for object " + o + + " (of type " + o.getClass().getName() + ")"); } } - public static ProtoIndexSerialiser forIndex(FreenetURI uri, short priorityClass) { + public static ProtoIndexSerialiser forIndex(FreenetURI uri, Priority priorityLevel) { // ProtoIndexSerialiser srl = srl_cls.get(FreenetURI.class); // if (srl == null) { // // java's type-inference isn't that smart, see @@ -88,7 +84,7 @@ public static ProtoIndexSerialiser forIndex(FreenetURI uri, short priorityClass) // One serialiser per application. See comments above re srl_cls. // java's type-inference isn't that smart, see - FreenetArchiver> arx = Library.makeArchiver(ProtoIndexComponentSerialiser.yamlrw, MIME_TYPE, 0x80 * ProtoIndex.BTREE_NODE_MIN, priorityClass); + LiveArchiver, SimpleProgress> arx = FactoryRegister.getArchiverFactory().newArchiver(ProtoIndexComponentSerialiser.yamlrw, MIME_TYPE, 0x80 * ProtoIndex.BTREE_NODE_MIN, priorityLevel); return new ProtoIndexSerialiser(arx); } @@ -172,7 +168,7 @@ public IndexTranslator(LiveArchiver, SimpleProgress> subsrl) throw new IllegalArgumentException("Data structure is not bare. Try calling deflate() first."); } Map map = new LinkedHashMap(); - map.put("serialVersionUID", idx.serialVersionUID); + map.put("serialVersionUID", ProtoIndex.serialVersionUID); map.put("serialFormatUID", idx.serialFormatUID); map.put("insID", idx.insID); map.put("name", idx.name); @@ -187,27 +183,53 @@ public IndexTranslator(LiveArchiver, SimpleProgress> subsrl) } /*@Override**/ public ProtoIndex rev(Map map) throws DataFormatException { - long magic = (Long)map.get("serialVersionUID"); + Object serialVersionUID = map.get("serialVersionUID"); + long magic; + if (serialVersionUID instanceof String) { // FIXME + magic = Long.parseLong((String) map.get("serialVersionUID")); + } else { + magic = (Long) serialVersionUID; + } if (magic == ProtoIndex.serialVersionUID) { try { // FIXME yet more hacks related to the lack of proper asynchronous FreenetArchiver... - ProtoIndexComponentSerialiser cmpsrl = ProtoIndexComponentSerialiser.get((Integer)map.get("serialFormatUID"), subsrl); - FreenetURI reqID = (FreenetURI)map.get("reqID"); + Object serialFormatUIDObj = map.get("serialFormatUID"); + int serialFormatUID; + if (serialFormatUIDObj instanceof String) { // FIXME + serialFormatUID = Integer.parseInt((String) map.get("serialFormatUID")); + } else { + serialFormatUID = (Integer) serialFormatUIDObj; + } + ProtoIndexComponentSerialiser cmpsrl = ProtoIndexComponentSerialiser.get(serialFormatUID, subsrl); + + FreenetURI reqID = (FreenetURI) map.get("reqID"); String name = (String)map.get("name"); String ownerName = (String)map.get("ownerName"); String ownerEmail = (String)map.get("ownerEmail"); - // FIXME yaml idiocy??? It seems to give a Long if the number is big enough to need one, and an Integer otherwise. + Object totalPagesObj = map.get("totalPages"); long totalPages; - Object o = map.get("totalPages"); - if(o instanceof Long) - totalPages = (Long)o; - else // Integer - totalPages = (Integer)o; - Date modified = (Date)map.get("modified"); + if (totalPagesObj instanceof String) { // FIXME + totalPages = Long.parseLong((String) totalPagesObj); + } else if (totalPagesObj instanceof Long) { // FIXME yaml??? It seems to give a Long if the number + totalPages = (Long) totalPagesObj; // is big enough to need one, and an Integer otherwise. + } else { + totalPages = (Integer) totalPagesObj; + } + Object modifiedObj = map.get("modified"); + Date modified; + if (modifiedObj instanceof String) { // FIXME + try { + modified = new SimpleDateFormat("yyyy-MM-dd").parse((String) modifiedObj); + } catch (ParseException ignored) { + modified = null; + } + } else { + modified = (Date) modifiedObj; + } Map extra = (Map)map.get("extra"); SkeletonBTreeMap> utab = utrans.rev((Map)map.get("utab")); - SkeletonBTreeMap> ttab = ttrans.rev((Map)map.get("ttab")); + SkeletonBTreeMap> ttab = ttrans.rev((Map) map.get("ttab")); return cmpsrl.setSerialiserFor(new ProtoIndex(reqID, name, ownerName, ownerEmail, totalPages, modified, extra, utab, ttab)); diff --git a/src/plugins/Library/index/TermEntry.java b/src/plugins/Library/index/TermEntry.java index e5832b5a..68d92eeb 100644 --- a/src/plugins/Library/index/TermEntry.java +++ b/src/plugins/Library/index/TermEntry.java @@ -37,7 +37,8 @@ public enum EntryType { public TermEntry(String s, float r) { if (s == null) { - throw new IllegalArgumentException("can't have a null subject!"); +// throw new IllegalArgumentException("can't have a null subject!"); + s = "null"; // FIXME } if (r < 0/* || r > 1*/) { // FIXME: I don't see how our relevance algorithm can be guaranteed to produce relevance <1. throw new IllegalArgumentException("Relevance must be in the half-closed interval (0,1]. Supplied: " + r); @@ -80,6 +81,26 @@ public TermEntry(TermEntry t, float newRel) { return a.compareTo(b); } + /** + * Is this term a part of a key. + * + * Spider creates a lot of terms like this. It looks like whenever + * it finds a key in the text of a page it creates terms for the parts + * of the key. For now, we would like to avoid changing spider and + * remove them here instead. This also drops torrent references. + * + * @return true if likely part of a key. + */ + public boolean toBeDropped() { + if (subj.matches(".*[0-9][^0-9][^0-9][^0-9]*[0-9].*")) { + return true; + } + if (subj.matches(".*[0-9][^0-9][^0-9]*[0-9][0-9]*[^0-9][^0-9]*[0-9].*")) { + return true; + } + return false; + } + /** ** {@inheritDoc} ** diff --git a/src/plugins/Library/index/TermEntryReaderWriter.java b/src/plugins/Library/index/TermEntryReaderWriter.java index 1b73fd36..9e9ea529 100644 --- a/src/plugins/Library/index/TermEntryReaderWriter.java +++ b/src/plugins/Library/index/TermEntryReaderWriter.java @@ -3,24 +3,35 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.index; + import plugins.Library.io.DataFormatException; import plugins.Library.io.ObjectStreamReader; import plugins.Library.io.ObjectStreamWriter; - -import freenet.keys.FreenetURI; +import plugins.Library.io.FreenetURI; +import freenet.support.Base64; import java.util.Map; import java.util.HashMap; +import java.util.regex.Pattern; +import java.io.ByteArrayInputStream; import java.io.InputStream; import java.io.OutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.net.URLEncoder; + +import org.yaml.snakeyaml.reader.ReaderException; /** ** Reads and writes {@link TermEntry}s in binary form, for performance. ** +** This needs to be able to read FreenetURI:s as generated from Spider but +** everything written uses the simpler String-version of the FreenetURI. +** To distringuish between them, when using the String-version, it is always +** preceeded with a short that is 0. +** ** @author infinity0 */ public class TermEntryReaderWriter implements ObjectStreamReader, ObjectStreamWriter { @@ -37,6 +48,89 @@ public static TermEntryReaderWriter getInstance() { return readObject(new DataInputStream(is)); } + static final byte CHK = 1; + static final byte SSK = 2; + static final byte KSK = 3; + static final byte USK = 4; + static final short ClientCHK_EXTRA_LENGTH = 5; + static final short ClientSSK_EXTRA_LENGTH = 5; + + /** + * This is to be able to read the data created by Spider. + */ + private String readFreenetURI(DataInputStream dis1) throws IOException { + int len = dis1.readShort(); + if (len == 0) { + // This is the new format. + return dis1.readUTF(); + } + byte[] buf = new byte[len]; + dis1.readFully(buf); + ByteArrayInputStream bais = new ByteArrayInputStream(buf); + DataInputStream dis = new DataInputStream(bais); + byte type = dis.readByte(); + String keyType; + if(type == CHK) + keyType = "CHK"; + else if(type == SSK) + keyType = "SSK"; + else if(type == KSK) + keyType = "KSK"; + else + throw new IOException("Unrecognized FreenetURI type " + type); + byte[] routingKey = null; + byte[] cryptoKey = null; + byte[] extra = null; + if((type == CHK) || (type == SSK)) { + // routingKey is a hash, so is exactly 32 bytes + routingKey = new byte[32]; + dis.readFully(routingKey); + // cryptoKey is a 256 bit AES key, so likewise + cryptoKey = new byte[32]; + dis.readFully(cryptoKey); + // Number of bytes of extra depends on key type + int extraLen; + extraLen = (type == CHK ? ClientCHK_EXTRA_LENGTH : ClientSSK_EXTRA_LENGTH); + extra = new byte[extraLen]; + dis.readFully(extra); + } + + String docName = null; + if(type != CHK) + docName = dis.readUTF(); + int count = dis.readInt(); + String[] metaStrings = new String[count]; + for(int i = 0; i < metaStrings.length; i++) + metaStrings[i] = dis.readUTF(); + + StringBuilder b = new StringBuilder(); + + b.append(keyType).append('@'); + if(!"KSK".equals(keyType)) { + if(routingKey != null) + b.append(Base64.encode(routingKey)); + if(cryptoKey != null) + b.append(',').append(Base64.encode(cryptoKey)); + if(extra != null) + b.append(',').append(Base64.encode(extra)); + if(docName != null) + b.append('/'); + } + + if(docName != null) + b.append(URLEncoder.encode(docName, "UTF-8")); + + //if(keyType.equals("USK")) { + // b.append('/'); + // b.append(suggestedEdition); + // } + + for(int i = 0; i < metaStrings.length; i++) { + b.append('/').append(URLEncoder.encode(metaStrings[i], "UTF-8")); + } + return b.toString(); + } + public TermEntry readObject(DataInputStream dis) throws IOException { long svuid = dis.readLong(); if (svuid != TermEntry.serialVersionUID) { @@ -61,6 +155,9 @@ public TermEntry readObject(DataInputStream dis) throws IOException { if (size < 0) { title = dis.readUTF(); size = ~size; + if (!isValid(title)) { + title = clean(title); + } } Map pos = new HashMap(size<<1); for (int i=0; i for ObjectBlueprint but will really always be a SortedIntSet. */ + /** + * Positions where the term occurs. May be null if we don't have that data. + * + * Retrieved from the YamlReaderWriter so must be public and + * have the Set type to be able to read the old indexes. + */ final public Set positions; /** @@ -81,36 +98,30 @@ public TermPageEntry(String s, float r, FreenetURI u, String t, Map(p.keySet()); } } /** ** For serialisation. */ - public TermPageEntry(String s, float r, FreenetURI u, String t, Set pos, Map frags) { + public TermPageEntry(String s, float r, Object u, String t, Set pos, Map frags) { super(s, r); if (u == null) { throw new IllegalArgumentException("can't have a null page"); } - page = u.intern(); // OPT LOW make the translator use the same URI object as from the URI table? + if (u instanceof String) { + try { + page = new FreenetURI((String) u); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } else { + page = ((FreenetURI) u).intern(); // OPT LOW make the translator use the same URI object as from the URI table? + } title = t; if(pos != null) { - if(pos instanceof SortedIntSet) - this.positions = (SortedIntSet) pos; - else { - Integer[] p = pos.toArray(new Integer[pos.size()]); - int[] pp = new int[p.length]; - for(int i=0;i(pos); } else positions = null; if(frags != null) { @@ -177,19 +188,9 @@ public Map positionsMap() { if(positions == null) return null; if(posFragments != null) return posFragments; HashMap ret = new HashMap(positions.size()); - if(positions instanceof SortedIntSet) { - int[] array = ((SortedIntSet)positions).toArrayRaw(); - for(int x : array) - ret.put(x, null); - return ret; - } else { - Integer[] array = positions.toArray(new Integer[positions.size()]); - if(!(positions instanceof SortedSet)) - Arrays.sort(array); - for(int x : array) - ret.put(x, null); - return ret; - } + for(int x : positions) + ret.put(x, null); + return ret; } public boolean hasPosition(int i) { @@ -197,35 +198,23 @@ public boolean hasPosition(int i) { } public ArrayList positions() { - if(positions instanceof SortedIntSet) { - int[] array = ((SortedIntSet)positions).toArrayRaw(); - ArrayList pos = new ArrayList(array.length); - for(int x : array) - pos.add(x); - return pos; - } else { - Integer[] array = positions.toArray(new Integer[positions.size()]); - if(!(positions instanceof SortedSet)) - Arrays.sort(array); - ArrayList ret = new ArrayList(positions.size()); - for(int i=0;i ret = new ArrayList(positions.size()); + for(int i=0;i map) throws InstantiationException, IllegalAcce String property = en.getKey(); Class type = en.getValue(); Object value = map.get(property); + if (value != null && value.equals("null")) { // FIXME: case when type Map and value String "null" + value = null; + } try { if (type.isPrimitive()) { value = boxCast(type, value); @@ -335,25 +338,25 @@ public Map objectAsMap(final T object) { return new AbstractMap() { - final private Set> entrySet = new AbstractSet>() { + final private Set> entrySet = new AbstractSet>() { @Override public int size() { return properties.size(); } - @Override public Iterator> iterator() { - return new Iterator>() { - final Iterator> props = properties.entrySet().iterator(); + @Override public Iterator> iterator() { + return new Iterator>() { + final Iterator> props = properties.entrySet().iterator(); /*@Override**/ public boolean hasNext() { return props.hasNext(); } - /*@Override**/ public Map.Entry next() { - Map.Entry en = props.next(); + /*@Override**/ public Entry next() { + Entry en = props.next(); final String property = en.getKey(); final String method_name = en.getValue(); - return new Map.Entry() { + return new Entry() { /*@Override**/ public String getKey() { return property; } /*@Override**/ public Object getValue() { return get(property, method_name); } /*@Override**/ public Object setValue(Object o) { throw new UnsupportedOperationException("Cannot modify an object in this way."); } @@ -395,7 +398,7 @@ protected Object get(String property, String method_name) { } } - @Override public Set> entrySet() { + @Override public Set> entrySet() { return entrySet; } diff --git a/src/plugins/Library/io/YamlReaderWriter.java b/src/plugins/Library/io/YamlReaderWriter.java index 4c3118e6..e0bb1ccc 100644 --- a/src/plugins/Library/io/YamlReaderWriter.java +++ b/src/plugins/Library/io/YamlReaderWriter.java @@ -3,13 +3,12 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.io; -import plugins.Library.io.DataFormatException; +import freenet.support.SortedIntSet; +import org.yaml.snakeyaml.nodes.Tag; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.error.YAMLException; import org.yaml.snakeyaml.error.Mark; -import org.yaml.snakeyaml.Loader; -import org.yaml.snakeyaml.Dumper; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.representer.Representer; import org.yaml.snakeyaml.representer.Represent; @@ -19,42 +18,37 @@ import org.yaml.snakeyaml.constructor.Constructor; import org.yaml.snakeyaml.constructor.AbstractConstruct; -import java.util.Collections; -import java.util.Arrays; -import java.util.Map; -import java.util.LinkedHashMap; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; +import java.util.*; import java.util.concurrent.Semaphore; -import java.io.File; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.InputStream; import java.io.InputStreamReader; import java.io.IOException; +import java.net.MalformedURLException; /* class definitions added to the extended Yaml processor */ +import org.yaml.snakeyaml.resolver.Resolver; import plugins.Library.io.serial.Packer; import plugins.Library.index.TermEntry; import plugins.Library.index.TermPageEntry; import plugins.Library.index.TermIndexEntry; import plugins.Library.index.TermTermEntry; -import freenet.keys.FreenetURI; - /** -** Converts between an object and a stream containing a YAML document. By -** default, this uses a {@link Yaml} processor with additional object and tag -** definitions relevant to the Library plugin. -** -** (Ideally this would implement {@link java.io.ObjectInput} and {@link -** java.io.ObjectOutput} but they have too many methods to bother with...) -** -** @see Yaml -** @see Loader -** @see Dumper -** @author infinity0 +* Converts between an object and a stream containing a YAML document. By +* default, this uses a {@link Yaml} processor with additional object and tag +* definitions relevant to the Library plugin. +* +* (Ideally this would implement {@link java.io.ObjectInput} and {@link +* java.io.ObjectOutput} but they have too many methods to bother with...) +* +* @see Yaml +* @author infinity0 */ -public class YamlReaderWriter -implements ObjectStreamReader, ObjectStreamWriter { +public class YamlReaderWriter implements ObjectStreamReader, ObjectStreamWriter { final public static String MIME_TYPE = "text/yaml"; final public static String FILE_EXTENSION = ".yml"; @@ -66,10 +60,11 @@ public class YamlReaderWriter public YamlReaderWriter() { } - /*@Override**/ public Object readObject(InputStream is) throws IOException { + @Override + public Object readObject(InputStream is) throws IOException { parallelLimiter.acquireUninterruptibly(); try { - return makeYAML().load(new InputStreamReader(is, "UTF-8")); + return makeYAML().load(new InputStreamReader(is, StandardCharsets.UTF_8)); } catch (YAMLException e) { throw new DataFormatException("Yaml could not process the stream: " + is, e, is, null, null); } finally { @@ -77,10 +72,11 @@ public YamlReaderWriter() { } } - /*@Override**/ public void writeObject(Object o, OutputStream os) throws IOException { + @Override + public void writeObject(Object o, OutputStream os) throws IOException { parallelLimiter.acquireUninterruptibly(); try { - makeYAML().dump(o, new OutputStreamWriter(os, "UTF-8")); + makeYAML().dump(o, new OutputStreamWriter(os, StandardCharsets.UTF_8)); } catch (YAMLException e) { throw new DataFormatException("Yaml could not process the object", e, o, null, null); } finally { @@ -88,15 +84,20 @@ public YamlReaderWriter() { } } - /** We do NOT keep this thread-local, because the Composer is only cleared after + /** + * We do NOT keep this thread-local, because the Composer is only cleared after * the next call to load(), so it can persist with a lot of useless data if we - * then use a different thread. So lets just construct them as needed. */ + * then use a different thread. So lets just construct them as needed. + */ private Yaml makeYAML() { DumperOptions opt = new DumperOptions(); opt.setWidth(Integer.MAX_VALUE); opt.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - return new Yaml(new Loader(new ExtendedConstructor()), - new Dumper(new ExtendedRepresenter(), opt)); + return new Yaml(new ExtendedConstructor(), new ExtendedRepresenter(), opt, new Resolver() { + @Override + protected void addImplicitResolvers() { // disable implicit resolvers + } + }); } final public static ObjectBlueprint tebp_term; @@ -104,37 +105,40 @@ private Yaml makeYAML() { final public static ObjectBlueprint tebp_page; static { try { - tebp_term = new ObjectBlueprint(TermTermEntry.class, Arrays.asList("subj", "rel", "term")); - tebp_index = new ObjectBlueprint(TermIndexEntry.class, Arrays.asList("subj", "rel", "index")); - tebp_page = new ObjectBlueprint(TermPageEntry.class, Arrays.asList("subj", "rel", "page", "title", "positions", "posFragments")); - } catch (NoSuchFieldException e) { - throw new AssertionError(e); - } catch (NoSuchMethodException e) { + tebp_term = new ObjectBlueprint<>(TermTermEntry.class, + Arrays.asList("subj", "rel", "term")); + tebp_index = new ObjectBlueprint<>(TermIndexEntry.class, + Arrays.asList("subj", "rel", "index")); + tebp_page = new ObjectBlueprint<>(TermPageEntry.class, + Arrays.asList("subj", "rel", "page", "title", "positions", "posFragments")); + } catch (NoSuchFieldException | NoSuchMethodException e) { throw new AssertionError(e); } } - /************************************************************************ - ** DOCUMENT + /** + * DOCUMENT */ public static class ExtendedRepresenter extends Representer { public ExtendedRepresenter() { this.representers.put(FreenetURI.class, new Represent() { - /*@Override**/ public Node representData(Object data) { - return representScalar("!FreenetURI", ((FreenetURI) data).toString()); + @Override + public Node representData(Object data) { + return representScalar(new Tag("!FreenetURI"), data.toString()); } }); this.representers.put(Packer.BinInfo.class, new Represent() { - /*@Override**/ public Node representData(Object data) { - Packer.BinInfo inf = (Packer.BinInfo)data; - Map map = Collections.singletonMap(inf.getID(), inf.getWeight()); - return representMapping("!BinInfo", map, true); + @Override + public Node representData(Object data) { + Packer.BinInfo inf = (Packer.BinInfo) data; + Map map = Collections.singletonMap(inf.getID(), inf.getWeight()); + return representMapping(new Tag("!BinInfo"), map, DumperOptions.FlowStyle.FLOW); } }); - this.representers.put(TermTermEntry.class, new RepresentTermEntry(tebp_term)); - this.representers.put(TermIndexEntry.class, new RepresentTermEntry(tebp_index)); - this.representers.put(TermPageEntry.class, new RepresentTermEntry(tebp_page)); + this.representers.put(TermTermEntry.class, new RepresentTermEntry<>(tebp_term)); + this.representers.put(TermIndexEntry.class, new RepresentTermEntry<>(tebp_index)); + this.representers.put(TermPageEntry.class, new RepresentTermEntry<>(tebp_page)); } public class RepresentTermEntry implements Represent { @@ -147,45 +151,52 @@ public RepresentTermEntry(ObjectBlueprint bp) { tag = "!" + bp.getObjectClass().getSimpleName(); } - /*@Override**/ public Node representData(Object data) { - return representMapping(tag, blueprint.objectAsMap((T)data), true); + @Override + public Node representData(Object data) { + return representMapping(new Tag(tag), blueprint.objectAsMap((T) data), DumperOptions.FlowStyle.FLOW); } - } - } - - /************************************************************************ - ** DOCUMENT + /** + * DOCUMENT */ public static class ExtendedConstructor extends Constructor { public ExtendedConstructor() { - this.yamlConstructors.put("!FreenetURI", new AbstractConstruct() { - /*@Override**/ public Object construct(Node node) { - String uri = (String) constructScalar((ScalarNode)node); + this.yamlConstructors.put(new Tag("!FreenetURI"), new AbstractConstruct() { + @Override + public Object construct(Node node) { + String uri = constructScalar((ScalarNode) node); try { return new FreenetURI(uri); - } catch (java.net.MalformedURLException e) { - throw new ConstructorException("while constructing a FreenetURI", node.getStartMark(), "found malformed URI " + uri, null); + } catch (MalformedURLException e) { + throw new ConstructorException("while constructing a FreenetURI", node.getStartMark(), + "found malformed URI " + uri, null); } } }); - this.yamlConstructors.put("!BinInfo", new AbstractConstruct() { - /*@Override**/ public Object construct(Node node) { - Map map = (Map) constructMapping((MappingNode)node); + this.yamlConstructors.put(new Tag("!BinInfo"), new AbstractConstruct() { + @Override + public Object construct(Node node) { + Map map = constructMapping((MappingNode) node); if (map.size() != 1) { - throw new ConstructorException("while constructing a Packer.BinInfo", node.getStartMark(), "found incorrectly sized map data " + map, null); + throw new ConstructorException("while constructing a Packer.BinInfo", node.getStartMark(), + "found incorrectly sized map data " + map, null); } - for (Map.Entry en: map.entrySet()) { - return new Packer.BinInfo(en.getKey(), (Integer)en.getValue()); + + Map.Entry entry = map.entrySet().iterator().next(); + int w; // FIXME + if (entry.getValue() instanceof String) { + w = Integer.parseInt((String) entry.getValue()); + } else { + w = (Integer) entry.getValue(); } - throw new AssertionError(); + return new Packer.BinInfo(entry.getKey(), w); } }); - this.yamlConstructors.put("!TermTermEntry", new ConstructTermEntry(tebp_term)); - this.yamlConstructors.put("!TermIndexEntry", new ConstructTermEntry(tebp_index)); - this.yamlConstructors.put("!TermPageEntry", new ConstructTermEntry(tebp_page)); + this.yamlConstructors.put(new Tag("!TermTermEntry"), new ConstructTermEntry<>(tebp_term)); + this.yamlConstructors.put(new Tag("!TermIndexEntry"), new ConstructTermEntry<>(tebp_index)); + this.yamlConstructors.put(new Tag("!TermPageEntry"), new ConstructTermEntry<>(tebp_page)); } public class ConstructTermEntry extends AbstractConstruct { @@ -196,19 +207,63 @@ public ConstructTermEntry(ObjectBlueprint bp) { blueprint = bp; } - /*@Override**/ public Object construct(Node node) { - Map map = (Map)constructMapping((MappingNode)node); - map.put("rel", new Float(((Double)map.get("rel")).floatValue())); + @Override + public Object construct(Node node) { + Map map = constructMapping((MappingNode)node); + Object relObj = map.get("rel"); + float rel; + if (relObj instanceof Double) { // FIXME + rel = ((Double) relObj).floatValue(); + } else { + rel = Float.parseFloat((String) relObj); + } + map.put("rel", rel); + + // FIXME + Object posObj = map.get("positions"); + if (posObj != null) { + if ("null".equals(posObj)) { + map.put("positions", null); + } else { + Set pos = new SortedIntSet(); + for (Object p : (Set) posObj) { + if (p instanceof String) { + pos.add("null".equals(p) ? null : Integer.parseInt((String) p)); + } else { + pos.add((Integer) p); + } + } + map.put("positions", pos); + } + } + + // FIXME + Object posFragmentsObj = map.get("posFragments"); + if (posFragmentsObj != null) { + if ("null".equals(posFragmentsObj)) { + map.put("posFragments", null); + } else { + Map frags = new HashMap<>(); + for (Map.Entry entry : ((Map) posFragmentsObj).entrySet()) { + Integer key; + if (entry.getKey() instanceof String) { + key = "null".equals(entry.getKey()) ? null : Integer.parseInt((String) entry.getKey()); + } else { + key = (Integer) entry.getKey(); + } + frags.put(key, "null".equals(entry.getValue()) ? null : (String) entry.getValue()); + } + map.put("posFragments", frags); + } + } + try { return blueprint.objectFromMap(map); - } catch (Exception e) { - //java.lang.InstantiationException - //java.lang.IllegalAccessException - //java.lang.reflect.InvocationTargetException - throw new ConstructorException("while constructing a " + blueprint.getObjectClass().getSimpleName(), node.getStartMark(), "could not instantiate map " + map, null, e); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + throw new ConstructorException("while constructing a " + blueprint.getObjectClass().getSimpleName(), + node.getStartMark(), "could not instantiate map " + map, null, e); } } - } } @@ -221,6 +276,4 @@ public ConstructorException(String context, Mark contextMark, String problem, Ma super(context, contextMark, problem, problemMark, cause); } } - - } diff --git a/src/plugins/Library/io/serial/Archiver.java b/src/plugins/Library/io/serial/Archiver.java index 62ddd136..7e08b0ac 100644 --- a/src/plugins/Library/io/serial/Archiver.java +++ b/src/plugins/Library/io/serial/Archiver.java @@ -3,12 +3,8 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.io.serial; -import plugins.Library.io.serial.Serialiser.*; import plugins.Library.util.exec.TaskAbortException; -import java.util.Collection; -import java.util.Map; - /** ** An interface that handles a single {@link Serialiser.Task}. ** @@ -31,7 +27,7 @@ public interface Archiver extends Serialiser { ** ** @param task The task to execute */ - public void pull(PullTask task) throws TaskAbortException; + void pull(PullTask task) throws TaskAbortException; /** ** Execute a {@link PushTask}, returning only when the task is done. @@ -49,6 +45,5 @@ public interface Archiver extends Serialiser { ** ** @param task The task to execute */ - public void push(PushTask task) throws TaskAbortException; - + void push(PushTask task) throws TaskAbortException; } diff --git a/src/plugins/Library/io/serial/FileArchiver.java b/src/plugins/Library/io/serial/FileArchiver.java index 34290471..0adfbf7f 100644 --- a/src/plugins/Library/io/serial/FileArchiver.java +++ b/src/plugins/Library/io/serial/FileArchiver.java @@ -9,12 +9,12 @@ import java.io.IOException; import java.nio.channels.FileLock; -import plugins.Library.io.ObjectStreamReader; -import plugins.Library.io.ObjectStreamWriter; -import plugins.Library.io.serial.Serialiser.Task; import plugins.Library.util.exec.SimpleProgress; import plugins.Library.util.exec.TaskAbortException; +import plugins.Library.io.ObjectStreamReader; +import plugins.Library.io.ObjectStreamWriter; + /** ** Converts between a map of {@link String} to {@link Object}, and a file on ** disk. An {@link ObjectStreamReader} and an {@link ObjectStreamWriter} is @@ -132,7 +132,8 @@ protected File getFile(Object meta) { public interface LiveArchiver ========================================================================*/ - /*@Override**/ public void pull(PullTask t) throws TaskAbortException { + @Override + public void pull(PullTask t) throws TaskAbortException { File file = getFile(t.meta); try { FileInputStream is = new FileInputStream(file); @@ -153,7 +154,8 @@ public interface LiveArchiver } } - /*@Override**/ public void push(PushTask t) throws TaskAbortException { + @Override + public void push(PushTask t) throws TaskAbortException { if (random) { t.meta = java.util.UUID.randomUUID().toString(); } File file = getFile(t.meta); try { @@ -175,7 +177,8 @@ public interface LiveArchiver } } - /*@Override**/ public void pullLive(PullTask t, SimpleProgress p) throws TaskAbortException { + @Override + public void pullLive(PullTask t, SimpleProgress p) throws TaskAbortException { try { pull(t); if (testmode) { randomWait(p); } @@ -185,7 +188,8 @@ public interface LiveArchiver } } - /*@Override**/ public void pushLive(PushTask t, SimpleProgress p) throws TaskAbortException { + @Override + public void pushLive(PushTask t, SimpleProgress p) throws TaskAbortException { try { push(t); if (testmode) { randomWait(p); } @@ -195,4 +199,8 @@ public interface LiveArchiver } } + @Override + public void waitForAsyncInserts() { + } + } diff --git a/src/plugins/Library/io/serial/IterableSerialiser.java b/src/plugins/Library/io/serial/IterableSerialiser.java index ee457833..15418b46 100644 --- a/src/plugins/Library/io/serial/IterableSerialiser.java +++ b/src/plugins/Library/io/serial/IterableSerialiser.java @@ -3,7 +3,6 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.io.serial; -import plugins.Library.io.serial.Serialiser.*; import plugins.Library.util.exec.TaskAbortException; /** @@ -19,7 +18,7 @@ public interface IterableSerialiser extends Archiver { ** ** @param tasks The group of tasks to execute */ - public void pull(Iterable> tasks) throws TaskAbortException; + void pull(Iterable> tasks) throws TaskAbortException; // FIXME OPT NORM // Split up pull() into startPull and endPull. startPull would do the actual @@ -35,7 +34,7 @@ public interface IterableSerialiser extends Archiver { ** ** @param tasks The group of tasks to execute */ - public void push(Iterable> tasks) throws TaskAbortException; + void push(Iterable> tasks) throws TaskAbortException; // FIXME OPT HIGH // Split up push() into startPush and endPush. startPush will return as soon diff --git a/src/plugins/Library/io/serial/LiveArchiver.java b/src/plugins/Library/io/serial/LiveArchiver.java index 85d03422..54e3c944 100644 --- a/src/plugins/Library/io/serial/LiveArchiver.java +++ b/src/plugins/Library/io/serial/LiveArchiver.java @@ -3,7 +3,6 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.io.serial; -import plugins.Library.io.serial.Serialiser.*; import plugins.Library.util.exec.Progress; import plugins.Library.util.exec.TaskAbortException; @@ -27,7 +26,7 @@ public interface LiveArchiver extends Archiver { ** ** '''If this does not occur, deadlock will result'''. */ - public void pullLive(PullTask task, P p) throws TaskAbortException; + void pullLive(PullTask task, P p) throws TaskAbortException; /** ** Executes a {@link PushTask} and update the progress associated with it. @@ -41,6 +40,7 @@ public interface LiveArchiver extends Archiver { ** ** '''If this does not occur, deadlock will result'''. */ - public void pushLive(PushTask task, P p) throws TaskAbortException; + void pushLive(PushTask task, P p) throws TaskAbortException; + void waitForAsyncInserts() throws TaskAbortException; } diff --git a/src/plugins/Library/io/serial/MapSerialiser.java b/src/plugins/Library/io/serial/MapSerialiser.java index 89a65888..f0e5bba9 100644 --- a/src/plugins/Library/io/serial/MapSerialiser.java +++ b/src/plugins/Library/io/serial/MapSerialiser.java @@ -3,11 +3,10 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.io.serial; -import plugins.Library.io.serial.Serialiser.*; -import plugins.Library.util.exec.TaskAbortException; - import java.util.Map; +import plugins.Library.util.exec.TaskAbortException; + /** ** An interface that handles a map of {@link Serialiser.Task}s. As well as the ** metadata associated with each individual task, each map also has an @@ -31,7 +30,7 @@ public interface MapSerialiser extends Serialiser { ** @param tasks The map of tasks to execute ** @param mapmeta The map-wide metadata */ - public void pull(Map> tasks, Object mapmeta) throws TaskAbortException; + void pull(Map> tasks, Object mapmeta) throws TaskAbortException; /** ** Execute everything in a map of {@link PushTask}s, returning only when @@ -45,6 +44,5 @@ public interface MapSerialiser extends Serialiser { ** @param tasks The map of tasks to execute ** @param mapmeta The map-wide metadata */ - public void push(Map> tasks, Object mapmeta) throws TaskAbortException; - + void push(Map> tasks, Object mapmeta) throws TaskAbortException; } diff --git a/src/plugins/Library/io/serial/Packer.java b/src/plugins/Library/io/serial/Packer.java index 363155ec..9a7c8802 100644 --- a/src/plugins/Library/io/serial/Packer.java +++ b/src/plugins/Library/io/serial/Packer.java @@ -3,16 +3,10 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.io.serial; -import plugins.Library.io.serial.Serialiser.*; -import plugins.Library.util.IdentityComparator; -import plugins.Library.util.concurrent.ObjectProcessor; -import plugins.Library.util.exec.TaskAbortException; -import plugins.Library.util.exec.TaskCompleteException; import java.util.Collections; import java.util.Collection; import java.util.Iterator; -import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.Map; @@ -23,6 +17,10 @@ import java.util.TreeSet; import freenet.support.Logger; +import plugins.Library.io.serial.Serialiser.*; +import plugins.Library.util.IdentityComparator; +import plugins.Library.util.exec.TaskAbortException; +import plugins.Library.util.exec.TaskCompleteException; /** ** {@link MapSerialiser} that packs a map of weighable elements (eg. objects @@ -71,9 +69,9 @@ public class Packer private static volatile boolean logMINOR; private static volatile boolean logDEBUG; - static { - Logger.registerClass(Packer.class); - } + // static { + // Logger.registerClass(Packer.class); + // } /** ** Maximum weight of a bin (except one; see {@link #push(Map, Object)} for @@ -551,7 +549,11 @@ protected void pullUnloaded(Map> tasks, Object meta) throws TaskA PullTask> bintask = en.getValue(); for (Map.Entry el: bintask.data.entrySet()) { if (tasks.containsKey(el.getKey())) { - throw new TaskAbortException("Packer found an extra unexpected element (" + el.getKey() + ") inside a bin (" + en.getKey() + "). Either the data is corrupt, or the child serialiser is buggy.", new Exception("internal error")); + Logger.error(this, "Skip '" + el.getKey() + "' don't know why"); + continue; +// throw new TaskAbortException("Packer found an extra unexpected element (" + el.getKey() + ") " + +// "inside a bin (" + en.getKey() + "). Either the data is corrupt, " + +// "or the child serialiser is buggy.", new Exception("internal error")); } PullTask task = new PullTask(scale.makeMeta(en.getKey(), scale.weigh(el.getValue()))); task.data = el.getValue(); @@ -604,7 +606,7 @@ protected void pullUnloaded(Map> tasks, Object meta) throws TaskA try { // read local copy of aggression int agg = getAggression(); - if(logDEBUG) Logger.debug(this, "Aggression = "+agg+" tasks size = "+tasks.size()); + // if(logDEBUG) Logger.debug(this, "Aggression = "+agg+" tasks size = "+tasks.size()); IDGenerator gen = generator(); Inventory inv = new Inventory(this, tasks); diff --git a/src/plugins/Library/io/serial/ParallelSerialiser.java b/src/plugins/Library/io/serial/ParallelSerialiser.java index f59cd10a..119f1425 100644 --- a/src/plugins/Library/io/serial/ParallelSerialiser.java +++ b/src/plugins/Library/io/serial/ParallelSerialiser.java @@ -3,17 +3,7 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.io.serial; -import plugins.Library.io.serial.Serialiser.*; -import plugins.Library.util.TaskAbortExceptionConvertor; -import plugins.Library.util.concurrent.Scheduler; -import plugins.Library.util.concurrent.ObjectProcessor; -import plugins.Library.util.concurrent.Executors; -import plugins.Library.util.exec.Progress; -import plugins.Library.util.exec.TaskAbortException; -import plugins.Library.util.exec.TaskInProgressException; -import plugins.Library.util.exec.TaskCompleteException; -import plugins.Library.util.func.SafeClosure; -import static plugins.Library.util.func.Tuples.X2; // also imports the class +import static plugins.Library.util.func.Tuples.X2; import java.util.Iterator; import java.util.List; @@ -27,6 +17,17 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.ConcurrentMap; +import plugins.Library.io.serial.Serialiser.*; +import plugins.Library.util.TaskAbortExceptionConvertor; +import plugins.Library.util.concurrent.Executors; +import plugins.Library.util.concurrent.ObjectProcessor; +import plugins.Library.util.concurrent.Scheduler; +import plugins.Library.util.exec.Progress; +import plugins.Library.util.exec.TaskAbortException; +import plugins.Library.util.exec.TaskCompleteException; +import plugins.Library.util.exec.TaskInProgressException; +import plugins.Library.util.func.SafeClosure; + /** ** An {@link IterableSerialiser} that uses threads to handle tasks given to it ** in parallel, and keeps track of task progress. diff --git a/src/plugins/Library/io/serial/ProgressTracker.java b/src/plugins/Library/io/serial/ProgressTracker.java index 8769193c..0c1b65af 100644 --- a/src/plugins/Library/io/serial/ProgressTracker.java +++ b/src/plugins/Library/io/serial/ProgressTracker.java @@ -3,15 +3,16 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.io.serial; -import plugins.Library.io.serial.Serialiser.*; -import plugins.Library.util.exec.Progress; -import plugins.Library.util.exec.TaskInProgressException; -import plugins.Library.util.CompositeIterable; import java.util.Iterator; import java.util.Map; import java.util.WeakHashMap; +import plugins.Library.io.serial.Serialiser.*; +import plugins.Library.util.CompositeIterable; +import plugins.Library.util.exec.Progress; +import plugins.Library.util.exec.TaskInProgressException; + /** ** Keeps track of a task's progress and provides methods to retrieve this data. ** For this to function properly, the data/meta for push/pull tasks MUST NOT diff --git a/src/plugins/Library/io/serial/ScheduledSerialiser.java b/src/plugins/Library/io/serial/ScheduledSerialiser.java index 09020c8d..949d08fe 100644 --- a/src/plugins/Library/io/serial/ScheduledSerialiser.java +++ b/src/plugins/Library/io/serial/ScheduledSerialiser.java @@ -3,15 +3,13 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.io.serial; -import plugins.Library.io.serial.Serialiser.*; -import plugins.Library.util.concurrent.Scheduler; +import java.util.Map; +import java.util.concurrent.BlockingQueue; + import plugins.Library.util.concurrent.ObjectProcessor; import plugins.Library.util.exec.TaskAbortException; import plugins.Library.util.func.Tuples.X2; -import java.util.Map; -import java.util.concurrent.BlockingQueue; - /** ** An interface for asynchronous task execution. The methods return objects ** for managing and scheduling tasks. @@ -29,11 +27,11 @@ public interface ScheduledSerialiser extends IterableSerialiser { ** @param deposit Map of tasks to deposits */ // TODO LOW public Scheduler pullSchedule( - public ObjectProcessor, E, TaskAbortException> pullSchedule( - BlockingQueue> input, - BlockingQueue, TaskAbortException>> output, - Map, E> deposit - ); + ObjectProcessor, E, TaskAbortException> pullSchedule( + BlockingQueue> input, + BlockingQueue, TaskAbortException>> output, + Map, E> deposit + ); /** ** Creates a {@link ObjectProcessor} for executing {@link PushTask}s, which @@ -44,10 +42,9 @@ public ObjectProcessor, E, TaskAbortException> pullSchedule( ** @param deposit Map of tasks to deposits */ // TODO LOW public Scheduler pushSchedule( - public ObjectProcessor, E, TaskAbortException> pushSchedule( - BlockingQueue> input, - BlockingQueue, TaskAbortException>> output, - Map, E> deposit - ); - + ObjectProcessor, E, TaskAbortException> pushSchedule( + BlockingQueue> input, + BlockingQueue, TaskAbortException>> output, + Map, E> deposit + ); } diff --git a/src/plugins/Library/io/serial/Serialiser.java b/src/plugins/Library/io/serial/Serialiser.java index 736d41cf..7872d19c 100644 --- a/src/plugins/Library/io/serial/Serialiser.java +++ b/src/plugins/Library/io/serial/Serialiser.java @@ -3,11 +3,12 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.io.serial; -import plugins.Library.util.exec.Progress; import java.util.Collection; import java.util.Map; +import plugins.Library.util.exec.Progress; + /** ** An empty marker interface for serialisation classes. It defines some nested ** subclasses that acts as a unified interface between these classes. @@ -34,7 +35,7 @@ public interface Serialiser { ** Defines a serialisation task for an object. Contains two fields - data ** and metadata. */ - abstract public static class Task { + abstract class Task { /** ** Field for the metadata. This should be serialisable as defined in the @@ -46,7 +47,6 @@ abstract public static class Task { ** Field for the data. */ public T data = null; - } /************************************************************************ @@ -62,7 +62,7 @@ abstract public static class Task { ** Consequently, {@code Serialiser}s should be implemented so that its ** push operations generate metadata for which this property holds. */ - final public static class PullTask extends Task { + final class PullTask extends Task { public PullTask(Object m) { if (m == null) { throw new IllegalArgumentException("Cowardly refusing to make a PullTask with null metadata."); } @@ -77,7 +77,6 @@ public PullTask(Object m) { @Override public int hashCode() { return System.identityHashCode(meta); } - } /************************************************************************ @@ -93,7 +92,7 @@ public PullTask(Object m) { ** Consequently, {@code Serialiser}s may require that extra data is passed ** to its push operations such that it can... */ - final public static class PushTask extends Task { + final class PushTask extends Task { public PushTask(T d) { if (d == null) { throw new IllegalArgumentException("Cowardly refusing to make a PushTask with null data."); } @@ -115,29 +114,26 @@ public PushTask(T d, Object m) { @Override public int hashCode() { return System.identityHashCode(data); } - } /** ** Represents a serialiser that exposes the progress status of its ** running operations via a {@link ProgressTracker}. */ - public interface Trackable extends Serialiser { - - public ProgressTracker getTracker(); + interface Trackable extends Serialiser { + ProgressTracker getTracker(); } /** ** Represents a serialiser which uses a {@link Translator} to do much of - ** its work. This can be used alongside a {@link Serialiser.Composite}. + ** its work. This can be used alongside a {@link Composite}. ** ** TODO LOW rename this to something better... */ - public interface Translate extends Serialiser { - - public Translator getTranslator(); + interface Translate extends Serialiser { + Translator getTranslator(); } /** @@ -147,15 +143,13 @@ public interface Translate extends Serialiser { ** Serialiser} (which may also be composite). ** ** The conversion can be handled by a {@link Translator}, in which case the - ** class might also implement {@link Serialiser.Translate}. + ** class might also implement {@link Translate}. ** ** TODO HIGH find a tidy way to make this extend Serialiser ** "Composite> extends Serialiser" will work... */ - public interface Composite> /*extends Serialiser*/ { - - public S getChildSerialiser(); + interface Composite> /*extends Serialiser*/ { + S getChildSerialiser(); } - } diff --git a/src/plugins/Library/io/serial/Translator.java b/src/plugins/Library/io/serial/Translator.java index a6ffd35a..8dfa15f8 100644 --- a/src/plugins/Library/io/serial/Translator.java +++ b/src/plugins/Library/io/serial/Translator.java @@ -24,9 +24,9 @@ public interface Translator { ** particularly with regards to throwing {@link IllegalArgumentException}. ** ** TODO LOW maybe this could throw {@link DataFormatException} like {@link - ** #rev(Object)}? (probably no point...) + ** #rev(Object)}? (probably no point...) */ - I app(T translatee);// throws DataFormatException; + I app(T translatee); // throws DataFormatException; /** ** Reverse the translation. @@ -40,5 +40,4 @@ public interface Translator { ** output from being constructed. */ T rev(I intermediate) throws DataFormatException; - } diff --git a/src/plugins/Library/package-info.java b/src/plugins/Library/package-info.java index fc42a0c2..99681248 100644 --- a/src/plugins/Library/package-info.java +++ b/src/plugins/Library/package-info.java @@ -10,4 +10,3 @@ ** @author infinity0 */ package plugins.Library; - diff --git a/src/plugins/Library/search/ResultSet.java b/src/plugins/Library/search/ResultSet.java index b9df2990..bab48e7f 100644 --- a/src/plugins/Library/search/ResultSet.java +++ b/src/plugins/Library/search/ResultSet.java @@ -3,13 +3,13 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.search; + import plugins.Library.index.TermEntry; import plugins.Library.index.TermIndexEntry; -import plugins.Library.index.TermTermEntry; import plugins.Library.index.TermPageEntry; +import plugins.Library.index.TermTermEntry; import plugins.Library.util.exec.Execution; import plugins.Library.util.exec.TaskAbortException; - import freenet.support.Logger; import java.util.Iterator; @@ -330,7 +330,7 @@ private void phrase(Collection... collections) { } // if this termentry has any positions remaining, add it if(positions != null && positions.size() > 0) - addInternal(new TermPageEntry(subject, termPageEntry.rel, termPageEntry.page, termPageEntry.title, positions)); + addInternal(new TermPageEntry(subject, termPageEntry.rel, termPageEntry.getPage(), termPageEntry.title, positions)); } } @@ -343,7 +343,7 @@ private TermEntry convertEntry(TermEntry termEntry, float rel) { if (termEntry instanceof TermTermEntry) entry = new TermTermEntry(subject, rel, ((TermTermEntry)termEntry).term ); else if (termEntry instanceof TermPageEntry) - entry = new TermPageEntry(subject, rel, ((TermPageEntry)termEntry).page, ((TermPageEntry)termEntry).title, ((TermPageEntry)termEntry).positionsMap() ); + entry = new TermPageEntry(subject, rel, ((TermPageEntry)termEntry).getPage(), ((TermPageEntry)termEntry).title, ((TermPageEntry)termEntry).positionsMap() ); else if (termEntry instanceof TermIndexEntry) entry = new TermIndexEntry(subject, rel, ((TermIndexEntry)termEntry).index ); else @@ -373,7 +373,7 @@ private TermEntry mergeEntries(TermEntry... entries) { if(combination instanceof TermIndexEntry){ combination = new TermIndexEntry(subject, entries[0].rel, ((TermIndexEntry)combination).index); } else if(combination instanceof TermPageEntry){ - combination = new TermPageEntry(subject, entries[0].rel, ((TermPageEntry)combination).page, ((TermPageEntry)combination).title, ((TermPageEntry)combination).positionsMap()); + combination = new TermPageEntry(subject, entries[0].rel, ((TermPageEntry)combination).getPage(), ((TermPageEntry)combination).title, ((TermPageEntry)combination).positionsMap()); } else if(combination instanceof TermTermEntry){ combination = new TermTermEntry(subject, entries[0].rel, ((TermTermEntry)combination).term); } else @@ -405,7 +405,7 @@ else if(pageentry1.hasPositions()){ if(pageentry2.hasPositions()) newPos.putAll(pageentry2.positionsMap()); } - return new TermPageEntry(pageentry1.subj, newRel, pageentry1.page, (pageentry1.title!=null)?pageentry1.title:pageentry2.title, newPos); + return new TermPageEntry(pageentry1.subj, newRel, pageentry1.getPage(), (pageentry1.title!=null)?pageentry1.title:pageentry2.title, newPos); } else if(entry1 instanceof TermIndexEntry){ TermIndexEntry castEntry = (TermIndexEntry) entry1; diff --git a/src/plugins/Library/search/Search.java b/src/plugins/Library/search/Search.java index f01a79a9..974469c8 100644 --- a/src/plugins/Library/search/Search.java +++ b/src/plugins/Library/search/Search.java @@ -15,9 +15,9 @@ import java.util.regex.Matcher; import plugins.Library.Library; -import plugins.Library.index.TermEntry; import plugins.Library.search.ResultSet.ResultOperation; import plugins.Library.ui.ResultNodeGenerator; +import plugins.Library.index.TermEntry; import plugins.Library.util.exec.AbstractExecution; import plugins.Library.util.exec.CompositeProgress; import plugins.Library.util.exec.Execution; diff --git a/src/plugins/Library/search/inter/IndexQuery.java b/src/plugins/Library/search/inter/IndexQuery.java index 08c819ec..d939fb69 100644 --- a/src/plugins/Library/search/inter/IndexQuery.java +++ b/src/plugins/Library/search/inter/IndexQuery.java @@ -3,12 +3,13 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.search.inter; -import plugins.Library.Index; import java.util.Collections; import java.util.Set; import java.util.HashSet; +import plugins.Library.index.Index; + /** ** DOCUMENT ** diff --git a/src/plugins/Library/search/inter/Interdex.java b/src/plugins/Library/search/inter/Interdex.java index 86226621..4e5add99 100644 --- a/src/plugins/Library/search/inter/Interdex.java +++ b/src/plugins/Library/search/inter/Interdex.java @@ -3,12 +3,12 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.search.inter; + +import freenet.keys.FreenetURI; +import plugins.Library.index.Index; import plugins.Library.index.TermEntry; import plugins.Library.index.TermIndexEntry; import plugins.Library.index.TermTermEntry; -import plugins.Library.Index; - -import freenet.keys.FreenetURI; import java.util.Set; import java.util.Map; diff --git a/src/plugins/Library/search/inter/TermResults.java b/src/plugins/Library/search/inter/TermResults.java index 2595e7f5..80c1c6ce 100644 --- a/src/plugins/Library/search/inter/TermResults.java +++ b/src/plugins/Library/search/inter/TermResults.java @@ -41,7 +41,7 @@ public class TermResults { ** Interdex#getEffectiveRelevance()} when the search is still ongoing. ** ** TODO this should be a good estimate that is always greater than the - ** actual value... + ** actual value... */ public float getCachedAveragePerceivedRelevance() { return av_pcv_rel_; @@ -51,6 +51,4 @@ public TermResults(String q, String s) { query_subject = q; subject = s; } - - } diff --git a/src/plugins/Library/ui/ConfigPageToadlet.java b/src/plugins/Library/ui/ConfigPageToadlet.java new file mode 100644 index 00000000..745af0ef --- /dev/null +++ b/src/plugins/Library/ui/ConfigPageToadlet.java @@ -0,0 +1,319 @@ +/* This code is part of Freenet. It is distributed under the GNU General + * Public License, version 2 (or at your option any later version). See + * http://www.gnu.org/ for further details of the GPL. */ +package plugins.Library.ui; + +import plugins.Library.Library; + +import freenet.client.HighLevelSimpleClient; +import freenet.clients.http.PageNode; +import freenet.clients.http.RedirectException; +import freenet.clients.http.Toadlet; +import freenet.clients.http.ToadletContext; +import freenet.clients.http.ToadletContextClosedException; +import freenet.keys.FreenetURI; +import freenet.node.NodeClientCore; +import freenet.pluginmanager.PluginRespirator; +import freenet.support.HTMLNode; +import freenet.support.MultiValueTable; +import freenet.support.api.HTTPRequest; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; + +/** + * Encapsulates the ConfigPage in a Toadlet + * @author Debora Wöpcke + */ +public class ConfigPageToadlet extends Toadlet { + static final String PATH = "/config/library/"; + private NodeClientCore core; + private final Library library; + private final PluginRespirator pr; + + public ConfigPageToadlet(HighLevelSimpleClient client, + Library library, + NodeClientCore core, + PluginRespirator pr) { + super(client); + this.core = core; + this.library = library; + this.pr = pr; + } + + @Override + public String path() { + return PATH; + } + + public String menu() { + return "FProxyToadlet.categoryBrowsing"; + } + + /** post commands */ + private static enum Commands { + /** saves selected bookmarks */ + select, + + /** adds a new index to the bookmarks in Library */ + addbookmark, + + /** deletes a bookmark from the Library, requires an integer parameter between 0 and the number of bookmarks */ + removebookmark + } + + + /** + * Class containing errors to be shown in the page. + */ + private class ConfigPageError { + String message; + + // Key error + boolean keyError; + String key; + String uri; + + /** + * Constructor for key error. + */ + ConfigPageError(String m, String k, String u) { + keyError = true; + message = m; + key = k; + uri = u; + } + } + + /** + * @param ctx + */ + private void configForm(final ToadletContext ctx, + final ConfigPageError pageError) { + PageNode p = ctx.getPageMaker().getPageNode("Configure indices (" + + Library.plugName + + ")", + ctx); + p.headNode.addChild("link", + new String[]{"rel", "href", "type"}, + new String[]{"stylesheet", + path() + "static/style.css", + "text/css"}); + + HTMLNode pageNode = p.outer; + HTMLNode contentNode = p.content; + + HTMLNode searchForm = pr.addFormChild(contentNode, path(), "searchform"); + MultiValueTable headers = new MultiValueTable(); + HTMLNode indexeslist = searchForm.addChild("ul", "class", + "index-bookmark-list", + "Select indexes"); + for (String bm : library.bookmarkKeys()) { + HTMLNode bmItem = indexeslist.addChild("li"); + bmItem.addChild("input", + new String[]{"name", + "type", + "value", + "title", + (library.selectedIndices.contains(bm) ? + "checked" : + "size" ) + }, + new String[]{"~"+bm, + "checkbox", + bm, + "Index uri : "+library.getBookmark(bm), + "1" }, + bm); + bmItem.addChild("input", + new String[]{"name", + "type", + "value", + "title", + "class" + }, + new String[]{Commands.removebookmark+bm, + "submit", + "X", + "Delete this bookmark", + "index-bookmark-delete" + }); + String bookmark = library.getBookmark(bm); + if (bookmark != null) { + try { + FreenetURI uri = new FreenetURI(bookmark); + if (uri.isUSK()) { + bmItem.addChild("#", "(" + uri.getEdition() + ")"); + } + } catch (MalformedURLException e) { + // Don't add index. + } + } + } + + indexeslist.addChild("li").addChild("input", + new String[]{"name", + "type", + "value", + "title", + "class" + }, + new String[]{Commands.select.toString(), + "submit", + "Save", + "Save selected indices", + "index-bookmark-select" + }); + + + HTMLNode bmItem = indexeslist.addChild("li"); + if (pageError != null && pageError.keyError) { + bmItem.addChild("div", + new String[]{"class"}, + new String[]{"index"}, + pageError.message); + } + + bmItem.addChild("div", + new String[]{"class"}, + new String[]{"index"}, + "Token:"); + + String keyValue = ""; + if (pageError != null && pageError.key != null) { + keyValue = pageError.key; + } + bmItem.addChild("input", + new String[]{"name", + "type", + "class", + "title", + "value", + "size", + "maxsize" + }, + new String[]{"addindexname", + "text", + "index", + "Token of the index", + keyValue, + "32", + "32" + }); + String uriValue = ""; + if (pageError != null && pageError.uri != null) { + uriValue = pageError.uri; + } + bmItem.addChild("div", + new String[]{"class"}, + new String[]{"index"}, + "Key:"); + bmItem.addChild("input", + new String[]{"name", + "type", + "class", + "title", + "value", + "size", + "maxsize" + }, + new String[]{"addindexuri", + "text", + "index", + "Key of the index", + uriValue, + "100", + "256" + }); + bmItem.addChild("input", + new String[]{"name", + "type", + "value", + "title", + "class" + }, + new String[]{Commands.addbookmark.toString(), + "submit", + "Add", + "Create this index", + "index-bookmark-add" + }); + + // write reply + try { + writeHTMLReply(ctx, 200, "OK", headers, pageNode.generate()); + } catch (ToadletContextClosedException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void handleMethodGET(URI uri, + final HTTPRequest request, + final ToadletContext ctx) { + configForm(ctx, null); + } + + public void handleMethodPOST(URI uri, + HTTPRequest request, + final ToadletContext ctx) + throws ToadletContextClosedException, IOException, RedirectException { + + boolean hasFormPassword = ctx.hasFormPassword(request); + boolean userAccess = ctx.isAllowedFullAccess(); + + PageNode p = ctx.getPageMaker().getPageNode(Library.plugName, ctx); + HTMLNode pageNode = p.outer; + MultiValueTable headers = new MultiValueTable(); + boolean locationIsSet = false; + + if(userAccess && hasFormPassword) { + for (String bm : library.bookmarkKeys()) { + if (request.isPartSet("~" + bm)) { + library.selectedIndices.add(bm); + } else { + library.selectedIndices.remove(bm); + } + } + + if (request.isPartSet(Commands.select.toString())) { + headers.put("Location", MainPage.path()); + locationIsSet = true; + } + + for (String bm : library.bookmarkKeys()) { + if (request.isPartSet(Commands.removebookmark + bm)) { + library.removeBookmark(bm); + break; + } + } + + if (request.isPartSet(Commands.addbookmark.toString())) { + String addindexname = request.getPartAsStringFailsafe("addindexname", 32).trim(); + String addindexuri = request.getPartAsStringFailsafe("addindexuri", 256).trim(); + if (addindexname.length() == 0) { + configForm(ctx, + new ConfigPageError("Incorrect Token, too short", + addindexname, + addindexuri)); + return; + } + + if (library.addBookmark(addindexname, addindexuri) == null) { + configForm(ctx, new ConfigPageError("Incorrect URI.", + addindexname, + addindexuri)); + return; + } + } + } + + if (!locationIsSet) { + headers.put("Location", path()); + } + writeHTMLReply(ctx, 303, "See complete list", headers, pageNode.generate()); + } +} diff --git a/src/plugins/Library/ui/MainPage.java b/src/plugins/Library/ui/MainPage.java index ddbb17c1..bc72a818 100644 --- a/src/plugins/Library/ui/MainPage.java +++ b/src/plugins/Library/ui/MainPage.java @@ -6,13 +6,13 @@ import plugins.Library.Library; import plugins.Library.search.InvalidSearchException; import plugins.Library.search.Search; + +import freenet.keys.FreenetURI; import plugins.Library.util.exec.ChainedProgress; import plugins.Library.util.exec.CompositeProgress; import plugins.Library.util.exec.Progress; import plugins.Library.util.exec.ProgressParts; import plugins.Library.util.exec.TaskAbortException; - -import freenet.keys.FreenetURI; import freenet.pluginmanager.PluginRespirator; import freenet.support.HTMLNode; import freenet.support.Logger; @@ -145,30 +145,10 @@ public static MainPage processPostRequest(HTTPRequest request, HTMLNode contentN // Get bookmarked index list page.indexstring = ""; - for (String bm : library.bookmarkKeys()){ + for (String bm : library.selectedIndices){ String bmid = (Library.BOOKMARK_PREFIX + bm).trim(); - //Logger.normal(this, "checking for ~" + bm + " - "+request.isPartSet("~"+bm)); - if(request.isPartSet("~"+bm)){ - page.indexstring += bmid + " "; - page.selectedBMIndexes.add(bmid); - } - } - // Get other index list - //Logger.normal(page, "extra indexes : "+request.getIntPart("extraindexcount", 0)); - for (int i = 0; i < request.getIntPart("extraindexcount", 0); i++) { - if (request.isPartSet("index"+i)){ - String otherindexuri = request.getPartAsStringFailsafe("index"+i, 256); - //Logger.normal(page, "Added index : "+otherindexuri); - page.indexstring += otherindexuri + " "; - page.selectedOtherIndexes.add(otherindexuri); - }//else - //Logger.normal(page, "other index #"+i+" unchecked"); - } - for (String string : etcIndexes) { - if(string.length()>0){ - page.indexstring += string + " "; - page.selectedOtherIndexes.add(string); - } + page.indexstring += bmid + " "; + page.selectedBMIndexes.add(bmid); } page.indexstring = page.indexstring.trim(); @@ -285,10 +265,8 @@ public void writeContent(HTMLNode contentNode, MultiValueTable h if (search.isDone()) { if(search.hasGeneratedResultNode()){ contentNode.addChild(search.getHTMLNode()); - //Logger.normal(this, "Got pre generated result node."); }else try { - //Logger.normal(this, "Blocking to generate resultnode."); ResultNodeGenerator nodegenerator = new ResultNodeGenerator(search.getResult(), groupusk, showold, true); // js is being switch on always currently due to detection being off nodegenerator.run(); contentNode.addChild(nodegenerator.getPageEntryNode()); @@ -310,8 +288,6 @@ public void writeContent(HTMLNode contentNode, MultiValueTable h // refresh will GET so use a request id if (!js) { headers.put("Refresh", "2;url=" + refreshURL); - //contentNode.addChild("script", new String[]{"type", "src"}, new String[]{"text/javascript", path() + "static/" + (js ? "script.js" : "detect.js") + "?request="+search.hashCode()+(showold?"&showold=on":"")}).addChild("%", " "); - //contentNode.addChild("script", new String[]{"type", "src"}, new String[]{"text/javascript", path() + "static/" + (js ? "script.js" : "detect.js") + "?request="+search.hashCode()+(showold?"&showold=on":"")}).addChild("%", " "); } } }catch(TaskAbortException e) { @@ -346,23 +322,26 @@ private HTMLNode searchBox(){ searchBox.addChild("br"); searchBox.addChild("input", new String[]{"name", "size", "type", "value", "title"}, new String[]{"search", "40", "text", query, "Enter a search query. You can use standard search syntax such as 'and', 'or', 'not' and \"\" double quotes around phrases"}); searchBox.addChild("input", new String[]{"name", "type", "value", "tabindex"}, new String[]{"find", "submit", "Find!", "1"}); - if(js) + + searchBox.addChild("#", "Will use indices: " + library.selectedIndices + " "); + searchBox.addChild("a", new String[]{ "href", }, new String[]{ ConfigPageToadlet.PATH, }).addChild("#", "change"); + +/* if(js) searchBox.addChild("input", new String[]{"type","name"}, new String[]{"hidden","js"}); // Shows the list of bookmarked indexes TODO show descriptions on mouseover ?? HTMLNode indexeslist = searchBox.addChild("ul", "class", "index-bookmark-list", "Select indexes"); - for (String bm : library.bookmarkKeys()){ - //Logger.normal(this, "Checking for bm="+Library.BOOKMARK_PREFIX+bm+" in \""+indexuri + " = " + selectedBMIndexes.contains(Library.BOOKMARK_PREFIX+bm)+" "+indexuri.contains(Library.BOOKMARK_PREFIX+bm)); + for (String bm : library.bookmarkKeys()) { HTMLNode bmItem = indexeslist.addChild("li"); - bmItem.addChild("input", new String[]{"name", "type", "value", "title", (selectedBMIndexes.contains(Library.BOOKMARK_PREFIX+bm) ? "checked" : "size" )}, new String[]{"~"+bm, "checkbox", Library.BOOKMARK_PREFIX+bm, "Index uri : "+library.getBookmark(bm), "1" } , bm); - bmItem.addChild("input", new String[]{"name", "type", "value", "title", "class"}, new String[]{Commands.removebookmark+bm, "submit", "X", "Delete this bookmark", "index-bookmark-delete" }); + bmItem.addChild("input", new String[]{"name", "type", "value", "title", (selectedBMIndexes.contains(Library.BOOKMARK_PREFIX+bm) ? "checked" : "size" )}, new String[]{"~"+bm, "checkbox", Library.BOOKMARK_PREFIX+bm, "Index uri : "+library.getBookmark(bm), "1" } , bm); + bmItem.addChild("input", new String[]{"name", "type", "value", "title", "class"}, new String[]{Commands.removebookmark+bm, "submit", "X", "Delete this bookmark", "index-bookmark-delete" }); } int i=0; for (String uri : selectedOtherIndexes) { HTMLNode removeItem = indexeslist.addChild("li"); String showuri; - try{ + try { showuri = (new FreenetURI(uri)).toShortString(); - }catch(MalformedURLException e){ + } catch (MalformedURLException e) { showuri = uri; } removeItem.addChild("input", new String[]{"type", "name", "value", "checked"}, new String[]{"checkbox", "index"+i, uri, "checked", } , showuri); @@ -372,7 +351,7 @@ private HTMLNode searchBox(){ indexeslist.addChild("input", new String[]{"name", "type", "value"}, new String[]{"extraindexcount", "hidden", ""+selectedOtherIndexes.size()}); indexeslist.addChild("li") .addChild("input", new String[]{"name", "type", "value", "class", "title"}, new String[]{"indexuris", "text", "", "index", "URI or path of other index(s) to search on"}); - +*/ HTMLNode optionsBox = searchForm.addChild("div", "style", "margin: 20px 0px 20px 20px; display: inline-table; text-align: left;", "Options"); HTMLNode optionsList = optionsBox.addChild("ul", "class", "options-list"); @@ -380,7 +359,7 @@ private HTMLNode searchBox(){ .addChild("input", new String[]{"name", "type", groupusk?"checked":"size", "title"}, new String[]{"groupusk", "checkbox", "1", "If set, the results are returned grouped by site and edition, this makes the results quicker to scan through but will disrupt ordering on relevance, if applicable to the indexs you are using."}, "Group sites and editions"); optionsList.addChild("li") .addChild("input", new String[]{"name", "type", showold?"checked":"size", "title"}, new String[]{"showold", "checkbox", "1", "If set, older editions are shown in the results greyed out, otherwise only the most recent are shown."}, "Show older editions"); - +/* HTMLNode newIndexInput = optionsBox.addChild("div", new String[]{"class", "style"}, new String[]{"index", "display: inline-table;"}, "Add an index:"); newIndexInput.addChild("br"); newIndexInput.addChild("div", "style", "display: inline-block; width: 50px;", "Name:"); @@ -390,6 +369,7 @@ private HTMLNode searchBox(){ newIndexInput.addChild("input", new String[]{"name", "type", "class", "title", "value"}, new String[]{"addindexuri", "text", "index", "URI or path of index to add to bookmarks, including the main index filename at the end of a Freenet uri will help Library not to block in order to discover the index type.", addindexuri}); newIndexInput.addChild("br"); newIndexInput.addChild("input", new String[]{"name", "type", "value"}, new String[]{"addbookmark", "submit", "Add Bookmark"}); + */ }else searchDiv.addChild("#", "No PluginRespirater, so Form cannot be displayed"); return searchDiv; @@ -443,7 +423,7 @@ public static void addError(HTMLNode node, Throwable error, StringBuilder messag error1.addChild("br"); error1.addChild("#", " -- "+ste.toString()); } - if(error.getCause()!=null) + if (error.getCause()!=null) addError(error1, error.getCause(), messages); } } @@ -456,17 +436,17 @@ public static void addError(HTMLNode node, Throwable error, StringBuilder messag */ public static HTMLNode progressBar(Progress progress, boolean canFail) throws TaskAbortException { synchronized (progress){ - if( progress instanceof CompositeProgress && ((CompositeProgress) progress).getSubProgress()!=null && ((CompositeProgress) progress).getSubProgress().iterator().hasNext()){ + if (progress instanceof CompositeProgress && ((CompositeProgress) progress).getSubProgress()!=null && ((CompositeProgress) progress).getSubProgress().iterator().hasNext()){ // Put together progress bars for all the subProgress HTMLNode block = new HTMLNode("#"); block.addChild("tr").addChild("td", "colspan", "6", progress.getSubject() + " : "+progress.getStatus()); TaskAbortException firstError = null; boolean anySuccess = false; - if(canFail && progress instanceof Search) { + if (canFail && progress instanceof Search) { if(!(((Search)progress).innerCanFailAndStillComplete())) canFail = false; } else canFail = false; - if(((CompositeProgress) progress).getSubProgress() != null) + if (((CompositeProgress) progress).getSubProgress() != null) for (Progress progress1 : ((CompositeProgress) progress).getSubProgress()) { try { block.addChild(progressBar(progress1, canFail)); diff --git a/src/plugins/Library/ui/RelevanceComparator.java b/src/plugins/Library/ui/RelevanceComparator.java index 179eb91c..8df657cd 100644 --- a/src/plugins/Library/ui/RelevanceComparator.java +++ b/src/plugins/Library/ui/RelevanceComparator.java @@ -8,7 +8,8 @@ import plugins.Library.util.IdentityComparator; /** - * Compares the relevance of two TermEntrys, extends IdentityComparator so that two unique entries will not return a comparison of 0 + * Compares the relevance of two TermEntrys, extends IdentityComparator + * so that two unique entries will not return a comparison of 0 * * @author MikeB */ diff --git a/src/plugins/Library/ui/ResultNodeGenerator.java b/src/plugins/Library/ui/ResultNodeGenerator.java index ac0bc942..235f2284 100644 --- a/src/plugins/Library/ui/ResultNodeGenerator.java +++ b/src/plugins/Library/ui/ResultNodeGenerator.java @@ -3,12 +3,12 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.ui; + import plugins.Library.index.TermEntry; import plugins.Library.index.TermIndexEntry; import plugins.Library.index.TermPageEntry; import plugins.Library.index.TermTermEntry; - -import freenet.keys.FreenetURI; +import plugins.Library.io.FreenetURI; import freenet.support.HTMLNode; import freenet.support.Logger; @@ -109,7 +109,7 @@ private void parseResult(){ long uskEdition = Long.MIN_VALUE; // Get the key and name FreenetURI uri; - uri = pageEntry.page; + uri = pageEntry.getPage(); // convert usk's if(uri.isSSKForUSK()){ uri = uri.uskForSSK(); @@ -117,7 +117,7 @@ private void parseResult(){ uskEdition = uri.getEdition(); } // Get the site base name, key + documentname - uskversion - sitebase = uri.setMetaString(null).setSuggestedEdition(0).toString().replaceFirst("/0", ""); + sitebase = uri.getRoot(); Logger.minor(this, sitebase); // Add site @@ -240,9 +240,9 @@ private void generatePageEntryNode(){ * @param newestVersion if set, the result is shown in full brightness, if unset the result is greyed out */ private HTMLNode termPageEntryNode(TermPageEntry entry,boolean newestVersion) { - FreenetURI uri = entry.page; + FreenetURI uri = entry.getPage(); String showtitle = entry.title; - String showurl = uri.toShortString(); + String showurl = uri.toString(); if (showtitle == null || showtitle.trim().length() == 0) { showtitle = showurl; } diff --git a/src/plugins/Library/ui/TermPageGroupEntry.java b/src/plugins/Library/ui/TermPageGroupEntry.java index fbc87ba7..93991705 100644 --- a/src/plugins/Library/ui/TermPageGroupEntry.java +++ b/src/plugins/Library/ui/TermPageGroupEntry.java @@ -12,6 +12,7 @@ import plugins.Library.index.TermEntry; import plugins.Library.index.TermPageEntry; + /** * TODO make this fit the TermEntry contract * @author MikeB diff --git a/src/plugins/Library/ui/TestInterface.java b/src/plugins/Library/ui/TestInterface.java index 86e19e79..9cd1d217 100644 --- a/src/plugins/Library/ui/TestInterface.java +++ b/src/plugins/Library/ui/TestInterface.java @@ -3,10 +3,10 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.ui; -import plugins.Library.util.exec.Execution; import plugins.Library.search.Search; import plugins.Library.*; +import plugins.Library.util.exec.Execution; import freenet.support.Logger; import java.util.HashMap; @@ -14,7 +14,7 @@ import java.io.InputStreamReader; public class TestInterface{ - static HashMap requests = new HashMap(); + static HashMap requests = new HashMap<>(); static int requestcount =0; public static void main(String[] args){ diff --git a/src/plugins/Library/ui/WebInterface.java b/src/plugins/Library/ui/WebInterface.java index 9c1c8927..28abcbdb 100644 --- a/src/plugins/Library/ui/WebInterface.java +++ b/src/plugins/Library/ui/WebInterface.java @@ -23,6 +23,7 @@ public class WebInterface { private MainPageToadlet pluginsToadlet; private MainPageToadlet mainToadlet; private StaticToadlet staticToadlet; + private ConfigPageToadlet configToadlet; /** * // @param spider @@ -47,6 +48,13 @@ public void load() { mainToadlet = new MainPageToadlet(client, library, core, pr); toadletContainer.register(mainToadlet, mainToadlet.menu(), mainToadlet.path(), true, mainToadlet.name(), mainToadlet.name(), true, null ); + configToadlet = new ConfigPageToadlet(client, library, core, pr); + toadletContainer.register(configToadlet, + configToadlet.menu(), + configToadlet.path(), + true, + false); + // Ive just realised that the form filter allows access to /plugins/... so /library/ wont be allowed, this is a temporary Toadlet untilthere is a whitelist for formfilter and /library is on it TODO put /library on formfilter whitelist pluginsToadlet = new MainPageToadlet(client, library, core, pr); toadletContainer.register(pluginsToadlet, null, "/plugins/plugin.Library.FreesiteSearch", true, null, null, true, null ); @@ -61,6 +69,7 @@ public void load() { public void unload() { toadletContainer.unregister(mainToadlet); pageMaker.removeNavigationLink(mainToadlet.menu(), mainToadlet.name()); + toadletContainer.unregister(configToadlet); toadletContainer.unregister(pluginsToadlet); toadletContainer.unregister(staticToadlet); } diff --git a/src/plugins/Library/util/BTreeMap.java b/src/plugins/Library/util/BTreeMap.java index 11b21aeb..4ac8896f 100644 --- a/src/plugins/Library/util/BTreeMap.java +++ b/src/plugins/Library/util/BTreeMap.java @@ -3,13 +3,9 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.util; -import plugins.Library.util.CompositeIterable; -import plugins.Library.util.func.Tuples.X2; -import plugins.Library.util.func.Tuples.X3; import java.util.Comparator; import java.util.Iterator; -import java.util.Collection; import java.util.List; import java.util.Set; import java.util.Map; @@ -21,7 +17,9 @@ import java.util.HashMap; import java.util.Stack; import java.util.NoSuchElementException; -import java.util.ConcurrentModificationException; + +import plugins.Library.util.func.Tuples.X2; +import plugins.Library.util.func.Tuples.X3; /** ** General purpose B-tree implementation. '''This class is not a general-use @@ -371,7 +369,7 @@ protected Node(K lk, K rk, boolean lf) { /** ** Add the given entries and subnodes to this node. The subnodes are - ** added with {@link #addChildNode(BTreeMap.Node)}. + ** added with {@link #addChildNode(Node)}. ** ** It is '''assumed''' that the entries are located exactly exclusively ** between the nodes (ie. like in an actual node); it is up to the @@ -796,12 +794,12 @@ public String toTreeString(String istr) { StringBuilder s = new StringBuilder(); s.append(istr).append('(').append(lkey).append(')').append('\n'); if (isLeaf) { - for (Map.Entry en: entries.entrySet()) { + for (Entry en: entries.entrySet()) { s.append(istr).append(en.getKey()).append(" : ").append(en.getValue()).append('\n'); } } else { s.append(rnodes.get(lkey).toTreeString(nistr)); - for (Map.Entry en: entries.entrySet()) { + for (Entry en: entries.entrySet()) { s.append(istr).append(en.getKey()).append(" : ").append(en.getValue()).append('\n'); s.append(rnodes.get(en.getKey()).toTreeString(nistr)); } @@ -953,10 +951,6 @@ final void verifyNodeIntegrity(Node node) { verify(node.nodeSize() + 1 == node.rnodes.size()); verify(node.nodeSize() + 1 == node.lnodes.size()); } - /* DEBUG if (node._size > 0 && node._size != s) { - System.out.println(node._size + " vs " + s); - System.out.println(node.toTreeString("\t")); - }*/ verify(node._size < 0 || node._size == s); verify(node.nodeSize() <= ENT_MAX); @@ -1062,14 +1056,14 @@ private K split(Node parent, Node child) { if (child.isLeaf()) { // this is just the same as the code in the else block, but with leaf // references to rnodes and lnodes removed (since they are null) - Iterator> it = child.entries.entrySet().iterator(); + Iterator> it = child.entries.entrySet().iterator(); for (int i=0; i entry = it.next(); it.remove(); + Entry entry = it.next(); it.remove(); K key = entry.getKey(); lnode.entries.put(key, entry.getValue()); } - Map.Entry median = it.next(); it.remove(); + Entry median = it.next(); it.remove(); mkey = median.getKey(); lnode.lkey = child.lkey; @@ -1083,16 +1077,16 @@ private K split(Node parent, Node child) { } else { lnode.rnodes.put(child.lkey, child.rnodes.remove(child.lkey)); - Iterator> it = child.entries.entrySet().iterator(); + Iterator> it = child.entries.entrySet().iterator(); for (int i=0; i entry = it.next(); it.remove(); + Entry entry = it.next(); it.remove(); K key = entry.getKey(); lnode.entries.put(key, entry.getValue()); lnode.lnodes.put(key, child.lnodes.remove(key)); lnode.rnodes.put(key, child.rnodes.remove(key)); } - Map.Entry median = it.next(); it.remove(); + Entry median = it.next(); it.remove(); mkey = median.getKey(); lnode.lnodes.put(mkey, child.lnodes.remove(mkey)); @@ -1334,7 +1328,7 @@ public int minKeysFor(int entries) { /** ** Returns the entry at a particular (zero-based) index. */ - public Map.Entry getEntry(int index) { + public Entry getEntry(int index) { if (index < 0 || index >= size) { throw new IndexOutOfBoundsException("Index outside of range [0," + size + ")"); } @@ -1344,7 +1338,7 @@ public Map.Entry getEntry(int index) { for (;;) { if (node.isLeaf()) { - for (Map.Entry en: node.entries.entrySet()) { + for (Entry en: node.entries.entrySet()) { if (current == index) { return en; } ++current; } @@ -1365,7 +1359,7 @@ public Map.Entry getEntry(int index) { if (index < next) { node = nextnode; continue; } current = next; - for (Map.Entry en: node.entries.entrySet()) { + for (Entry en: node.entries.entrySet()) { if (current == index) { return en; } ++current; @@ -1671,13 +1665,13 @@ public interface Map nextlnodes = new HashMap(k<<1); nextmap = new TreeMap(comparator); - Iterator> it = map.entrySet().iterator(); + Iterator> it = map.entrySet().iterator(); K prevkey = null; // NULLNOTICE // allocate all entries into these k nodes, except for k-1 parents for (Integer n: Integers.allocateEvenly(map.size()-k+1, k)) { // put n entries into a new leaf - Map.Entry en = makeNode(it, n, prevkey, lnodes, nextlnodes); + Entry en = makeNode(it, n, prevkey, lnodes, nextlnodes); if (en != null) { nextmap.put(prevkey = en.getKey(), en.getValue()); } } @@ -1703,12 +1697,12 @@ public interface Map ** @param lnodes Complete map of keys to their lnodes at this level ** @param nextlnodes In-construction lnodes map for the next level */ - private Map.Entry makeNode(Iterator> it, int n, K prevkey, Map lnodes, Map nextlnodes) { + private Entry makeNode(Iterator> it, int n, K prevkey, Map lnodes, Map nextlnodes) { Node node; if (lnodes == null) { // create leaf nodes node = newNode(prevkey, null, true); for (int i=0; i en = it.next(); + Entry en = it.next(); K key = en.getKey(); node.entries.put(key, en.getValue()); prevkey = key; @@ -1716,7 +1710,7 @@ private Map.Entry makeNode(Iterator> it, int n, K prevkey, } else { node = newNode(prevkey, null, false); for (int i=0; i en = it.next(); + Entry en = it.next(); K key = en.getKey(); node.entries.put(key, en.getValue()); Node subnode = lnodes.get(key); @@ -1726,7 +1720,7 @@ private Map.Entry makeNode(Iterator> it, int n, K prevkey, } } - Map.Entry next; + Entry next; K key; if (it.hasNext()) { next = it.next(); @@ -1755,23 +1749,23 @@ public void restructure() { putAll(this); } - private Set> entrySet = null; - @Override public Set> entrySet() { + private Set> entrySet = null; + @Override public Set> entrySet() { if (entrySet == null) { - entrySet = new AbstractSet>() { + entrySet = new AbstractSet>() { @Override public int size() { return BTreeMap.this.size(); } - @Override public Iterator> iterator() { + @Override public Iterator> iterator() { // FIXME LOW - this does NOT yet throw ConcurrentModificationException // use a modCount counter - return new Iterator>() { + return new Iterator>() { Stack nodestack = new Stack(); - Stack>> itstack = new Stack>>(); + Stack>> itstack = new Stack>>(); Node cnode = BTreeMap.this.root; - Iterator> centit = cnode.entries.entrySet().iterator(); + Iterator> centit = cnode.entries.entrySet().iterator(); K lastkey = null; boolean removeok = false; @@ -1787,7 +1781,7 @@ public void restructure() { /*@Override**/ public boolean hasNext() { // TODO LOW ideally iterate in the reverse order - for (Iterator> it: itstack) { + for (Iterator> it: itstack) { if (it.hasNext()) { return true; } } if (centit.hasNext()) { return true; } @@ -1795,7 +1789,7 @@ public void restructure() { return false; } - /*@Override**/ public Map.Entry next() { + /*@Override**/ public Entry next() { if (cnode.isLeaf()) { while (!centit.hasNext()) { if (nodestack.empty()) { @@ -1818,7 +1812,7 @@ public void restructure() { centit = cnode.entries.entrySet().iterator(); } } - Map.Entry next = centit.next(); lastkey = next.getKey(); + Entry next = centit.next(); lastkey = next.getKey(); removeok = true; return next; } @@ -1872,14 +1866,14 @@ private K findstopkey(K stopkey) { @Override public boolean contains(Object o) { if (!(o instanceof Map.Entry)) { return false; } - Map.Entry e = (Map.Entry)o; + Entry e = (Entry)o; Object value = BTreeMap.this.get(e.getKey()); return value != null && value.equals(e.getValue()); } @Override public boolean remove(Object o) { if (contains(o)) { - Map.Entry e = (Map.Entry)o; + Entry e = (Entry)o; BTreeMap.this.remove(e.getKey()); return true; } diff --git a/src/plugins/Library/util/BTreeSet.java b/src/plugins/Library/util/BTreeSet.java index 75e24054..6d6fc2ec 100644 --- a/src/plugins/Library/util/BTreeSet.java +++ b/src/plugins/Library/util/BTreeSet.java @@ -71,6 +71,4 @@ public int heightEstimate() { public E get(int i) { return bkmap.getEntry(i).getKey(); } - - } diff --git a/src/plugins/Library/util/BytePrefixKey.java b/src/plugins/Library/util/BytePrefixKey.java index ae0efad0..d6e8edb2 100644 --- a/src/plugins/Library/util/BytePrefixKey.java +++ b/src/plugins/Library/util/BytePrefixKey.java @@ -3,10 +3,9 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.util; -import plugins.Library.util.PrefixTree.PrefixKey; -import plugins.Library.util.PrefixTree.AbstractPrefixKey; - import java.util.Arrays; +import plugins.Library.util.PrefixTree.AbstractPrefixKey; +import plugins.Library.util.PrefixTree.PrefixKey; /** ** A PrefixKey backed by an array of bytes. diff --git a/src/plugins/Library/util/Maps.java b/src/plugins/Library/util/Maps.java index 34678982..9d0791f2 100644 --- a/src/plugins/Library/util/Maps.java +++ b/src/plugins/Library/util/Maps.java @@ -3,8 +3,6 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.util; -import static plugins.Library.util.Maps.$; - import java.util.Map.Entry; // WORKAROUND javadoc bug #4464323 import java.util.Map; @@ -18,7 +16,7 @@ final public class Maps { /** ** A simple {@link Entry} with no special properties. */ - public static class BaseEntry implements Map.Entry { + public static class BaseEntry implements Entry { final public K key; protected V val; @@ -32,8 +30,9 @@ public static class BaseEntry implements Map.Entry { @Override public boolean equals(Object o) { if (o == this) { return true; } if (!(o instanceof Map.Entry)) { return false; } - Map.Entry en = (Map.Entry)o; - return (key == null? en.getKey() == null: key.equals(en.getKey())) && (val == null? en.getValue() == null : val.equals(en.getValue())); + Entry en = (Entry)o; + return (key == null ? en.getKey() == null : key.equals(en.getKey())) && + (val == null ? en.getValue() == null : val.equals(en.getValue())); } @Override public int hashCode() { @@ -91,28 +90,28 @@ private Maps() { } /** ** Returns a new {@link BaseEntry} with the given key and value. */ - public static Map.Entry $(final K k, final V v) { + public static Entry $(final K k, final V v) { return new BaseEntry(k, v); } /** ** Returns a new {@link ImmutableEntry} with the given key and value. */ - public static Map.Entry $$(final K k, final V v) { + public static Entry $$(final K k, final V v) { return new ImmutableEntry(k, v); } /** ** Returns a new {@link KeyEntry} with the given key and value. */ - public static Map.Entry $K(K k, V v) { + public static Entry $K(K k, V v) { return new KeyEntry(k, v); } - public static Map of(Class mapcl, Map.Entry... items) { + public static Map of(Class mapcl, Entry... items) { try { Map map = mapcl.newInstance(); - for (Map.Entry en: items) { + for (Entry en: items) { map.put(en.getKey(), en.getValue()); } return map; diff --git a/src/plugins/Library/util/Skeleton.java b/src/plugins/Library/util/Skeleton.java index 652e808a..a7cdcaf5 100644 --- a/src/plugins/Library/util/Skeleton.java +++ b/src/plugins/Library/util/Skeleton.java @@ -20,39 +20,39 @@ public interface Skeleton> { /** ** Whether the skeleton is fully loaded and has no data missing. */ - public boolean isLive(); + boolean isLive(); /** ** Whether the skeleton is bare and has no data loaded at all. */ - public boolean isBare(); + boolean isBare(); /** ** Get the serialiser for this skeleton. */ - public S getSerialiser(); + S getSerialiser(); /** ** Get the meta data associated with this skeleton. */ - public Object getMeta(); + Object getMeta(); /** ** Set the meta data associated with this skeleton. */ - public void setMeta(Object m); + void setMeta(Object m); /** ** Inflate the entire skeleton so that after the method call, {@link ** #isLive()} returns true. */ - public void inflate() throws TaskAbortException; + void inflate() throws TaskAbortException; /** ** Deflate the entire skeleton so that after the method call, {@link ** #isBare()} returns true. */ - public void deflate() throws TaskAbortException; + void deflate() throws TaskAbortException; /** ** Partially inflate the skeleton based on some parameter object. This @@ -60,7 +60,7 @@ public interface Skeleton> { ** ** @param param The parameter for the partial inflate. */ - public void inflate(K param) throws TaskAbortException; + void inflate(K param) throws TaskAbortException; /** ** Partially deflate the skeleton based on some parameter object. This @@ -68,6 +68,5 @@ public interface Skeleton> { ** ** @param param The parameter for the partial deflate. */ - public void deflate(K param) throws TaskAbortException; - + void deflate(K param) throws TaskAbortException; } diff --git a/src/plugins/Library/util/SkeletonBTreeMap.java b/src/plugins/Library/util/SkeletonBTreeMap.java index 84628770..2b6a78ab 100644 --- a/src/plugins/Library/util/SkeletonBTreeMap.java +++ b/src/plugins/Library/util/SkeletonBTreeMap.java @@ -3,17 +3,6 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.util; -import plugins.Library.io.serial.Serialiser.*; -import plugins.Library.io.serial.IterableSerialiser; -import plugins.Library.io.serial.ScheduledSerialiser; -import plugins.Library.io.serial.MapSerialiser; -import plugins.Library.io.serial.Translator; -import plugins.Library.io.DataFormatException; -import plugins.Library.util.exec.TaskAbortException; -import plugins.Library.util.exec.TaskCompleteException; -import plugins.Library.util.func.Tuples.X2; -import plugins.Library.util.func.Tuples.X3; - import java.util.AbstractSet; import java.util.Comparator; import java.util.Iterator; @@ -29,22 +18,11 @@ // TODO NORM tidy this import java.util.Queue; import java.util.PriorityQueue; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.PriorityBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import plugins.Library.util.exec.Progress; -import plugins.Library.util.exec.ProgressParts; -import plugins.Library.util.exec.BaseCompositeProgress; -import plugins.Library.io.serial.Serialiser; -import plugins.Library.io.serial.ProgressTracker; -import plugins.Library.util.exec.TaskCompleteException; -import plugins.Library.util.BTreeMap.Node; -import plugins.Library.util.concurrent.Scheduler; import java.util.Collections; import java.util.SortedSet; @@ -53,17 +31,26 @@ import java.util.TreeMap; import java.util.HashMap; -import freenet.support.Logger; -import plugins.Library.util.Sorted; +import plugins.Library.io.DataFormatException; +import plugins.Library.io.serial.IterableSerialiser; +import plugins.Library.io.serial.MapSerialiser; +import plugins.Library.io.serial.ProgressTracker; +import plugins.Library.io.serial.ScheduledSerialiser; +import plugins.Library.io.serial.Serialiser; +import plugins.Library.io.serial.Translator; +import plugins.Library.io.serial.Serialiser.*; import plugins.Library.util.concurrent.BoundedPriorityBlockingQueue; import plugins.Library.util.concurrent.ExceptionConvertor; +import plugins.Library.util.concurrent.Executors; import plugins.Library.util.concurrent.Notifier; import plugins.Library.util.concurrent.ObjectProcessor; -import plugins.Library.util.concurrent.Executors; -import plugins.Library.util.event.TrackingSweeper; import plugins.Library.util.event.CountingSweeper; +import plugins.Library.util.event.TrackingSweeper; +import plugins.Library.util.exec.*; import plugins.Library.util.func.Closure; import plugins.Library.util.func.SafeClosure; +import plugins.Library.util.func.Tuples.X2; +import plugins.Library.util.func.Tuples.X3; import static plugins.Library.util.Maps.$K; /** @@ -150,9 +137,9 @@ public void setSerialiser(IterableSerialiser n, MapSerialiser> CMP_PULL = new Comparator>() { /*@Override**/ public int compare(PullTask t1, PullTask t2) { @@ -166,8 +153,8 @@ public void setSerialiser(IterableSerialiser n, MapSerialiser> CMP_ENTRY = new Comparator>() { - /*@Override**/ public int compare(Map.Entry t1, Map.Entry t2) { + final public Comparator> CMP_ENTRY = new Comparator>() { + /*@Override**/ public int compare(Entry t1, Entry t2) { return SkeletonBTreeMap.this.compare(t1.getKey(), t2.getKey()); } }; @@ -562,7 +549,7 @@ public interface SkeletonMap final Queue nodequeue = new PriorityQueue(); - Map, ProgressTracker> ids = null; + Map, ProgressTracker> ids = null; ProgressTracker ntracker = null;; if (nsrl instanceof Serialiser.Trackable) { @@ -579,8 +566,6 @@ public interface SkeletonMap new LinkedBlockingQueue, TaskAbortException>>(0x10), new HashMap, SkeletonNode>() ); - //System.out.println("Using scheduler"); - //int DEBUG_pushed = 0, DEBUG_popped = 0; try { nodequeue.add((SkeletonNode)root); @@ -589,8 +574,6 @@ public interface SkeletonMap // operation fails do { - //System.out.println("pushed: " + DEBUG_pushed + "; popped: " + DEBUG_popped); - // handle the inflated tasks and attach them to the tree. // THREAD progress tracker should prevent this from being run twice for the // same node, but what if we didn't use a progress tracker? hmm... @@ -654,8 +637,6 @@ public interface SkeletonMap throw new TaskAbortException("interrupted", e); } finally { proc_pull.close(); - //System.out.println("pushed: " + DEBUG_pushed + "; popped: " + DEBUG_popped); - //assert(DEBUG_pushed == DEBUG_popped); } } @@ -738,7 +719,7 @@ public void update(SortedMap putmap, SortedSet remkey) throws TaskAbort ** @param value_handler Closure to retrieve the value for each putkey ** @see #update(SortedSet, SortedSet, SortedMap, Closure) */ - public void update(SortedSet putkey, SortedSet remkey, Closure, X> value_handler, ExceptionConvertor conv) throws TaskAbortException { + public void update(SortedSet putkey, SortedSet remkey, Closure, X> value_handler, ExceptionConvertor conv) throws TaskAbortException { update(putkey, remkey, null, value_handler, conv); } @@ -812,7 +793,7 @@ public void update(SortedSet putkey, SortedSet remke */ protected void update( SortedSet putkey, SortedSet remkey, - final SortedMap putmap, Closure, X> value_handler, + final SortedMap putmap, Closure, X> value_handler, ExceptionConvertor conv ) throws TaskAbortException { @@ -851,7 +832,7 @@ protected void update( protected void update( SortedSet putkey, - final SortedMap putmap, Closure, X> value_handler, + final SortedMap putmap, Closure, X> value_handler, ExceptionConvertor conv, final SortedSet rejected ) throws TaskAbortException { @@ -931,7 +912,7 @@ protected void update( /** ** Deposit for a value-retrieval operation */ - class DeflateNode extends TrackingSweeper> implements Runnable, SafeClosure> { + class DeflateNode extends TrackingSweeper> implements Runnable, SafeClosure> { /** The node to be pushed, when run() is called. */ final SkeletonNode node; @@ -971,10 +952,10 @@ public void run() { ** Update the key's value in the node. Runs whenever an entry is ** popped from proc_val. */ - public void invoke(Map.Entry en) { + public void invoke(Entry en) { assert(node.entries.containsKey(en.getKey())); node.entries.put(en.getKey(), en.getValue()); - if(logMINOR) Logger.minor(this, "New value for key "+en.getKey()+" : "+en.getValue()+" in "+node+" parent = "+parNClo); + // if(logMINOR) Logger.minor(this, "New value for key "+en.getKey()+" : "+en.getValue()+" in "+node+" parent = "+parNClo); } public void deflate() throws TaskAbortException { @@ -991,11 +972,11 @@ public void deflate() throws TaskAbortException { // The proc_val output queue must comfortably cover everything generated by a single invocation of InflateChildNodes, or else we will deadlock. // Note that this is all deflated sub-trees, so they are relatively small - some of the other queues // handle much larger structures in their counters. - final ObjectProcessor, DeflateNode, X> proc_val = (value_handler == null)? null - : new ObjectProcessor, DeflateNode, X>( - new BoundedPriorityBlockingQueue>(0x10, CMP_ENTRY), - new LinkedBlockingQueue, X>>(ENT_MAX*3), - new HashMap, DeflateNode>(), + final ObjectProcessor, DeflateNode, X> proc_val = (value_handler == null)? null + : new ObjectProcessor, DeflateNode, X>( + new BoundedPriorityBlockingQueue>(0x10, CMP_ENTRY), + new LinkedBlockingQueue, X>>(ENT_MAX*3), + new HashMap, DeflateNode>(), value_handler, VALUE_EXECUTOR, conv, notifier // These can block so pool them separately. ).autostart(); @@ -1111,7 +1092,6 @@ public void run() { reassignKeyToSweeper(key, parVClo); } - //System.out.println("parent:"+parent.getRange()+"\nseps:"+keys+"\nheld:"+held); parNClo.open(); // for each split-node, create a sweeper that will run when all its (k,v) @@ -1122,12 +1102,7 @@ public void run() { // reassign appropriate keys to the split-node's sweeper SortedSet subheld = subSet(held, n.lkey, n.rkey); - //try { assert(subheld.isEmpty() || compareL(n.lkey, subheld.first()) < 0 && compareR(subheld.last(), n.rkey) < 0); - //} catch (AssertionError e) { - // System.out.println(n.lkey + " " + subheld.first() + " " + subheld.last() + " " + n.rkey); - // throw e; - //} for (K key: subheld) { reassignKeyToSweeper(key, vClo); } @@ -1168,7 +1143,7 @@ private void reassignKeyToSweeper(K key, DeflateNode clo) { //assert(((UpdateValue)value_closures.get(key)).node == node); proc_val.update($K(key, (V)null), clo); // FIXME what if it has already run??? - if(logMINOR) Logger.minor(this, "Reassigning key "+key+" to "+clo+" on "+this+" parent="+parent+" parent split node "+parNClo+" parent deflate node "+parVClo); + // if(logMINOR) Logger.minor(this, "Reassigning key "+key+" to "+clo+" on "+this+" parent="+parent+" parent split node "+parNClo+" parent deflate node "+parVClo); // nodeVClo.release(key); // this is unnecessary since nodeVClo() will only be used if we did not // split its node (and never called this method) @@ -1276,7 +1251,7 @@ public void invoke(SkeletonNode node) { } catch (ClassCastException e) { // This has been seen in practice. I have no idea what it means. // FIXME HIGH !!! - Logger.error(this, "Node is already loaded?!?!?!: "+node.selectNode(rng.first())); + // Logger.error(this, "Node is already loaded?!?!?!: "+node.selectNode(rng.first())); continue; } PullTask task = new PullTask(n); @@ -1308,7 +1283,7 @@ public void invoke(SkeletonNode node) { private void handleLocalPut(SkeletonNode n, K key, DeflateNode vClo) { V oldval = n.entries.put(key, null); vClo.acquire(key); - if(logMINOR) Logger.minor(this, "handleLocalPut for key "+key+" old value "+oldval+" for deflate node "+vClo+" - passing to proc_val"); + // if(logMINOR) Logger.minor(this, "handleLocalPut for key "+key+" old value "+oldval+" for deflate node "+vClo+" - passing to proc_val"); ObjectProcessor.submitSafe(proc_val, $K(key, oldval), vClo); } @@ -1334,18 +1309,16 @@ private void handleLocalRemove(SkeletonNode n, K key, TrackingSweeper 10)) { count = 0; -// if(ccount++ > 10) { - System.out.println(/*System.identityHashCode(this) + " " + */proc_val + " " + proc_pull + " " + proc_push+ " "+proc_deflate); -// ccount = 0; -// } + // Logger.debug(this, + // "SkeletonBTreeMap update " + + // proc_val + " " + + // proc_pull + " " + + // proc_push + " " + + // proc_deflate); notifier.waitUpdate(1000); } progress = false; @@ -1369,8 +1342,6 @@ private void handleLocalRemove(SkeletonNode n, K key, TrackingSweeper res = proc_deflate.accept(); DeflateNode sw = res._0; @@ -1385,8 +1356,8 @@ private void handleLocalRemove(SkeletonNode n, K key, TrackingSweeper, DeflateNode, X> res = proc_val.accept(); - Map.Entry en = res._0; + X3, DeflateNode, X> res = proc_val.accept(); + Entry en = res._0; DeflateNode sw = res._1; X ex = res._2; if (ex != null) { @@ -1508,17 +1479,35 @@ public NodeTranslator(Translator k, Translator, R> m boolean notleaf = map.containsKey("subnodes"); List gh = null; if (notleaf) { - Map subnodes = (Map)map.get("subnodes"); - gh = new ArrayList(subnodes.size()); - for (Map.Entry en: subnodes.entrySet()) { - GhostNode ghost = new GhostNode(null, null, null, en.getValue()); + Map subnodes = (Map) map.get("subnodes"); + gh = new ArrayList<>(subnodes.size()); + for (Entry en: subnodes.entrySet()) { + Object sObj = en.getValue(); + int s; + if (sObj instanceof String) { // FIXME + s = Integer.parseInt((String) sObj); + } else { + s = (Integer) sObj; + } + GhostNode ghost = new GhostNode(null, null, null, s); ghost.setMeta(en.getKey()); gh.add(ghost); } } + + // FIXME + Object lkey = map.get("lkey"); + if ("null".equals(lkey)) { + lkey = null; + } + Object rkey = map.get("rkey"); + if ("null".equals(rkey)) { + rkey = null; + } + SkeletonNode node = new SkeletonNode( - (ktr == null)? (K)map.get("lkey"): ktr.rev((Q)map.get("lkey")), - (ktr == null)? (K)map.get("rkey"): ktr.rev((Q)map.get("rkey")), + (ktr == null) ? (K) lkey : ktr.rev((Q) lkey), + (ktr == null) ? (K) rkey : ktr.rev((Q) rkey), !notleaf, (mtr == null)? (SkeletonTreeMap)map.get("entries") : mtr.rev((R)map.get("entries")), @@ -1575,8 +1564,20 @@ public TreeTranslator(Translator k, Translator, ?> m /*@Override**/ public SkeletonBTreeMap rev(Map map) throws DataFormatException { try { - SkeletonBTreeMap tree = new SkeletonBTreeMap((Integer)map.get("node_min")); - tree.size = (Integer)map.get("size"); + Object nodeMinObj = map.get("node_min"); + int nodeMin; + if (nodeMinObj instanceof String) { // FIXME + nodeMin = Integer.parseInt((String) nodeMinObj); + } else { + nodeMin = (Integer) map.get("node_min"); + } + SkeletonBTreeMap tree = new SkeletonBTreeMap<>(nodeMin); + Object sizeObj = map.get("size"); + if (sizeObj instanceof String) { + tree.size = Integer.parseInt((String) sizeObj); + } else { + tree.size = (Integer) sizeObj; + } // map.put("lkey", null); // NULLNOTICE: get() gives null which matches // map.put("rkey", null); // NULLNOTICE: get() gives null which matches tree.root = tree.makeNodeTranslator(ktr, mtr).rev(map); @@ -1616,10 +1617,10 @@ public Set keySetAutoDeflate() { return new Iterator() { Stack nodestack = new Stack(); - Stack>> itstack = new Stack>>(); + Stack>> itstack = new Stack>>(); SkeletonNode cnode = (SkeletonNode)(SkeletonBTreeMap.this.root); - Iterator> centit = cnode.entries.entrySet().iterator(); + Iterator> centit = cnode.entries.entrySet().iterator(); K lastkey = null; boolean removeok = false; @@ -1636,7 +1637,7 @@ public Set keySetAutoDeflate() { /*@Override**/ public boolean hasNext() { // TODO LOW ideally iterate in the reverse order - for (Iterator> it: itstack) { + for (Iterator> it: itstack) { if (it.hasNext()) { return true; } } if (centit.hasNext()) { return true; } diff --git a/src/plugins/Library/util/SkeletonBTreeSet.java b/src/plugins/Library/util/SkeletonBTreeSet.java index 33647eec..b29e2cd6 100644 --- a/src/plugins/Library/util/SkeletonBTreeSet.java +++ b/src/plugins/Library/util/SkeletonBTreeSet.java @@ -3,20 +3,19 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.util; -import plugins.Library.io.DataFormatException; -import plugins.Library.io.serial.Serialiser.*; -import plugins.Library.io.serial.IterableSerialiser; -import plugins.Library.io.serial.MapSerialiser; -import plugins.Library.io.serial.Translator; -import plugins.Library.util.exec.TaskAbortException; import java.util.Comparator; import java.util.Collection; import java.util.Map; -import java.util.SortedMap; import java.util.SortedSet; import java.util.ArrayList; +import plugins.Library.io.DataFormatException; +import plugins.Library.io.serial.IterableSerialiser; +import plugins.Library.io.serial.MapSerialiser; +import plugins.Library.io.serial.Translator; +import plugins.Library.util.exec.TaskAbortException; + /** ** {@link Skeleton} of a {@link BTreeSet}. DOCUMENT ** diff --git a/src/plugins/Library/util/SkeletonMap.java b/src/plugins/Library/util/SkeletonMap.java index 0e4bc7a1..16dbee6c 100644 --- a/src/plugins/Library/util/SkeletonMap.java +++ b/src/plugins/Library/util/SkeletonMap.java @@ -3,11 +3,12 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.util; -import plugins.Library.io.serial.MapSerialiser; -import plugins.Library.util.exec.TaskAbortException; import java.util.Map; +import plugins.Library.io.serial.MapSerialiser; +import plugins.Library.util.exec.TaskAbortException; + /** ** A {@link Skeleton} of a {@link Map}. ** diff --git a/src/plugins/Library/util/SkeletonTreeMap.java b/src/plugins/Library/util/SkeletonTreeMap.java index 866fb2bc..802ba199 100644 --- a/src/plugins/Library/util/SkeletonTreeMap.java +++ b/src/plugins/Library/util/SkeletonTreeMap.java @@ -3,12 +3,6 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.util; -import plugins.Library.io.serial.Serialiser.*; -import plugins.Library.io.serial.Translator; -import plugins.Library.io.serial.MapSerialiser; -import plugins.Library.io.DataFormatException; -import plugins.Library.util.exec.TaskAbortException; -import plugins.Library.util.exec.TaskCompleteException; import java.util.Iterator; import java.util.Comparator; @@ -22,7 +16,12 @@ import java.util.TreeMap; import java.util.HashMap; -import freenet.support.Logger; +import plugins.Library.io.DataFormatException; +import plugins.Library.io.serial.MapSerialiser; +import plugins.Library.io.serial.Translator; +import plugins.Library.io.serial.Serialiser.*; +import plugins.Library.util.exec.TaskAbortException; +import plugins.Library.util.exec.TaskCompleteException; /** ** A {@link SkeletonMap} of a {@link TreeMap}. DOCUMENT @@ -78,7 +77,7 @@ public SkeletonTreeMap(SortedMap m) { public SkeletonTreeMap(SkeletonTreeMap m) { skmap = new TreeMap>(m.comparator()); - for (Map.Entry> en: m.skmap.entrySet()) { + for (Entry> en: m.skmap.entrySet()) { skmap.put(en.getKey(), en.getValue().clone()); } ghosts = m.ghosts; @@ -149,14 +148,14 @@ public interface SkeletonMap Map> tasks = new HashMap>(size()<<1); // load only the tasks that need pulling - for (Map.Entry> en: skmap.entrySet()) { + for (Entry> en: skmap.entrySet()) { SkeletonValue skel = en.getValue(); if (skel.isLoaded()) { continue; } tasks.put(en.getKey(), new PullTask(skel.meta())); } serialiser.pull(tasks, mapmeta); - for (Map.Entry> en: tasks.entrySet()) { + for (Entry> en: tasks.entrySet()) { assert(skmap.get(en.getKey()) != null); // TODO NORM atm old metadata is retained, could update? put(en.getKey(), en.getValue().data); @@ -169,7 +168,7 @@ public interface SkeletonMap Map> tasks = new HashMap>(size()<<1); // load all the tasks, most MapSerialisers need this info - for (Map.Entry> en: skmap.entrySet()) { + for (Entry> en: skmap.entrySet()) { SkeletonValue skel = en.getValue(); if(skel.data() == null) { if(skel.isLoaded()) @@ -191,7 +190,7 @@ else if(skel.meta() == null) } serialiser.push(tasks, mapmeta); - for (Map.Entry> en: tasks.entrySet()) { + for (Entry> en: tasks.entrySet()) { assert(skmap.get(en.getKey()) != null); putGhost(en.getKey(), en.getValue().meta); } @@ -217,7 +216,7 @@ else if(skel.meta() == null) // TODO NORM atm old metadata is retained, could update? if (tasks.isEmpty()) { return; } - for (Map.Entry> en: tasks.entrySet()) { + for (Entry> en: tasks.entrySet()) { // other keys may also have been inflated, so add them, but only if the // generated metadata match. PullTask t = en.getValue(); @@ -253,7 +252,7 @@ else if(skel.meta() == null) // OPTMISE think of a way to let the serialiser signal what // information it needs? will be quite painful to implement that... tasks.put(key, new PushTask(skel.data(), skel.meta())); - for (Map.Entry> en: skmap.entrySet()) { + for (Entry> en: skmap.entrySet()) { skel = en.getValue(); if (skel.isLoaded()) { continue; } assert(skel.data() == null); @@ -261,7 +260,7 @@ else if(skel.meta() == null) } serialiser.push(tasks, mapmeta); - for (Map.Entry> en: tasks.entrySet()) { + for (Entry> en: tasks.entrySet()) { assert(skmap.get(en.getKey()) != null); // serialiser may have pulled and re-pushed some of the // eg. Packer does this. so set all the metadata for all the tasks @@ -301,11 +300,11 @@ public static Map app(SkeletonTreeMap map, Map> en: map.skmap.entrySet()) { + for (Entry> en: map.skmap.entrySet()) { intm.put(ktr.app(en.getKey()), en.getValue().meta()); } } else { - for (Map.Entry> en: map.skmap.entrySet()) { + for (Entry> en: map.skmap.entrySet()) { intm.put(en.getKey().toString(), en.getValue().meta()); } } @@ -321,16 +320,16 @@ public static Map app(SkeletonTreeMap map, Map SkeletonTreeMap rev(Map intm, SkeletonTreeMap map, Translator ktr) throws DataFormatException { - if (ktr == null) { + if (ktr == null) { // this case when K is String and therefore no keyTranslator is needed try { - for (Map.Entry en: intm.entrySet()) { - map.putGhost((K)en.getKey(), en.getValue()); + for (Entry en: intm.entrySet()) { + map.putGhost((K) en.getKey(), en.getValue()); } } catch (ClassCastException e) { throw new DataFormatException("TreeMapTranslator: reverse translation failed. Try supplying a non-null key-translator.", e, intm, null, null); } } else { - for (Map.Entry en: intm.entrySet()) { + for (Entry en: intm.entrySet()) { map.putGhost(ktr.rev(en.getKey()), en.getValue()); } } @@ -406,13 +405,13 @@ public interface Map } else if (map instanceof SkeletonTreeMap.UnwrappingSortedSubMap) { putmap = ((UnwrappingSortedSubMap)map).bkmap; } else { - for (Map.Entry en: map.entrySet()) { + for (Entry en: map.entrySet()) { put((K)en.getKey(), (V)en.getValue()); } return; } int g = 0; - for (Map.Entry> en: putmap.entrySet()) { + for (Entry> en: putmap.entrySet()) { SkeletonValue sk = en.getValue(); SkeletonValue old = skmap.put(en.getKey(), sk); if (old == null) { @@ -446,8 +445,8 @@ public interface Map return sk.data(); } - private transient Set> entries; - @Override public Set> entrySet() { + private transient Set> entries; + @Override public Set> entrySet() { if (entries == null) { entries = new UnwrappingEntrySet(skmap, false); } @@ -589,13 +588,13 @@ public class Object */ protected class UnwrappingIterator implements Iterator { - final protected Iterator>> iter; + final protected Iterator>> iter; /** ** Last entry returned by the backing iterator. Used by {@link ** #remove()} to update the {@link #ghosts} counter correctly. */ - protected Map.Entry> last; + protected Entry> last; final protected static int KEY = 0; final protected static int VALUE = 1; @@ -625,7 +624,7 @@ protected class UnwrappingIterator implements Iterator { ** @param it The backing iterator ** @param t The {@link #type} of iterator */ - protected UnwrappingIterator(Iterator>> it, int t, boolean sub) { + protected UnwrappingIterator(Iterator>> it, int t, boolean sub) { assert(t == KEY || t == VALUE || t == ENTRY); type = t; iter = it; @@ -665,18 +664,18 @@ public void remove() { /************************************************************************ - ** {@link java.util.Map.Entry} backed by one whose value is wrapped inside a {@link + ** {@link Entry} backed by one whose value is wrapped inside a {@link ** SkeletonValue}. This class will unwrap the value when it is required, ** throwing {@link DataNotLoadedException} as necessary. ** ** @author infinity0 */ - protected class UnwrappingEntry implements Map.Entry { + protected class UnwrappingEntry implements Entry { final K key; final SkeletonValue skel; - protected UnwrappingEntry(Map.Entry> en) { + protected UnwrappingEntry(Entry> en) { key = en.getKey(); skel = en.getValue(); } @@ -713,7 +712,7 @@ public V setValue(V value) { @Override public boolean equals(Object o) { if (!(o instanceof Map.Entry)) { return false; } verifyLoaded(); - Map.Entry en = (Map.Entry)o; + Entry en = (Entry)o; return (key==null? en.getKey()==null: key.equals(en.getKey())) && (skel.data()==null? en.getValue()==null: skel.data().equals(en.getValue())); } @@ -760,8 +759,8 @@ protected UnwrappingSortedSubMap(SortedMap> sub) { return bkmap.lastKey(); } - private transient Set> entries; - @Override public Set> entrySet() { + private transient Set> entries; + @Override public Set> entrySet() { if (entries == null) { entries = new UnwrappingEntrySet(bkmap, true); } @@ -806,7 +805,7 @@ protected UnwrappingSortedSubMap(SortedMap> sub) { ** ** @author infinity0 */ - protected class UnwrappingEntrySet extends AbstractSet> { + protected class UnwrappingEntrySet extends AbstractSet> { final protected SortedMap> bkmap; /** @@ -823,8 +822,8 @@ protected UnwrappingEntrySet(SortedMap> bk, boolean sub) { @Override public int size() { return bkmap.size(); } - @Override public Iterator> iterator() { - return new UnwrappingIterator>(bkmap.entrySet().iterator(), UnwrappingIterator.ENTRY, issub); + @Override public Iterator> iterator() { + return new UnwrappingIterator>(bkmap.entrySet().iterator(), UnwrappingIterator.ENTRY, issub); } @Override public void clear() { @@ -834,7 +833,7 @@ protected UnwrappingEntrySet(SortedMap> bk, boolean sub) { @Override public boolean contains(Object o) { if (!(o instanceof Map.Entry)) { return false; } - Map.Entry en = (Map.Entry)o; + Entry en = (Entry)o; Object key = en.getKey(); Object val = en.getValue(); // return SkeletonTreeMap.this.get(e.getKey()).equals(e.getValue()); SkeletonValue sk = bkmap.get(key); @@ -847,7 +846,7 @@ protected UnwrappingEntrySet(SortedMap> bk, boolean sub) { if (issub) { throw new UnsupportedOperationException("not implemented"); } // SkeletonTreeMap.remove() called, which automatically updates _ghosts boolean c = contains(o); - if (c) { SkeletonTreeMap.this.remove(((Map.Entry)o).getKey()); } + if (c) { SkeletonTreeMap.this.remove(((Entry)o).getKey()); } return c; } diff --git a/src/plugins/Library/util/SortedArraySet.java b/src/plugins/Library/util/SortedArraySet.java index bf9564e6..ba10ccba 100644 --- a/src/plugins/Library/util/SortedArraySet.java +++ b/src/plugins/Library/util/SortedArraySet.java @@ -3,7 +3,7 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.util; -import freenet.support.Fields; // JDK6: use this instead of Arrays.binarySearch +// import freenet.support.Fields; // JDK6: use this instead of Arrays.binarySearch import java.util.Comparator; import java.util.Iterator; diff --git a/src/plugins/Library/util/SortedSetMap.java b/src/plugins/Library/util/SortedSetMap.java index ff14de5e..0dd2dd5b 100644 --- a/src/plugins/Library/util/SortedSetMap.java +++ b/src/plugins/Library/util/SortedSetMap.java @@ -77,18 +77,18 @@ public interface Map return bkset.remove(o)? (E)o: null; } - private transient Set> entries; - @Override public Set> entrySet() { + private transient Set> entries; + @Override public Set> entrySet() { if (entries == null) { - entries = new AbstractSet>() { + entries = new AbstractSet>() { @Override public int size() { return bkset.size(); } - @Override public Iterator> iterator() { - return new Iterator>() { + @Override public Iterator> iterator() { + return new Iterator>() { final Iterator it = bkset.iterator(); /*@Override**/ public boolean hasNext() { return it.hasNext(); } - /*@Override**/ public Map.Entry next() { + /*@Override**/ public Entry next() { E e = it.next(); return Maps.$$(e, e); } @@ -102,14 +102,14 @@ public interface Map @Override public boolean contains(Object o) { if (!(o instanceof Map.Entry)) { return false; } - Map.Entry en = (Map.Entry)o; + Entry en = (Entry)o; if (en.getKey() != en.getValue()) { return false; } return bkset.contains(en.getKey()); } @Override public boolean remove(Object o) { boolean c = contains(o); - if (c) { bkset.remove(((Map.Entry)o).getKey()); } + if (c) { bkset.remove(((Entry)o).getKey()); } return c; } diff --git a/src/plugins/Library/util/TaskAbortExceptionConvertor.java b/src/plugins/Library/util/TaskAbortExceptionConvertor.java index 5cefdd02..3f3aa5ee 100644 --- a/src/plugins/Library/util/TaskAbortExceptionConvertor.java +++ b/src/plugins/Library/util/TaskAbortExceptionConvertor.java @@ -9,5 +9,4 @@ public class TaskAbortExceptionConvertor implements public TaskAbortException convert(RuntimeException e) { return new TaskAbortException(e.getMessage(), e); } - } diff --git a/src/plugins/Library/util/concurrent/BoundedPriorityBlockingQueue.java b/src/plugins/Library/util/concurrent/BoundedPriorityBlockingQueue.java index d9e4c802..35335c6f 100644 --- a/src/plugins/Library/util/concurrent/BoundedPriorityBlockingQueue.java +++ b/src/plugins/Library/util/concurrent/BoundedPriorityBlockingQueue.java @@ -192,5 +192,4 @@ public void remove() { } // writeObject() is safe - } diff --git a/src/plugins/Library/util/concurrent/ExceptionConvertor.java b/src/plugins/Library/util/concurrent/ExceptionConvertor.java index 62454895..0ec1034f 100644 --- a/src/plugins/Library/util/concurrent/ExceptionConvertor.java +++ b/src/plugins/Library/util/concurrent/ExceptionConvertor.java @@ -2,6 +2,5 @@ public interface ExceptionConvertor { - public X convert(RuntimeException e); - + X convert(RuntimeException e); } diff --git a/src/plugins/Library/util/concurrent/Executors.java b/src/plugins/Library/util/concurrent/Executors.java index b8d9513f..0f78e8ea 100644 --- a/src/plugins/Library/util/concurrent/Executors.java +++ b/src/plugins/Library/util/concurrent/Executors.java @@ -3,8 +3,6 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.util.concurrent; -import plugins.Library.util.func.SafeClosure; - import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadPoolExecutor; @@ -12,6 +10,8 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import plugins.Library.util.func.SafeClosure; + /** ** Class providing various {@link Executor}s. ** @@ -49,9 +49,9 @@ public class Executors { synchronized (Executors.class) { if (default_exec == null) { default_exec = new ThreadPoolExecutor( - 1, 0x40, 60, TimeUnit.SECONDS, + 0, 0x40, 60, TimeUnit.SECONDS, new LinkedBlockingQueue(), - new ThreadPoolExecutor.CallerRunsPolicy() + new CallerRunsPolicy() ); } } @@ -69,5 +69,4 @@ public static synchronized void setDefaultExecutor(Executor e) { private static Executor default_exec = null; private Executors() { } - } diff --git a/src/plugins/Library/util/concurrent/Notifier.java b/src/plugins/Library/util/concurrent/Notifier.java index c1a6fd7f..02b7b332 100644 --- a/src/plugins/Library/util/concurrent/Notifier.java +++ b/src/plugins/Library/util/concurrent/Notifier.java @@ -27,5 +27,4 @@ public synchronized void waitUpdate(int maxWait) { } notified = false; } - } diff --git a/src/plugins/Library/util/concurrent/ObjectProcessor.java b/src/plugins/Library/util/concurrent/ObjectProcessor.java index b0049a34..2a554dbb 100644 --- a/src/plugins/Library/util/concurrent/ObjectProcessor.java +++ b/src/plugins/Library/util/concurrent/ObjectProcessor.java @@ -15,9 +15,9 @@ import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; +import freenet.support.Logger; import plugins.Library.util.func.Closure; import plugins.Library.util.func.SafeClosure; -import freenet.support.Logger; /** ** A class that wraps around an {@link Executor}, for processing any given type @@ -45,13 +45,6 @@ public class ObjectProcessor implements Scheduler { protected int dispatched = 0; protected int completed = 0; protected int started = 0; - - private static volatile boolean logMINOR; - private static volatile boolean logDEBUG; - - static { - Logger.registerClass(ObjectProcessor.class); - } // Most ObjectProcessor's are likely to be autostart()'ed, and this way we can // still use ConcurrentMap for pending while having garbage collection. @@ -103,7 +96,7 @@ public int outputSize() { */ public ObjectProcessor( BlockingQueue input, BlockingQueue> output, Map deposit, - Closure closure, Executor executor, ExceptionConvertor conv, Notifier n + Closure closure, Executor executor, ExceptionConvertor conv, Notifier notifier ) { in = input; out = output; @@ -111,7 +104,7 @@ public ObjectProcessor( clo = closure; exec = executor; convertor = conv; - notifier = n; + this.notifier = notifier; } public ObjectProcessor( @@ -224,7 +217,7 @@ public synchronized int size() { ** This method is provided for completeness, in case anyone needs it; ** {@link #auto()} should be adequate for most purposes. ** - ** @throws InterruptedExeception if interrupted whilst waiting + ** @throws InterruptedException if interrupted whilst waiting */ public synchronized void dispatchTake() throws InterruptedException { throw new UnsupportedOperationException("not implemented"); @@ -275,11 +268,9 @@ protected Runnable createJobFor(final T item) { /*@Override**/ public void run() { X ex = null; synchronized(ObjectProcessor.this) { ++started; } - RuntimeException ee = null; try { clo.invoke(item); } // FIXME NORM this could throw RuntimeException catch (RuntimeException e) { - Logger.error(this, "Caught "+e, e); System.err.println("In ObjProc-"+name+" : "+e); e.printStackTrace(); ex = convertor.convert(e); @@ -311,7 +302,9 @@ private static synchronized void ensureAutoHandler() { try { boolean o = proc.open; while (proc.dispatchPoll()); - if (!o) { it.remove(); } + if (!o) { + it.remove(); + } } catch (RejectedExecutionException e) { // FIXME NORM // neither Executors.DEFAULT_EXECUTOR nor Freenet's in-built executors @@ -328,7 +321,6 @@ private static synchronized void ensureAutoHandler() { } catch (InterruptedException e) { // TODO LOW log this somewhere } - // System.out.println("pending " + pending.size()); if (t > 0) { continue; } synchronized (ObjectProcessor.class) { @@ -363,7 +355,7 @@ public boolean auto() { /** ** Call {@link #auto()} and return {@code this}. */ - public ObjectProcessor autostart() { + public ObjectProcessor autostart() { auto(); return this; } @@ -396,5 +388,4 @@ public void setName(String n) { @Override public String toString() { return "ObjProc-" + name + ":{" + size() + "|" + dispatched + "|" + started + "|" + completed + "}"; } - } diff --git a/src/plugins/Library/util/concurrent/Scheduler.java b/src/plugins/Library/util/concurrent/Scheduler.java index e1759445..e46cc5e5 100644 --- a/src/plugins/Library/util/concurrent/Scheduler.java +++ b/src/plugins/Library/util/concurrent/Scheduler.java @@ -7,7 +7,7 @@ ** An interface for a general class that accepts objects to be acted on. ** ** TODO NORM maybe just get rid of this since we now have ObjectProcessor, or -** import its methods here? +** import its methods here? ** ** @author infinity0 */ @@ -16,6 +16,5 @@ public interface Scheduler extends java.io.Closeable { /** ** Stop accepting objects (but continue any started actions). */ - public void close(); - + void close(); } diff --git a/src/plugins/Library/util/exec/SimpleProgress.java b/src/plugins/Library/util/exec/SimpleProgress.java index de1f7426..e25f5b79 100644 --- a/src/plugins/Library/util/exec/SimpleProgress.java +++ b/src/plugins/Library/util/exec/SimpleProgress.java @@ -91,9 +91,10 @@ public boolean finalizedTotal() { */ public synchronized void addPartDone() { if (pdone == known) { - throw new IllegalStateException("Can't increased parts done above parts known"); + throw new IllegalStateException("More parts done than known"); } pdone++; + // System.err.println("DEBUG: " + this + " done " + pdone + "/" + known); if (finalizedTotal() && pdone == known) { inprogress = false; notifyAll(); @@ -116,12 +117,14 @@ public synchronized void addPartKnown(int parts, boolean finalise) { estimate = known + parts; known = estimate; estimate = ProgressParts.TOTAL_FINALIZED; + // System.err.println("DEBUG: " + this + " finalise " + pdone + "/" + known); } else { if (finalizedTotal()) { throw new IllegalArgumentException("Cannot un-finalise a final total!"); } estimate = ProgressParts.ESTIMATE_UNKNOWN; known += parts; + // System.err.println("DEBUG: " + this + " add " + pdone + "/" + known); } } diff --git a/src/plugins/Library/util/func/Closure.java b/src/plugins/Library/util/func/Closure.java index d32ca796..6cf959d1 100644 --- a/src/plugins/Library/util/func/Closure.java +++ b/src/plugins/Library/util/func/Closure.java @@ -35,6 +35,5 @@ */ public interface Closure { - public void invoke(P param) throws E; - + void invoke(P param) throws E; } diff --git a/src/plugins/Library/util/func/SafeClosure.java b/src/plugins/Library/util/func/SafeClosure.java index 1235781c..bda839ce 100644 --- a/src/plugins/Library/util/func/SafeClosure.java +++ b/src/plugins/Library/util/func/SafeClosure.java @@ -12,6 +12,5 @@ */ public interface SafeClosure

extends Closure { - public void invoke(P param); - + void invoke(P param); } diff --git a/test/plugins/Library/FreenetURIForTesting.java b/test/plugins/Library/FreenetURIForTesting.java new file mode 100644 index 00000000..5d8d3f9f --- /dev/null +++ b/test/plugins/Library/FreenetURIForTesting.java @@ -0,0 +1,18 @@ +package plugins.Library; + +import plugins.Library.io.FreenetURI; + +import java.net.MalformedURLException; +import java.util.Random; + +public class FreenetURIForTesting extends FreenetURI { + + public FreenetURIForTesting(String uri) throws MalformedURLException { + super(uri); + throw new RuntimeException("Cannot create for test."); + } + + public static FreenetURI generateRandomCHK(Random rand) { + throw new RuntimeException("Not implemented yet."); + } +} diff --git a/test/plugins/Library/Tester.java b/test/plugins/Library/Tester.java index bb16a4a8..088bda0a 100644 --- a/test/plugins/Library/Tester.java +++ b/test/plugins/Library/Tester.java @@ -4,18 +4,28 @@ package plugins.Library; import plugins.Library.client.*; -import plugins.Library.util.exec.*; -import plugins.Library.util.func.Closure; import plugins.Library.index.*; -import plugins.Library.io.*; -import plugins.Library.io.serial.*; -import plugins.Library.io.serial.Serialiser.*; import plugins.Library.util.*; import plugins.Library.*; import freenet.keys.FreenetURI; import freenet.node.RequestStarter; +import plugins.Library.Priority; +import plugins.Library.index.ProtoIndex; +import plugins.Library.index.ProtoIndexComponentSerialiser; +import plugins.Library.index.ProtoIndexSerialiser; +import plugins.Library.index.TermEntry; +import plugins.Library.io.YamlReaderWriter; +import plugins.Library.io.serial.LiveArchiver; +import plugins.Library.io.serial.Serialiser.*; +import plugins.Library.util.SkeletonBTreeSet; +import plugins.Library.util.TaskAbortExceptionConvertor; +import plugins.Library.util.exec.ProgressParts; +import plugins.Library.util.exec.SimpleProgress; +import plugins.Library.util.exec.TaskAbortException; +import plugins.Library.util.func.Closure; + import java.util.*; import java.io.*; import java.net.*; @@ -76,7 +86,7 @@ public static String testPushProgress() { if (push_progress_thread == null) { push_progress_thread = new Thread() { YamlReaderWriter yamlrw = new YamlReaderWriter(); - FreenetArchiver> arx = Library.makeArchiver(yamlrw, "text/yaml", 0x10000, RequestStarter.INTERACTIVE_PRIORITY_CLASS); + LiveArchiver, SimpleProgress> arx = Library.makeArchiver(yamlrw, "text/yaml", 0x10000, RequestStarter.INTERACTIVE_PRIORITY_CLASS); @Override public void run() { push_progress_start = new Date(); @@ -144,13 +154,13 @@ public static String testPushIndex() { if (push_index_thread == null) { push_index_start = new Date(); push_index_thread = new Thread() { - ProtoIndexSerialiser srl = ProtoIndexSerialiser.forIndex(push_index_endURI, RequestStarter.INTERACTIVE_PRIORITY_CLASS); + ProtoIndexSerialiser srl = ProtoIndexSerialiser.forIndex(push_index_endURI.toASCIIString(), Priority.Interactive); ProtoIndex idx; Random rand = new Random(); @Override public void run() { try { - idx = new ProtoIndex(new FreenetURI("CHK@"), "test", null, null, 0); + idx = new ProtoIndex(new plugins.Library.io.FreenetURI("CHK@"), "test", null, null, 0); } catch (java.net.MalformedURLException e) { throw new AssertionError(e); } @@ -228,14 +238,14 @@ public static String testPushAndMergeIndex() { FreenetArchiver.setCacheDir(cacheDir); push_index_thread = new Thread() { - ProtoIndexSerialiser srl = ProtoIndexSerialiser.forIndex(push_index_endURI, RequestStarter.INTERACTIVE_PRIORITY_CLASS); + ProtoIndexSerialiser srl = ProtoIndexSerialiser.forIndex(push_index_endURI.toASCIIString(), Priority.Interactive); ProtoIndex idx; Random rand = new Random(); @Override public void run() { try { - idx = new ProtoIndex(new FreenetURI("CHK@"), "test", null, null, 0); + idx = new ProtoIndex(new plugins.Library.io.FreenetURI("CHK@"), "test", null, null, 0); } catch (java.net.MalformedURLException e) { throw new AssertionError(e); } diff --git a/test/plugins/Library/index/BIndexTest.java b/test/plugins/Library/index/BIndexTest.java index e3fadcf6..6fdac623 100644 --- a/test/plugins/Library/index/BIndexTest.java +++ b/test/plugins/Library/index/BIndexTest.java @@ -7,13 +7,14 @@ import static plugins.Library.util.Generators.rand; import plugins.Library.util.*; -import plugins.Library.util.func.*; -import plugins.Library.util.exec.*; -import plugins.Library.io.serial.*; -import plugins.Library.io.serial.Serialiser.*; -import plugins.Library.index.*; -import freenet.keys.FreenetURI; +import plugins.Library.io.serial.Serialiser.*; +import plugins.Library.util.SkeletonBTreeMap; +import plugins.Library.util.SkeletonBTreeSet; +import plugins.Library.util.TaskAbortExceptionConvertor; +import plugins.Library.util.exec.Execution; +import plugins.Library.util.exec.TaskAbortException; +import plugins.Library.util.func.Closure; import java.util.*; import java.io.*; @@ -26,7 +27,7 @@ public class BIndexTest extends TestCase { // mininum number of entries in a b-tree node. - final public static int node_size = 0x04; + final public static int node_size = 1024; // base number of keys in an index. the final size (after async-update) will be around double this. final public static int index_size = 0x80; @@ -56,16 +57,16 @@ public long timeDiff() { time = System.currentTimeMillis(); return time - oldtime; } - + public BIndexTest() { f = new File("BindexTest"); f.mkdir(); srl = ProtoIndexSerialiser.forIndex(f); csrl = ProtoIndexComponentSerialiser.get(ProtoIndexComponentSerialiser.FMT_FILE_LOCAL, srl.getChildSerialiser()); } - + private final File f; - private final ProtoIndexSerialiser srl; + private final ProtoIndexSerialiser srl; private final ProtoIndexComponentSerialiser csrl; private ProtoIndex idx; @@ -84,7 +85,7 @@ public BIndexTest() { protected void newTestSkeleton() { try { - idx = new ProtoIndex(new FreenetURI("CHK@yeah"), "test", null, null, 0); + idx = new ProtoIndex(new plugins.Library.io.FreenetURI("CHK@yeah"), "test", null, null, 0); } catch (java.net.MalformedURLException e) { assertTrue(false); } @@ -219,7 +220,7 @@ public void fullInflateDeflateUpdate() throws TaskAbortException { assertTrue(count == idx.ttab.size()); System.out.println("Iterated keys, total is "+count+" size is "+idx.ttab.size()); assertTrue(idx.ttab.isBare()); - + // full inflate (2) PullTask task5 = new PullTask(task4.meta); srl.pull(task5); @@ -248,7 +249,6 @@ public void fullInflateDeflateUpdate() throws TaskAbortException { System.out.println("merge validated in " + timeDiff() + " ms."); System.out.println(""); - } public void testBasicMulti() throws TaskAbortException { @@ -302,7 +302,6 @@ public void partialInflate() throws TaskAbortException { assertTrue(idx.ttab.isLive()); assertFalse(idx.ttab.isBare()); System.out.println("inflated all terms separately in " + timeDiff() + " ms"); - } public void testPartialInflateMulti() throws TaskAbortException { @@ -375,7 +374,5 @@ public void testProgress() throws TaskAbortException { } assertTrue(count == entries.size()); System.out.println(count + " entries successfully got in " + rq1.getTimeElapsed() + "ms"); - } - } diff --git a/test/plugins/Library/index/ProtoIndexSerialiserTest.java b/test/plugins/Library/index/ProtoIndexSerialiserTest.java new file mode 100644 index 00000000..0f3fa9dc --- /dev/null +++ b/test/plugins/Library/index/ProtoIndexSerialiserTest.java @@ -0,0 +1,302 @@ +package plugins.Library.index; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.util.Map; + +import plugins.Library.io.FreenetURI; +import plugins.Library.io.ObjectStreamReader; +import plugins.Library.io.ObjectStreamWriter; +import plugins.Library.io.serial.FileArchiver; +import plugins.Library.io.serial.LiveArchiver; +import plugins.Library.io.serial.Serialiser.PullTask; +import plugins.Library.io.serial.Serialiser.PushTask; +import plugins.Library.io.serial.Translator; +import plugins.Library.ArchiverFactory; +import plugins.Library.FactoryRegister; +import plugins.Library.Priority; +import plugins.Library.util.SkeletonBTreeMap; +import plugins.Library.util.SkeletonBTreeSet; +import plugins.Library.util.exec.SimpleProgress; +import plugins.Library.util.exec.TaskAbortException; + +import junit.framework.TestCase; + +public class ProtoIndexSerialiserTest extends TestCase { + private MockLiveArchiver mockLiveArchiver; + private MockArchiverFactory mockArchiverFactory; + private ProtoIndexSerialiser tested_object; + private ProtoIndex mockProtoIndex; + + /** + * For pull, the meta is a String containing the contents of the stream. + */ + static class MockLiveArchiver implements LiveArchiver, SimpleProgress> { + ObjectStreamReader> reader; + ObjectStreamWriter> writer; + MockLiveArchiver(ObjectStreamReader> r, + ObjectStreamWriter> w) { + reader = r; + writer = w; + } + + byte[] bytesToParse; + String createdOutput; + int archiverResultNumber = 1; + + @Override + public void pull( + plugins.Library.io.serial.Serialiser.PullTask> task) + throws TaskAbortException { + assertNotNull(bytesToParse); + InputStream is = new ByteArrayInputStream(bytesToParse); + try { + task.data = reader.readObject(is); + } catch (IOException e) { + throw new TaskAbortException("byte array unparseable", e); + } + } + + @Override + public void push( + plugins.Library.io.serial.Serialiser.PushTask> task) + throws TaskAbortException { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try { + writer.writeObject(task.data, os); + } catch (IOException e) { + throw new TaskAbortException("Could not write", e); + } + createdOutput = os.toString(); + } + + @Override + public void pullLive( + plugins.Library.io.serial.Serialiser.PullTask> task, + SimpleProgress p) throws TaskAbortException { + fail("Not yet implemented"); // TODO + } + + @Override + public void pushLive( + plugins.Library.io.serial.Serialiser.PushTask> task, + SimpleProgress p) throws TaskAbortException { + if (p != null) { + p.addPartKnown(1, true); + } + push(task); + try { + task.meta = new FreenetURI("CHK@" + (archiverResultNumber++) + ",7,A6"); + } catch (MalformedURLException e) { + throw new TaskAbortException("URL problem", e); + } + if (p != null) { + p.addPartDone(); + } + } + + @Override + public void waitForAsyncInserts() throws TaskAbortException { + // Do nothing + } + } + + class MockArchiverFactory implements ArchiverFactory { + + @Override + public LiveArchiver newArchiver( + S rw, String mime, int size, Priority priorityLevel) { + assertNotNull(rw); + assertEquals(ProtoIndexComponentSerialiser.yamlrw, rw); + assertNotNull(mime); + assertNotSame(0, size); + assertEquals(Priority.Bulk, priorityLevel); + return (LiveArchiver) new MockLiveArchiver(rw, rw); + } + + @Override + public LiveArchiver newArchiver( + S rw, String mime, int size, + LiveArchiver archiver) { + fail("Not called by the tests."); + return null; + } + } + + protected void setUp() throws Exception { + super.setUp(); + mockArchiverFactory = new MockArchiverFactory(); + FactoryRegister.register(mockArchiverFactory); + mockLiveArchiver = new MockLiveArchiver( + (ObjectStreamReader) ProtoIndexComponentSerialiser.yamlrw, + (ObjectStreamWriter) ProtoIndexComponentSerialiser.yamlrw); + + tested_object = new ProtoIndexSerialiser(mockLiveArchiver); + + FreenetURI reqID = null; + mockProtoIndex = new ProtoIndex(reqID, "name", "owner", "email", 0); + } + + protected void tearDown() throws Exception { + super.tearDown(); + } + + public void testProtoIndexSerialiser() { + assertNotNull(tested_object); + } + + private void assertProtoIndexSerialiserURI(ProtoIndexSerialiser result) { + assertNotNull(result); + assertNotNull(result.getChildSerialiser()); + assertTrue(result.getChildSerialiser() instanceof MockLiveArchiver); + } + + public void testForIndexURIAsObjectPriority() throws MalformedURLException { + FreenetURI uri = new FreenetURI("CHK@"); + Object obj = uri; + Priority priority = Priority.Bulk; + + ProtoIndexSerialiser result = ProtoIndexSerialiser.forIndex(obj, priority); + + assertProtoIndexSerialiserURI(result); + } + + private void assertProtoIndexSerialiserFile(ProtoIndexSerialiser result) { + assertNotNull(result); + assertNotNull(result.getChildSerialiser()); + assertTrue(result.getChildSerialiser() instanceof FileArchiver); + } + + public void testForIndexFileAsObjectPriority() { + File file = new File("file"); + Object obj = file; + Priority priority = Priority.Bulk; + + ProtoIndexSerialiser result = ProtoIndexSerialiser.forIndex(obj, priority); + + assertProtoIndexSerialiserFile(result); + } + + public void testForIndexUnmatchedObjectPriority() throws MalformedURLException { + Object obj = new Object(); + Priority priority = Priority.Bulk; + + try { + ProtoIndexSerialiser.forIndex(obj, priority); + fail("Should have thrown."); + } catch (UnsupportedOperationException e) { + // OK. + } + } + + public void testForIndexFreenetURIPriority() throws MalformedURLException { + FreenetURI uri = new FreenetURI("CHK@"); + Priority priority = Priority.Bulk; + + ProtoIndexSerialiser result = ProtoIndexSerialiser.forIndex(uri, priority); + + assertProtoIndexSerialiserURI(result); + } + + public void testForIndexFile() { + final File prefix = new File("prefix"); + ProtoIndexSerialiser result = ProtoIndexSerialiser.forIndex(prefix); + + assertProtoIndexSerialiserFile(result); + } + + public void testGetChildSerialiser() { + final LiveArchiver, SimpleProgress> result = tested_object.getChildSerialiser(); + + assertNotNull(result); + } + + public void testGetTranslator() { + final Translator> translator = tested_object.getTranslator(); + assertNotNull(translator); + + translator.app(mockProtoIndex); + } + + public void testPull() throws TaskAbortException, MalformedURLException { + final FreenetURI req_id = new FreenetURI("CHK@"); + PullTask task = new PullTask(req_id); + final String name = "New Spider index."; + final long totalPages = 17; + mockLiveArchiver.bytesToParse = ( + "serialVersionUID: " + ProtoIndex.serialVersionUID + "\n" + + "serialFormatUID: !!int '" + ProtoIndexComponentSerialiser.FMT_DEFAULT + "'\n" + + "totalPages: !!int '" + totalPages + "'\n" + + "name: " + name + "\n" + + "utab:\n" + + " node_min: !!int '1024'\n" + + " size: !!int '0'\n" + + " entries: {}\n" + + "ttab:\n" + + " node_min: !!int '1024'\n" + + " size: !!int '2470'\n" + + " entries:\n" + + " adam: !BinInfo {? &id001 !!binary \"abcdef==\" : !!int '1'}\n" + + " subnodes:\n" + + " !FreenetURI 'CHK@123,456,A789': !!int '1234'\n" + + " !FreenetURI 'CHK@456,678,A890': !!int '1235'\n" + + "").getBytes(); + task.data = mockProtoIndex; + + tested_object.pull(task); + + assertEquals(req_id, task.data.reqID); + assertEquals(name, task.data.name); + assertEquals(totalPages, task.data.totalPages); + assertEquals(new SkeletonBTreeMap>(1024), + task.data.utab); + SkeletonBTreeMap> x = task.data.ttab; + } + + public void testPushEmpty() throws TaskAbortException { + PushTask task = new PushTask(mockProtoIndex); + + tested_object.push(task); + + assertTrue(mockLiveArchiver.createdOutput.contains("serialVersionUID: !!int '" + ProtoIndex.serialVersionUID)); + final String emptyBTree = "\n node_min: !!int '1024'\n size: !!int '0'\n entries: {}\n"; + assertTrue(mockLiveArchiver.createdOutput.contains("\nutab:" + emptyBTree)); + assertTrue(mockLiveArchiver.createdOutput.contains("\nttab:" + emptyBTree)); + } + + public void testPushContents() throws TaskAbortException, MalformedURLException { + ProtoIndexSerialiser srl = ProtoIndexSerialiser.forIndex(new FreenetURI("CHK@"), Priority.Bulk); + LiveArchiver,SimpleProgress> archiver = + (LiveArchiver,SimpleProgress>)(srl.getChildSerialiser()); + ProtoIndexComponentSerialiser leafsrl = ProtoIndexComponentSerialiser.get(ProtoIndexComponentSerialiser.FMT_DEFAULT, archiver); + + PushTask task = new PushTask(mockProtoIndex); + + final int ENTRIES = 10000; + for (int i = 0; i < ENTRIES; i++) { + final SkeletonBTreeSet value = new SkeletonBTreeSet(100); + value.add(new TermPageEntry("a", 1, new FreenetURI("CHK@1,2,A3"), "title", null)); + leafsrl.setSerialiserFor(value); + value.deflate(); + + mockProtoIndex.ttab.put("a" + i, value); + } + + leafsrl.setSerialiserFor(mockProtoIndex); + mockProtoIndex.ttab.deflate(); + + mockLiveArchiver.waitForAsyncInserts(); + + tested_object.push(task); + + assertTrue(mockLiveArchiver.createdOutput.contains("serialVersionUID: !!int '" + ProtoIndex.serialVersionUID)); + final String emptyBTree = "\n node_min: !!int '1024'\n size: !!int '0'\n entries: {}\n"; + assertTrue(mockLiveArchiver.createdOutput.contains("\nutab:" + emptyBTree)); + final String countBTreeProlog = "\n node_min: !!int '1024'\n size: !!int '" + ENTRIES + "'\n entries:\n"; + assertTrue(mockLiveArchiver.createdOutput.contains("\nttab:" + countBTreeProlog)); + } +} diff --git a/test/plugins/Library/index/TermEntryTest.java b/test/plugins/Library/index/TermEntryTest.java index fec1256f..687781bb 100644 --- a/test/plugins/Library/index/TermEntryTest.java +++ b/test/plugins/Library/index/TermEntryTest.java @@ -5,24 +5,19 @@ import junit.framework.TestCase; -import plugins.Library.io.serial.Serialiser.*; +import plugins.Library.io.YamlReaderWriter; import plugins.Library.io.serial.FileArchiver; -import plugins.Library.util.exec.TaskAbortException; import plugins.Library.io.serial.Packer; +import plugins.Library.io.FreenetURI; +import plugins.Library.io.serial.Serialiser.*; +import plugins.Library.util.exec.TaskAbortException; -import plugins.Library.io.YamlReaderWriter; - -import freenet.keys.FreenetURI; - -import java.util.Arrays; import java.util.List; import java.util.ArrayList; import java.util.Map; import java.util.HashMap; -import java.util.Iterator; -import java.util.UUID; -import java.net.MalformedURLException; import java.io.*; +import java.net.MalformedURLException; /** ** @author infinity0 @@ -39,7 +34,7 @@ public class TermEntryTest extends TestCase { z = new TermPageEntry("lol", 0.8f, new FreenetURI("CHK@9eDo5QWLQcgSuDh1meTm96R4oE7zpoMBuV15jLiZTps,3HJaHbdW~-MtC6YsSkKn6I0DTG9Z1gKDGgtENhHx82I,AAIC--8"), null); v = new TermPageEntry("lol", 0.8f, new FreenetURI("CHK@9eDo5QWLQcgSuDh1meTm96R4oE7zpoMBuV15jLiZTps,3HJaHbdW~-MtC6YsSkKn6I0DTG9Z1gKDGgtENhHx82I,AAIC--8"), "title", null); } catch (MalformedURLException e) { - throw new AssertionError(); + throw new AssertionError(e); } } final static TermTermEntry y = new TermTermEntry("test", 0.8f, "lol2"); @@ -58,19 +53,12 @@ public void testBasic() throws TaskAbortException { l.add(y); l.add(z); map.put("test", l); - try { - map.put("test2", new Packer.BinInfo(new FreenetURI("http://127.0.0.1:8888/CHK@WtWIvOZXLVZkmDrY5929RxOZ-woRpRoMgE8rdZaQ0VU,rxH~D9VvOOuA7bCnVuzq~eux77i9RR3lsdwVHUgXoOY,AAIC--8/Library.jar"), 123)); - } catch (java.net.MalformedURLException e) { - assert(false); - } + map.put("test2", new Packer.BinInfo("CHK@WtWIvOZXLVZkmDrY5929RxOZ-woRpRoMgE8rdZaQ0VU,rxH~D9VvOOuA7bCnVuzq~eux77i9RR3lsdwVHUgXoOY,AAIC--8/Library.jar", 123)); ym.push(new PushTask>(map)); PullTask> pt = new PullTask>(""); - try{ + ym.pull(pt); - } catch (Exception e) { - e.printStackTrace(); - } assertTrue(pt.data instanceof Map); Map m = pt.data; @@ -84,7 +72,7 @@ public void testBasic() throws TaskAbortException { assertTrue(m.get("test2") instanceof Packer.BinInfo); Packer.BinInfo inf = (Packer.BinInfo)m.get("test2"); - assertTrue(inf.getID() instanceof FreenetURI); + assertTrue(inf.getID() instanceof String); } public void testBinaryReadWrite() throws IOException, TaskAbortException { @@ -113,9 +101,33 @@ public void testBinaryReadWrite() throws IOException, TaskAbortException { } public static void assertEqualButNotIdentical(Object a, Object b) { - assertTrue(a != b); - assertTrue(a.equals(b)); - assertTrue(a.hashCode() == b.hashCode()); + assertTrue(a + " and " + b + " are identical.", a != b); + assertTrue(a + " and " + b + " not equal.", a.equals(b)); + assertTrue(a + " and " + b + " not same hashCode.", a.hashCode() == b.hashCode()); + } + + private TermEntry TE(String s) { + return new TermEntry(s, 0) { + @Override + public boolean equalsTarget(TermEntry entry) { + return false; + } + + @Override + public EntryType entryType() { + return null; + } + }; } + public void testToBeDropped() { + assertFalse(TE("").toBeDropped()); + assertFalse(TE("1h1").toBeDropped()); + assertTrue(TE("1hh1").toBeDropped()); + assertFalse(TE("r2d2").toBeDropped()); + assertFalse(TE("c3po").toBeDropped()); + assertTrue(TE("a1b2c3d4e5").toBeDropped()); + assertFalse(TE("conventional").toBeDropped()); + assertTrue(TE("abcdef12345fedcba54321aabbee").toBeDropped()); + } } diff --git a/test/plugins/Library/io/FreenetURITest.java b/test/plugins/Library/io/FreenetURITest.java new file mode 100644 index 00000000..f0e3c3ea --- /dev/null +++ b/test/plugins/Library/io/FreenetURITest.java @@ -0,0 +1,27 @@ +package plugins.Library.io; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.MalformedURLException; + +import org.junit.Test; + +import static org.junit.Assert.*; + +public class FreenetURITest { + + @Test + public void getUSKRootTest() throws MalformedURLException { + FreenetURI to = new FreenetURI("USK@aa,bb,Acc/file/12345/meta"); + assertEquals("USK@aa,bb,Acc/file", to.getRoot()); + } + + @Test + public void toYamlTest() throws IOException { + FreenetURI freenetURI = new FreenetURI("USK@aa,bb,Acc/file/12345/meta"); + YamlReaderWriter yamlReaderWriter = new YamlReaderWriter(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + yamlReaderWriter.writeObject(freenetURI, outputStream); + assertEquals("!FreenetURI 'USK@aa,bb,Acc/file/12345/meta'" + System.lineSeparator(), outputStream.toString()); + } +} diff --git a/test/plugins/Library/io/serial/PackerTest.java b/test/plugins/Library/io/serial/PackerTest.java index 7687f763..03229ee0 100644 --- a/test/plugins/Library/io/serial/PackerTest.java +++ b/test/plugins/Library/io/serial/PackerTest.java @@ -5,17 +5,13 @@ import junit.framework.TestCase; -import plugins.Library.util.Generators; -import plugins.Library.util.SkeletonTreeMap; -import plugins.Library.util.exec.TaskAbortException; -import plugins.Library.io.serial.Packer.Bin; -import plugins.Library.io.serial.Serialiser.*; - import java.util.Map; -import java.util.List; -import java.util.Iterator; import java.util.HashSet; import java.util.HashMap; +import java.util.UUID; + +import plugins.Library.io.serial.Serialiser.*; +import plugins.Library.util.exec.TaskAbortException; /** ** PRIORITY actually write some tests for this... @@ -26,6 +22,14 @@ public class PackerTest extends TestCase { final static public int NODE_MAX = 64; + private static String rndStr() { + return UUID.randomUUID().toString(); + } + + private static String rndKey() { + return rndStr().substring(0,8); + } + final public static Packer srl = new Packer( new IterableSerialiser>() { @@ -60,7 +64,7 @@ protected Map> generateTasks(int[] sizes) { for (int i=0; i(hs, meta)); + tasks.put(rndKey(), new PushTask(hs, meta)); } return tasks; } diff --git a/test/plugins/Library/io/serial/YamlMapTest.java b/test/plugins/Library/io/serial/YamlMapTest.java index 414e8a21..7ef8eccc 100644 --- a/test/plugins/Library/io/serial/YamlMapTest.java +++ b/test/plugins/Library/io/serial/YamlMapTest.java @@ -6,19 +6,14 @@ import junit.framework.TestCase; import org.yaml.snakeyaml.Yaml; -import org.yaml.snakeyaml.Yaml; -import org.yaml.snakeyaml.error.YAMLException; -import org.yaml.snakeyaml.Loader; -import org.yaml.snakeyaml.Dumper; import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.nodes.Tag; import org.yaml.snakeyaml.representer.Representer; import org.yaml.snakeyaml.representer.Represent; import org.yaml.snakeyaml.nodes.Node; import org.yaml.snakeyaml.nodes.ScalarNode; -import org.yaml.snakeyaml.nodes.MappingNode; import org.yaml.snakeyaml.constructor.Constructor; import org.yaml.snakeyaml.constructor.Construct; -import org.yaml.snakeyaml.constructor.ConstructorException; import java.io.*; import java.util.*; @@ -29,14 +24,13 @@ public class YamlMapTest extends TestCase { public void testYamlMap() throws IOException { - Map data = new TreeMap(); + Map data = new TreeMap<>(); data.put("key1", new Bean()); data.put("key2", new Bean()); data.put("key3", new Custom("test")); data.put("key4", new Wrapper("test", new Custom("test"))); - Yaml yaml = new Yaml(new Loader(new ExtendedConstructor()), - new Dumper(new ExtendedRepresenter(), new DumperOptions())); + Yaml yaml = new Yaml(new ExtendedConstructor(), new ExtendedRepresenter(), new DumperOptions()); File file = new File("beantest.yml"); FileOutputStream os = new FileOutputStream(file); @@ -44,15 +38,13 @@ public void testYamlMap() throws IOException { os.close(); FileInputStream is = new FileInputStream(file); - Object o = yaml.load(new InputStreamReader(is)); + Map map = yaml.load(new InputStreamReader(is)); is.close(); - assertTrue(o instanceof Map); - Map m = (Map)o; - assertTrue(m.get("key1") instanceof Bean); - assertTrue(m.get("key2") instanceof Bean); // NOTE these tests fail in snakeYAML 1.2 and below, fixed in 1.3 - assertTrue(m.get("key3") instanceof Custom); - assertTrue(m.get("key4") instanceof Wrapper); + assertTrue(map.get("key1") instanceof Bean); + assertTrue(map.get("key2") instanceof Bean); // NOTE these tests fail in snakeYAML 1.2 and below, fixed in 1.3 + assertTrue(map.get("key3") instanceof Custom); + assertTrue(map.get("key4") instanceof Wrapper); } public static class Bean { @@ -60,7 +52,6 @@ public static class Bean { public Bean() { a = ""; } public String getA() { return a; } public void setA(String s) { a = s; } - } public static class Wrapper { @@ -92,7 +83,6 @@ public Custom(Custom c) { public String toString() { return str; } } - public static class ExtendedRepresenter extends Representer { public ExtendedRepresenter() { this.representers.put(Custom.class, new RepresentCustom()); @@ -100,24 +90,22 @@ public ExtendedRepresenter() { private class RepresentCustom implements Represent { public Node representData(Object data) { - return representScalar("!Custom", ((Custom) data).toString()); + return representScalar(new Tag("!Custom"), data.toString()); } } } - public static class ExtendedConstructor extends Constructor { public ExtendedConstructor() { - this.yamlConstructors.put("!Custom", new ConstructCustom()); + this.yamlConstructors.put(new Tag("!Custom"), new ConstructCustom()); } private class ConstructCustom implements Construct { public Object construct(Node node) { - String str = (String) constructScalar((ScalarNode)node); + String str = constructScalar((ScalarNode)node); return new Custom(str); } public void construct2ndStep(Node node, Object object) { } } } - } diff --git a/test/plugins/Library/util/BTreeMapTest.java b/test/plugins/Library/util/BTreeMapTest.java index c555a59a..2fca4562 100644 --- a/test/plugins/Library/util/BTreeMapTest.java +++ b/test/plugins/Library/util/BTreeMapTest.java @@ -3,8 +3,6 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.util; -import junit.framework.TestCase; - import java.util.*; /** @@ -19,6 +17,14 @@ public class BTreeMapTest extends SortedMapTestSkeleton { final public static int sz0 = 0x400; final public static int sz1 = sz0<<2; //sz0<<6; + private static String rndStr() { + return UUID.randomUUID().toString(); + } + + private static String rndKey() { + return rndStr().substring(0,8); + } + public void testBasic() { BTreeMap testmap = new BTreeMap(0x40); @@ -26,7 +32,7 @@ public void testBasic() { try { for (int i=0; i backmap = new TreeMap(); for (int i=0; i skelmap; + String oneKey; + + private static long lastNumber = 0; + + private static synchronized Long getNextNumber() { + return new Long(++lastNumber); + } + + static class SkelMapNodeSerialiser + implements IterableSerialiser.SkeletonNode>, + ScheduledSerialiser.SkeletonNode> { + + final private Map store = Collections.synchronizedMap(new HashMap()); + SkelMapMapSerialiser mapSerialiser; + + Translator, Map> ttrans = new + SkeletonBTreeMap.TreeTranslator(null, null); + + SkeletonBTreeMap.NodeTranslator> ntrans; + + Translator, Map> tmtrans = new SkeletonTreeMap.TreeMapTranslator() { + + @Override + public Map app(SkeletonTreeMap translatee) { + return app(translatee, new TreeMap(), null); + } + + @Override + public SkeletonTreeMap rev(Map intermediate) throws DataFormatException { + return rev(intermediate, new SkeletonTreeMap(), null); + } + }; + + SkelMapNodeSerialiser(SkeletonBTreeMap skelmap, SkelMapMapSerialiser ms) { + mapSerialiser = ms; + ntrans = skelmap.makeNodeTranslator(null, tmtrans); + } + + @Override + public void pull(PullTask.SkeletonNode> task) throws TaskAbortException { + assert task != null; + assert task.meta != null; + assert task.meta instanceof SkeletonBTreeMap.GhostNode; + SkeletonBTreeMap.GhostNode gn = (SkeletonBTreeMap.GhostNode) task.meta; + assert gn.meta instanceof Long; + assert store.containsKey(gn.meta); + Map map = (Map) store.get(gn.meta); + SkeletonBTreeMap.SkeletonNode node; + try { + node = ntrans.rev(map); + } catch (DataFormatException e) { + throw new TaskAbortException("Unpacking SkeletonNode", e); + } + task.data = node; + } + + @Override + public void push(PushTask.SkeletonNode> task) throws TaskAbortException { + assert task.data.isBare(); + Map map = ntrans.app(task.data); + + Long pos = getNextNumber(); + store.put(pos, map); + task.meta = task.data.makeGhost(pos); + } + + @Override + public void pull(Iterable.SkeletonNode>> tasks) throws TaskAbortException { + throw new TaskAbortException("NIY", new Throwable()); + } + + @Override + public void push(Iterable.SkeletonNode>> tasks) throws TaskAbortException { + throw new TaskAbortException("NIY", new Throwable()); + } + + @Override + public ObjectProcessor.SkeletonNode>, E, TaskAbortException> pullSchedule( + BlockingQueue.SkeletonNode>> input, + BlockingQueue.SkeletonNode>, TaskAbortException>> output, + Map.SkeletonNode>, E> deposit) { + + return new ObjectProcessor.SkeletonNode>, E, TaskAbortException>(input, + output, deposit, null, Executors.DEFAULT_EXECUTOR, new TaskAbortExceptionConvertor()) { + @Override + protected Runnable createJobFor(final PullTask.SkeletonNode> task) { + return new Runnable() { + @Override + public void run() { + TaskAbortException ex = null; + try { + pull(task); + } catch (TaskAbortException e) { + ex = e; + } catch (RuntimeException e) { + ex = new TaskAbortException("pull failed", e); + } + postProcess.invoke(X2(task, ex)); + } + }; + } + }.autostart(); + + } + + @Override + public ObjectProcessor.SkeletonNode>, E, TaskAbortException> pushSchedule( + BlockingQueue.SkeletonNode>> input, + BlockingQueue.SkeletonNode>, TaskAbortException>> output, + Map.SkeletonNode>, E> deposit) { + ObjectProcessor.SkeletonNode>, E, TaskAbortException> objectProcessor = new ObjectProcessor.SkeletonNode>, E, TaskAbortException>( + input, output, deposit, null, Executors.DEFAULT_EXECUTOR, new TaskAbortExceptionConvertor()) { + @Override + protected Runnable createJobFor(final PushTask.SkeletonNode> task) { + return new Runnable() { + @Override + public void run() { + // Simulate push. + TaskAbortException ex = null; + try { + push(task); + } catch (TaskAbortException e) { + ex = e; + } catch (RuntimeException e) { + ex = new TaskAbortException("push failed", e); + } + postProcess.invoke(X2(task, ex)); + } + }; + } + }; + return objectProcessor.autostart(); + } + + } + + static class SkelMapMapSerialiser implements MapSerialiser { + final private Map store = Collections.synchronizedMap(new HashMap()); + + @Override + public void pull(Map> tasks, Object mapmeta) throws TaskAbortException { + for (Map.Entry> en : tasks.entrySet()) { + en.getValue().data = ((Map) store.get(en.getValue().meta)).get(en.getKey()); + } + } + + @Override + public void push(Map> tasks, Object mapmeta) throws TaskAbortException { + Map map = new HashMap(); + for (Map.Entry> en : tasks.entrySet()) { + map.put(en.getKey(), en.getValue().data); + } + Long pos = getNextNumber(); + store.put(pos, map); + for (Map.Entry> en : tasks.entrySet()) { + en.getValue().meta = pos; + } + } + + } + + private static String rndStr() { + return UUID.randomUUID().toString(); + } + + private static String rndKey() { + return rndStr().substring(0,8); + } + + protected void setUp() throws TaskAbortException { + skelmap = new SkeletonBTreeMap(2); + SkelMapMapSerialiser mapSerialiser = new SkelMapMapSerialiser(); + skelmap.setSerialiser(new SkelMapNodeSerialiser(skelmap, mapSerialiser), mapSerialiser); + assertTrue(skelmap.isBare()); + } + + private void add(int count, int laps) throws TaskAbortException { + int calculatedSize = skelmap.size(); + for (int l = 0; l < laps; ++l) { + SortedMap map = new TreeMap(); + for (int i = 0; i < count; ++i) { + String key = rndKey(); + map.put(key, i); + oneKey = key; + } + skelmap.update(map, new TreeSet()); + calculatedSize += count; + assertTrue(skelmap.isBare()); + assertEquals(calculatedSize, skelmap.size()); + } + } + + public void testSetup() { + assertTrue(true); + } + + public void test1() throws TaskAbortException { + add(1, 1); + } + + public void test3() throws TaskAbortException { + add(3, 1); + } + + public void test4() throws TaskAbortException { + add(4, 1); + } + + public void test10() throws TaskAbortException { + add(10, 1); + } + + public void test100() throws TaskAbortException { + add(100, 1); + } + + public void BIGtest1000() throws TaskAbortException { + add(1000, 1); + } + + public void BIGtest10000() throws TaskAbortException { + add(10000, 1); + } + + public void test1x3() throws TaskAbortException { + add(1, 3); + } + + public void test1x4() throws TaskAbortException { + add(1, 4); + } + + public void test1x5() throws TaskAbortException { + add(1, 5); + } + + public void test6x5() throws TaskAbortException { + add(6, 5); + } + + public void test10x5() throws TaskAbortException { + add(10, 5); + } + + public void BIGtest10x50() throws TaskAbortException { + add(10, 50); + } +} diff --git a/test/plugins/Library/util/SkeletonTreeMapTest.java b/test/plugins/Library/util/SkeletonTreeMapTest.java index 5573a2e1..c14b630c 100644 --- a/test/plugins/Library/util/SkeletonTreeMapTest.java +++ b/test/plugins/Library/util/SkeletonTreeMapTest.java @@ -3,10 +3,12 @@ * http://www.gnu.org/ for further details of the GPL. */ package plugins.Library.util; -import junit.framework.TestCase; - import java.util.Map; import java.util.SortedMap; +import java.util.UUID; + +import plugins.Library.io.serial.MapSerialiser; +import plugins.Library.util.exec.TaskAbortException; /** ** @author infinity0 @@ -15,10 +17,18 @@ public class SkeletonTreeMapTest extends SortedMapTestSkeleton { SkeletonTreeMap skelmap; + private static String rndStr() { + return UUID.randomUUID().toString(); + } + + private static String rndKey() { + return rndStr().substring(0,8); + } + protected void setUp() { skelmap = new SkeletonTreeMap(); for (int i=0; i<1024; ++i) { - skelmap.putGhost(Generators.rndKey(), Boolean.FALSE); + skelmap.putGhost(rndKey(), Boolean.FALSE); } } @@ -123,4 +133,78 @@ public void testIncompleteValues() { } } + class SkelMapMapSerializer implements MapSerialiser { + + @Override + public void pull(Map> tasks, Object mapmeta) throws TaskAbortException { + for (Map.Entry> en : tasks.entrySet()) { + // Simulate existing contents + en.getValue().data = 12; + } + } + + @Override + public void push(Map> tasks, Object mapmeta) throws TaskAbortException { + // Simulate storage. + } + + } + + public void testInflateAndDeflate() throws TaskAbortException { + skelmap.setSerialiser(new SkelMapMapSerializer()); + assertFalse(skelmap.isLive()); + assertTrue(skelmap.isBare()); + skelmap.inflate(); + assertTrue(skelmap.isLive()); + assertFalse(skelmap.isBare()); + for (Map.Entry en : skelmap.entrySet()) { + assertTrue(skelmap.entrySet().contains(en)); + assertNotNull(skelmap.get(en.getKey())); + assertEquals(skelmap.get(en.getKey()), new Integer(12)); + } + + assertTrue(skelmap.isLive()); + assertFalse(skelmap.isBare()); + skelmap.deflate(); + assertFalse(skelmap.isLive()); + assertTrue(skelmap.isBare()); + for (Map.Entry en : skelmap.entrySet()) { + try { + skelmap.entrySet().contains(en); + } catch (DataNotLoadedException e) { + continue; + } + fail("Data was loaded for " + en); + } + } + + public void testSingleInflateAndDeflate() throws TaskAbortException { + skelmap.setSerialiser(new SkelMapMapSerializer()); + assertFalse(skelmap.isLive()); + assertTrue(skelmap.isBare()); + skelmap.inflate(skelmap.firstKey()); + assertFalse(skelmap.isLive()); + assertFalse(skelmap.isBare()); + for (Map.Entry en : skelmap.entrySet()) { + assertTrue(skelmap.entrySet().contains(en)); + assertNotNull(skelmap.get(en.getKey())); + assertEquals(skelmap.get(en.getKey()), new Integer(12)); + break; + } + + assertFalse(skelmap.isLive()); + assertFalse(skelmap.isBare()); + skelmap.deflate(skelmap.firstKey()); + assertFalse(skelmap.isLive()); + assertTrue(skelmap.isBare()); + for (Map.Entry en : skelmap.entrySet()) { + try { + skelmap.entrySet().contains(en); + } catch (DataNotLoadedException e) { + continue; + } + fail("Data was loaded for " + en); + } + } + } diff --git a/test/plugins/Library/util/SortedMapTestSkeleton.java b/test/plugins/Library/util/SortedMapTestSkeleton.java index daf687f7..86bd0779 100644 --- a/test/plugins/Library/util/SortedMapTestSkeleton.java +++ b/test/plugins/Library/util/SortedMapTestSkeleton.java @@ -6,8 +6,10 @@ import junit.framework.TestCase; import java.util.Map; +import java.util.Random; import java.util.SortedMap; import java.util.Iterator; +import java.util.UUID; /** ** TODO maybe make some tailMap, headMap, subMap tests @@ -20,10 +22,16 @@ abstract public class SortedMapTestSkeleton extends TestCase { abstract protected SortedMap makeTestMap(); + final private static Random rand = new Random(); + + private static String rndStr() { + return UUID.randomUUID().toString(); + } + public void fillTestMap() { testmap = makeTestMap(); for (int i=0; i<0x1000; ++i) { - testmap.put(Generators.rndStr(), Generators.rand.nextInt()); + testmap.put(rndStr(), rand.nextInt()); } } diff --git a/test/plugins/Library/util/SortedTest.java b/test/plugins/Library/util/SortedTest.java index 2dffc627..12b587fe 100644 --- a/test/plugins/Library/util/SortedTest.java +++ b/test/plugins/Library/util/SortedTest.java @@ -4,9 +4,6 @@ package plugins.Library.util; import junit.framework.TestCase; -import static plugins.Library.util.Generators.rand; - -import plugins.Library.util.Sorted.Inclusivity; import java.util.Random; import java.util.Iterator; @@ -14,6 +11,9 @@ import java.util.List; import java.util.SortedSet; import java.util.TreeSet; +import java.util.UUID; + +import plugins.Library.util.Sorted.Inclusivity; /** ** @author infinity0 @@ -55,6 +55,16 @@ public class SortedTest extends TestCase { {8,16,24,32} }; + final private static Random rand = new Random(); + + private static String rndStr() { + return UUID.randomUUID().toString(); + } + + private static String rndKey() { + return rndStr().substring(0,8); + } + public void testSplitPredefined() { SortedSet sep = new TreeSet(Arrays.asList(split_sep)); @@ -152,13 +162,13 @@ public void verifySplit(SortedSet subj, SortedSet sep) { protected void randomSelectSplit(int n, int k) { SortedSet subj = new TreeSet(); - for (int i=0; i sep = new TreeSet(); List candsep = Sorted.select(subj, k); assertTrue(candsep.size() == k); for (String key: candsep) { - sep.add((rand.nextInt(2) == 0)? key: Generators.rndKey()); + sep.add((rand.nextInt(2) == 0)? key: rndKey()); } assertTrue(sep.size() == k); diff --git a/uploader/.gitignore b/uploader/.gitignore new file mode 100644 index 00000000..5e56e040 --- /dev/null +++ b/uploader/.gitignore @@ -0,0 +1 @@ +/bin diff --git a/uploader/src/freenet/library/uploader/AdHocDataReader.java b/uploader/src/freenet/library/uploader/AdHocDataReader.java new file mode 100644 index 00000000..90f80d0d --- /dev/null +++ b/uploader/src/freenet/library/uploader/AdHocDataReader.java @@ -0,0 +1,183 @@ +package freenet.library.uploader; + +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.logging.Level; +import java.util.logging.Logger; + +import plugins.Library.io.FreenetURI; +import plugins.Library.io.YamlReaderWriter; +import plugins.Library.io.serial.Packer; +import plugins.Library.io.serial.Packer.BinInfo; + +class AdHocDataReader { + /** Logger. */ + private static final Logger logger = Logger.getLogger(AdHocDataReader.class.getName()); + + interface UriProcessor { + public FreenetURI getURI(); + public int getLevel(); + public boolean processUri(FreenetURI uri); + public void childrenSeen(int level, int foundChildren); + public void uriSeen(); + public void stringSeen(); + } + + /** + * Convert an object from the yaml to a FreenetURI. + * + * The object can be a FreenetURI already (new style) or a string. + * + * @param obj + * @return a FreenetURI + * @throws MalformedURLException + */ + private FreenetURI getFreenetURI(Object obj, UriProcessor uriProcessor) throws MalformedURLException { + FreenetURI u; + if (obj instanceof FreenetURI) { + u = (FreenetURI) obj; + uriProcessor.uriSeen(); + } else { + u = new FreenetURI((String) obj); + uriProcessor.stringSeen(); + } + return u; + } + + + + private int processBinInfoValues(Map entries, UriProcessor uriProcessor) + throws MalformedURLException { + int foundChildren = 0; + for (BinInfo value : entries.values()) { + try { + if (uriProcessor.processUri(getFreenetURI(value.getID(), uriProcessor))) { + foundChildren ++; + } + + } catch (ClassCastException e) { + throw new RuntimeException("Cannot process BinInfo value " + value.getID() + " for " + uriProcessor.getURI(), e); + } + } + return foundChildren; + } + + private int processSubnodes(Map map, UriProcessor uriProcessor) + throws MalformedURLException { + int foundChildren = 0; + Map subnodes = + (Map) map.get("subnodes"); + for (Object key : subnodes.keySet()) { + if (uriProcessor.processUri(getFreenetURI(key, uriProcessor))) { + foundChildren ++; + } + } + return foundChildren; + } + + void readAndProcessYamlData(InputStream inputStream, UriProcessor uriProcessor, int page_level) + throws IOException { + int foundChildren = 0; + try { + Object readObject = new YamlReaderWriter().readObject(inputStream); + Map map = ((LinkedHashMap) readObject); + if (map.containsKey("ttab") && + map.containsKey("utab") && + map.containsKey("totalPages")) { + Map map2 = (Map) map.get("ttab"); + if (map2.containsKey("entries")) { + Map entries = + (Map) map2.get("entries"); + foundChildren += processBinInfoValues(entries, uriProcessor); + if (logger.isLoggable(Level.FINER)) { + Map subnodes = + (Map) map2.get("subnodes"); + logger.log(Level.FINER, "Contains ttab.entries (level {0}) with {1} subnodes", new Object[] { + uriProcessor.getLevel(), + subnodes.size(), + }); + } + foundChildren += processSubnodes(map2, uriProcessor); + return; + } + } + if (map.containsKey("lkey") && + map.containsKey("rkey") && + map.containsKey("entries")) { + // Must separate map and array! + if (map.containsKey("subnodes")) { + throw new RuntimeException("This parsing is not complex enough to handle subnodes for terms for " + + uriProcessor.getURI()); + } + if (map.get("entries") instanceof Map) { + Map entries = + (Map) map.get("entries"); + logger.log(Level.FINE, + "Contains from {1} to {2} (level {0}) with {3} entries.", + new Object[] { + uriProcessor.getLevel(), + map.get("lkey"), + map.get("rkey"), + entries.size() + }); + foundChildren += processBinInfoValues(entries, uriProcessor); + return; + } + if (map.get("entries") instanceof ArrayList) { + // Assuming this is a list of TermPageEntries. + logger.log(Level.FINE, + "Contains from {1} to {2} (level {0}) with page entries.", + new Object[] { + uriProcessor.getLevel(), + map.get("lkey"), + map.get("rkey") + }); + return; + } + } + Entry entry = map.entrySet().iterator().next(); + if (entry.getValue() instanceof Map) { + Map map2 = (Map) entry.getValue(); + if (map2.containsKey("node_min") + && map2.containsKey("size") + && map2.containsKey("entries")) { + String first = null; + String last = null; + for (Entry contents : map.entrySet()) { + if (contents.getValue() instanceof Map) { + if (first == null) { + first = contents.getKey(); + } + last = contents.getKey(); + Map map3 = (Map) contents.getValue(); + if (map3.containsKey("subnodes")) { + foundChildren += processSubnodes(map3, uriProcessor); + } + continue; + } + throw new RuntimeException("Cannot process entries. Entry for " + contents.getKey() + " is not String=Map for " + + uriProcessor.getURI()); + } + logger.log(Level.FINER, "Starts with entry for {1} and ended with entry {2} (level {0}).", new Object[] { + uriProcessor.getLevel(), + first, + last, + }); + return; + } + } + logger.severe("Cannot understand contents: " + map); + System.exit(1); + } finally { + uriProcessor.childrenSeen(page_level, foundChildren); + logger.exiting(AdHocDataReader.class.toString(), + "receivedAllData added " + foundChildren + " to the queue."); + } + + } +} diff --git a/uploader/src/freenet/library/uploader/DirectoryCreator.java b/uploader/src/freenet/library/uploader/DirectoryCreator.java new file mode 100644 index 00000000..9e060022 --- /dev/null +++ b/uploader/src/freenet/library/uploader/DirectoryCreator.java @@ -0,0 +1,112 @@ +package freenet.library.uploader; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.net.MalformedURLException; +import java.util.Map; +import java.util.Map.Entry; + +import plugins.Library.index.ProtoIndex; +import plugins.Library.index.ProtoIndexComponentSerialiser; +import plugins.Library.index.ProtoIndexSerialiser; +import plugins.Library.index.TermEntry; +import plugins.Library.io.FreenetURI; +import plugins.Library.io.serial.LiveArchiver; +import plugins.Library.io.serial.Serialiser.PushTask; +import plugins.Library.util.SkeletonBTreeSet; +import plugins.Library.util.exec.SimpleProgress; +import plugins.Library.util.exec.TaskAbortException; + +class DirectoryCreator { + private ProtoIndex idxDisk; + private ProtoIndexComponentSerialiser leafsrlDisk; + private int countTerms; + private ProtoIndexSerialiser srlDisk; + private File newIndexDir; + private String nextIndexDirName; + + DirectoryCreator(File directory) { + int nextIndexDirNumber = 0; + do { + nextIndexDirNumber ++; + nextIndexDirName = UploaderPaths.DISK_DIR_PREFIX + nextIndexDirNumber; + newIndexDir = new File(directory, nextIndexDirName); + } while (newIndexDir.exists()); + System.out.println("Writing into directory " + nextIndexDirName); + newIndexDir.mkdir(); + srlDisk = ProtoIndexSerialiser.forIndex(newIndexDir); + LiveArchiver, SimpleProgress> archiver = + (LiveArchiver, SimpleProgress>) srlDisk.getChildSerialiser(); + leafsrlDisk = ProtoIndexComponentSerialiser.get(ProtoIndexComponentSerialiser.FMT_FILE_LOCAL, archiver); + try { + idxDisk = new ProtoIndex(new FreenetURI("CHK@"), "test", null, null, 0L); + } catch (MalformedURLException e) { + throw new AssertionError(e); + } + leafsrlDisk.setSerialiserFor(idxDisk); + + countTerms = 0; + } + + private static boolean writeStringTo(File filename, String uri) { + FileOutputStream fos = null; + try { + fos = new FileOutputStream(filename); + OutputStreamWriter osw = new OutputStreamWriter(fos, "UTF-8"); + osw.write(uri.toString()); + osw.close(); + fos = null; + return true; + } catch (IOException e) { + System.out.println("Failed to write to "+filename+" : "+uri+" : "+e); + return false; + } finally { + try { + if (fos != null) { + fos.close(); + } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + public void putEntry(TermEntry tt) throws TaskAbortException { + SkeletonBTreeSet tree; + if (idxDisk.ttab.containsKey(tt.subj)) { + // merge + tree = idxDisk.ttab.get(tt.subj); + } else { + tree = new SkeletonBTreeSet(ProtoIndex.BTREE_NODE_MIN); + leafsrlDisk.setSerialiserFor(tree); + } + tree.add(tt); + idxDisk.ttab.put(tt.subj, tree); + countTerms++; + } + + public int size() { + return idxDisk.ttab.size(); + } + + public void done() throws TaskAbortException { + for (Entry> entry : idxDisk.ttab.entrySet()) { + SkeletonBTreeSet tree = entry.getValue(); + tree.deflate(); + assert(tree.isBare()); + idxDisk.ttab.put(entry.getKey(), tree); + } + idxDisk.ttab.deflate(); + assert(idxDisk.ttab.isBare()); + PushTask task4 = new PushTask(idxDisk); + srlDisk.push(task4); + String uri = (String) task4.meta; + writeStringTo(new File(newIndexDir, UploaderPaths.LAST_DISK_FILENAME), uri); + System.out.println("Created new directory " + nextIndexDirName + + ", file root at " + uri + + " with " + countTerms + " terms."); + } +} diff --git a/uploader/src/freenet/library/uploader/DirectoryUploader.java b/uploader/src/freenet/library/uploader/DirectoryUploader.java new file mode 100644 index 00000000..4710f052 --- /dev/null +++ b/uploader/src/freenet/library/uploader/DirectoryUploader.java @@ -0,0 +1,652 @@ +package freenet.library.uploader; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.lang.Math; +import java.net.MalformedURLException; +import java.util.Date; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; + +import plugins.Library.Priority; +import plugins.Library.index.ProtoIndex; +import plugins.Library.index.ProtoIndexComponentSerialiser; +import plugins.Library.index.ProtoIndexSerialiser; +import plugins.Library.index.TermEntry; +import plugins.Library.io.FreenetURI; +import plugins.Library.io.serial.LiveArchiver; +import plugins.Library.io.serial.Serialiser.PullTask; +import plugins.Library.io.serial.Serialiser.PushTask; +import plugins.Library.util.SkeletonBTreeMap; +import plugins.Library.util.SkeletonBTreeSet; +import plugins.Library.util.TaskAbortExceptionConvertor; +import plugins.Library.util.exec.SimpleProgress; +import plugins.Library.util.exec.TaskAbortException; +import plugins.Library.util.func.Closure; + +import net.pterodactylus.fcp.ClientPut; +import net.pterodactylus.fcp.FcpAdapter; +import net.pterodactylus.fcp.FcpConnection; +import net.pterodactylus.fcp.PutFailed; +import net.pterodactylus.fcp.PutSuccessful; +import net.pterodactylus.fcp.URIGenerated; +import net.pterodactylus.fcp.UploadFrom; + +class DirectoryUploader implements Runnable { + + FcpConnection connection; + File directory; + boolean forceCreateUSK; + + DirectoryUploader(FcpConnection c, File d, boolean fcu) { + connection = c; + directory = d; + forceCreateUSK = fcu; + } + + public void run() { + mergeToFreenet(directory); + } + + private FreenetURI lastUploadURI; + private boolean uskUploadDone; + + static final int MAX_HANDLING_COUNT = 5; + // When pushing is broken, allow max handling to reach this level + // before stalling forever to prevent running out of disk space. + + /** The temporary on-disk index. We merge stuff into this until it + * exceeds a threshold size, then we create a new diskIdx and + * merge the old one into the idxFreenet. */ + ProtoIndex idxDisk; + + /** idxDisk gets merged into idxFreenet this long after the last + * merge completed. */ + static final long MAX_TIME = 24*60*60*1000L; + + /** idxDisk gets merged into idxFreenet after this many incoming + * updates from Spider. */ + static final int MAX_UPDATES = 16; + + /** idxDisk gets merged into idxFreenet after it has grown to this + * many terms. Note that the entire main tree of terms (not the + * sub-trees with the positions and urls in) must fit into memory + * during the merge process. */ + static final int MAX_TERMS = 100*1000; + + /** idxDisk gets merged into idxFreenet after it has grown to this + * many terms. Note that the entire main tree of terms (not the + * sub-trees with the positions and urls in) must fit into memory + * during the merge process. */ + static final int MAX_TERMS_NOT_UPLOADED = 10*1000; + + /** Maximum size of a single entry, in TermPageEntry count, on + * disk. If we exceed this we force an insert-to-freenet and move + * on to a new disk index. The problem is that the merge to + * Freenet has to keep the whole of each entry in RAM. This is + * only true for the data being merged in - the on-disk index - + * and not for the data on Freenet, which is pulled on + * demand. SCALABILITY */ + static final int MAX_DISK_ENTRY_SIZE = 10000; + + /** Max time without creating a new USK for the + * index. Creating the USK is in fact publishing the new version + * of the index. While not creating a new USK, the index could be + * updated with new CHKs several times without publishing. This is + * to avoid too many USKs created (saving time for the creation + * and for the clients). + */ + private static final int MAX_DAYS_WITHOUT_NEW_USK = 4; + + static final String DISK_DIR_PREFIX = "library-temp-index-"; + /** Directory the current idxDisk is saved in. */ + File idxDiskDir; + + ProtoIndexSerialiser srl = null; + String lastDiskIndexName; + /** The uploaded index on Freenet. This never changes, it just + * gets updated. */ + ProtoIndex idxFreenet; + + // private final SpiderIndexURIs spiderIndexURIs; + + long pushNumber; + static final String LAST_URL_FILENAME = "library.index.lastpushed.chk"; + static final String PRIV_URI_FILENAME = "library.index.privkey"; + static final String PUB_URI_FILENAME = "library.index.pubkey"; + static final String EDITION_FILENAME = "library.index.last-edition"; + + static final String LAST_DISK_FILENAME = "library.index.lastpushed.disk"; + static final String REMOVED_TERMS_FILENAME = "terms.to.remove.disk"; + + static final String BASE_FILENAME_PUSH_DATA = "library.index.data."; + + ProtoIndexSerialiser srlDisk = null; + + private boolean writeStringTo(File filename, String uri) { + FileOutputStream fos = null; + try { + fos = new FileOutputStream(filename); + OutputStreamWriter osw = new OutputStreamWriter(fos, "UTF-8"); + osw.write(uri); + osw.close(); + fos = null; + return true; + } catch (IOException e) { + System.out.println("Failed to write to "+filename+" : "+uri+" : "+e); + return false; + } finally { + try { + if (fos != null) { + fos.close(); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + static String readStringFrom(File file) { + String ret; + FileInputStream fis = null; + try { + fis = new FileInputStream(file); + BufferedReader br = new BufferedReader(new InputStreamReader(fis, "UTF-8")); + ret = br.readLine(); + fis.close(); + fis = null; + return ret; + } catch (IOException e) { + // Ignore + return null; + } finally { + try { + if (fis != null) { + fis.close(); + } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + + static final String INDEX_DOCNAME = "index.yml"; + + private ProtoIndexComponentSerialiser leafsrl; + + /** Merge a disk dir to an on-Freenet index. Usually called on + * startup, i.e. we haven't just created the on-disk index so we + * need to setup the ProtoIndex etc. */ + protected void mergeToFreenet(File diskDir) { + ProtoIndexSerialiser s = ProtoIndexSerialiser.forIndex(diskDir); + LiveArchiver,SimpleProgress> archiver = + (LiveArchiver,SimpleProgress>)(s.getChildSerialiser()); + ProtoIndexComponentSerialiser leaf = ProtoIndexComponentSerialiser.get(ProtoIndexComponentSerialiser.FMT_FILE_LOCAL, archiver); + String f = DirectoryUploader.readStringFrom(new File(diskDir, LAST_DISK_FILENAME)); + String rf = DirectoryUploader.readStringFrom(new File(diskDir, REMOVED_TERMS_FILENAME)); + if(f == null && rf == null) { + if(diskDir.list().length == 0) { + System.err.println("Directory " + diskDir + " is empty - removing. Nothing to merge."); + diskDir.delete(); + return; + } + // Ignore + System.err.println("Unable to merge old data "+diskDir); + return; + } + + ProtoIndex idxDisk = null; + if (f != null) { + try { + PullTask pull = new PullTask(f); + System.out.println("Pulling previous index "+f+" from disk so can update it."); + s.pull(pull); + idxDisk = pull.data; + if(idxDisk.getSerialiser().getLeafSerialiser() != archiver) + throw new IllegalStateException("Different serialiser: "+idxDisk.getSerialiser()+" should be "+archiver); + } catch (TaskAbortException e) { + System.err.println("Failed to download previous index for spider update: "+e); + e.printStackTrace(); + return; + } + } + + ProtoIndex removeIdxDisk = null; + if (rf != null) { + try { + PullTask pull = new PullTask(rf); + System.out.println("Pulling index " + f + " with terms for removal from disk."); + s.pull(pull); + removeIdxDisk = pull.data; + if (removeIdxDisk.getSerialiser().getLeafSerialiser() != archiver) + throw new IllegalStateException("Different serialiser: " + removeIdxDisk.getSerialiser() + " should be " + archiver); + } catch (TaskAbortException e) { + System.err.println("Failed to download previous index for spider update: " + e); + e.printStackTrace(); + return; + } + } + + mergeToFreenet(idxDisk, removeIdxDisk, diskDir); + } + + /** Delete everything in a directory. Only use this when we are + * *very sure* there is no important data below it! */ + private static boolean removeAll(File wd) { + if(!wd.isDirectory()) { + System.out.println("DELETING FILE "+wd); + if(!wd.delete() && wd.exists()) { + System.err.println("Could not delete file: " + wd); + return false; + } + } else { + for(File subfile: wd.listFiles()) { + if(!removeAll(subfile)) { + return false; + } + } + if(!wd.delete()) { + System.err.println("Could not delete directory: " + wd); + return false; + } + } + return true; + } + + /** + * Create a new USK automatically if the old one is older than a + * specific time. + */ + private static boolean createUSK() { + File editionFile = new File(EDITION_FILENAME); + long fileChanged = editionFile.lastModified(); + if (Long.valueOf(new Date().getTime() - fileChanged).doubleValue() / + Long.valueOf(TimeUnit.DAYS.toMillis(MAX_DAYS_WITHOUT_NEW_USK)).doubleValue() + > new Random(fileChanged).nextDouble()) { + return true; + } + return false; + } + + private final Object inflateSync = new Object(); + + /** + * Merge from an on-disk index to an on-Freenet index. + * + * @param diskToMerge + * The on-disk index with new terms. None if null. + * @param removeIdxDisk + * The on-disk index with terms to remove. None if null. + * @param diskDir + * The folder the on-disk index is stored in. + */ + protected void mergeToFreenet(ProtoIndex diskToMerge, ProtoIndex removeIdxDisk, File diskDir) { + assert diskToMerge != null || removeIdxDisk != null; + if (lastUploadURI == null) { + try { + String lastURI = readStringFrom(new File(LAST_URL_FILENAME)); + if (lastURI != null) { // lastURI == null: case when the index is created for the first time + lastUploadURI = new FreenetURI(lastURI); + } + } catch (MalformedURLException e) { + throw new RuntimeException("File contents of " + LAST_URL_FILENAME + " invalid.", e); + } + } + setupFreenetCacheDir(); + + makeFreenetSerialisers(); + + if (diskToMerge != null) { + updateOverallMetadata(diskToMerge); + } + + final SkeletonBTreeMap> newtrees = diskToMerge != null ? diskToMerge.ttab : new SkeletonBTreeMap>(12); + + // Do the upload + + // async merge + Closure>, TaskAbortException> clo = + createMergeFromTreeClosure(newtrees); + try { + long mergeStartTime = System.currentTimeMillis(); + assert(idxFreenet.ttab.isBare()); + TreeSet terms = getAllTerms(diskToMerge); + TreeSet termsToRemove = getAllTerms(removeIdxDisk); + System.out.println("Merging " + + terms.size() + + " terms from disk to Freenet and removing " + + termsToRemove.size() + + " terms..."); + assert(idxFreenet.ttab.isBare()); + long entriesAdded = terms.size(); + long entriesRemoved = termsToRemove.size(); + // Run the actual merge. + System.out.println("Start update"); + idxFreenet.ttab.update(terms, termsToRemove, clo, new TaskAbortExceptionConvertor()); + assert(idxFreenet.ttab.isBare()); + // Deflate the main tree. + System.out.println("Start deflate"); + newtrees.deflate(); + assert(diskToMerge.ttab.isBare()); + + // Push the top node to a CHK. + PushTask task4 = new PushTask(idxFreenet); + task4.meta = "Unknown"; + System.out.println("Start pushing"); + srl.push(task4); + + // Now wait for the inserts to finish. They are started + // asynchronously in the above merge. + LiveArchiver, SimpleProgress> arch = srl.getChildSerialiser(); + System.out.println("Start waiting"); + arch.waitForAsyncInserts(); + System.out.println("Done waiting"); + + long mergeEndTime = System.currentTimeMillis(); + System.out.println(entriesAdded + " entries added and " + entriesRemoved + " entries removed in " + (mergeEndTime - mergeStartTime) + " ms, root at " + task4.meta); + FreenetURI uri; + if (task4.meta instanceof FreenetURI) { + uri = (FreenetURI) task4.meta; + } else if (task4.meta instanceof String){ + uri = new FreenetURI((String) task4.meta); + } else { + throw new RuntimeException("Unknown uri " + task4.meta); + } + lastUploadURI = uri; + if(writeStringTo(new File(LAST_URL_FILENAME), uri.toString())) { + newtrees.deflate(); + diskToMerge = null; + terms = null; + termsToRemove = null; + System.out.println("Finished with disk index "+diskDir); + removeAll(diskDir); + } + + // Create the USK to redirect to the CHK at the top of the index. + if (forceCreateUSK || createUSK()) { + uploadUSKForFreenetIndex(uri); + } + + } catch (TaskAbortException e) { + throw new RuntimeException(e); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + private TreeSet getAllTerms(ProtoIndex index) { + TreeSet terms = new TreeSet(); + if (index != null) { + assert (index.ttab.isBare()); + Iterator it = + index.ttab.keySetAutoDeflate().iterator(); + while (it.hasNext()) { + terms.add(it.next()); + } + assert (terms.size() == index.ttab.size()); + assert (index.ttab.isBare()); + } + return terms; + } + + static String readFileLine(final String filename) { + File f = new File(filename); + FileInputStream fis; + try { + fis = new FileInputStream(f); + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new RuntimeException(); + } + BufferedReader br = null; + String line; + try { + br = new BufferedReader(new InputStreamReader(fis, "UTF-8")); + line = br.readLine(); + } catch (UnsupportedEncodingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new RuntimeException(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new RuntimeException(); + } finally { + try { + if (br != null) { + br.close(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return line; + } + + protected void writeFileLine(String filename, String string) { + File f = new File(filename); + FileOutputStream fos; + try { + fos = new FileOutputStream(f); + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new RuntimeException(); + } + BufferedWriter bw = null; + try { + bw = new BufferedWriter(new OutputStreamWriter(fos, "UTF-8")); + bw.write(string); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + try { + if (bw != null) { + bw.close(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + private void uploadUSKForFreenetIndex(FreenetURI uri) { + String insertURI = readFileLine(PRIV_URI_FILENAME); + String keyPart = insertURI.substring("freenet:SSK@".length()); + int lastEdition = 0; + try { + lastEdition = Integer.parseInt(readFileLine(EDITION_FILENAME)); + } catch (RuntimeException ignore) { } // FileNotFound + System.out.println("lastEdition: " + lastEdition); + final ClientPut usk = new ClientPut("USK@" + keyPart + "/" + (lastEdition + 1), + "USKupload", + UploadFrom.redirect); + usk.setTargetURI(uri.toString()); + uskUploadDone = false; + FcpAdapter fcpListener = new FcpAdapter() { + public void receivedPutFailed(FcpConnection fcpConnection, PutFailed result) { + assert fcpConnection == connection; + assert result != null; + System.out.println("Could not upload USK"); + uskUploadDone = true; + synchronized (usk) { + usk.notifyAll(); + } + } + + public void receivedPutSuccessful(FcpConnection fcpConnection, PutSuccessful result) { + assert fcpConnection == connection; + assert result != null; + System.out.println("USK uploaded"); + uskUploadDone = true; + synchronized (usk) { + usk.notifyAll(); + } + } + + public void receivedURIGenerated(FcpConnection fcpConnection, URIGenerated uriGenerated) { + assert fcpConnection == connection; + assert uriGenerated != null; + System.out.println("URI generated " + uriGenerated.getURI()); + int editionStartPos = uriGenerated.getURI().lastIndexOf('/') + 1; + writeFileLine(EDITION_FILENAME, uriGenerated.getURI().substring(editionStartPos)); + } + + }; + connection.addFcpListener(fcpListener); + + try { + connection.sendMessage(usk); + while (!uskUploadDone) { + synchronized (usk) { + usk.wait(); + } + } + } catch (InterruptedException e) { + System.err.println("Could not upload USK"); + System.exit(1); + } catch (IOException e) { + System.err.println("IO Exception when uploading USK"); + System.exit(1); + } finally { + connection.removeFcpListener(fcpListener); + } + } + + + /** Create a Closure which will merge the subtrees from one index + * (on disk) into the subtrees of another index (on Freenet). It + * will be called with each subtree from the on-Freenet index, and + * will merge data from the relevant on-disk subtree. Both + * subtrees are initially deflated, and should be deflated when we + * leave the method, to avoid running out of memory. + * @param newtrees The on-disk tree of trees to get data from. + * @return + */ + private Closure>, TaskAbortException> createMergeFromTreeClosure(final SkeletonBTreeMap> newtrees) { + return new + Closure>, TaskAbortException>() { + /*@Override**/ public void invoke(Map.Entry> entry) throws TaskAbortException { + String key = entry.getKey(); + SkeletonBTreeSet tree = entry.getValue(); + boolean newTree = false; + if (tree == null) { + entry.setValue(tree = makeEntryTree(leafsrl)); + newTree = true; + } + assert(tree.isBare()); + SortedSet data; + // Can't be run in parallel. + synchronized(inflateSync) { + newtrees.inflate(key, true); + SkeletonBTreeSet entries; + entries = newtrees.get(key); + // CONCURRENCY: Because the lower-level trees are + // packed by the top tree, the bottom trees + // (SkeletonBTreeSet's) are not independant of + // each other. When the newtrees inflate above + // runs, it can deflate a tree that is still in + // use by another instance of this + // callback. Therefore we must COPY IT AND DEFLATE + // IT INSIDE THE LOCK. + entries.inflate(); + data = new TreeSet(entries); + entries.deflate(); + assert(entries.isBare()); + } + if (tree != null) { + if (newTree) { + tree.addAll(data); + assert(tree.size() == data.size()); + } else { + tree.update(data, null); + // Note that it is possible for data.size() + + // oldSize != tree.size(), because we might be + // merging data we've already merged. But + // most of the time it will add up. + } + tree.deflate(); + assert(tree.isBare()); + } + } + }; + } + + /** Update the overall metadata for the on-Freenet index from the + * on-disk index. */ + private void updateOverallMetadata(ProtoIndex diskToMerge) { + idxFreenet.setName(diskToMerge.getName()); + idxFreenet.setOwnerEmail(diskToMerge.getOwnerEmail()); + idxFreenet.setOwner(diskToMerge.getOwner()); + // This is roughly accurate, it might not be exactly so if we + // process a bit out of order. + idxFreenet.setTotalPages(diskToMerge.getTotalPages() + Math.max(0,idxFreenet.getTotalPages())); + } + + /** Setup the serialisers for uploading to Freenet. These convert + * tree nodes to and from blocks on Freenet, essentially. */ + private void makeFreenetSerialisers() { + if(srl == null) { + srl = ProtoIndexSerialiser.forIndex(lastUploadURI, Priority.Bulk); + LiveArchiver,SimpleProgress> archiver = + (LiveArchiver,SimpleProgress>)(srl.getChildSerialiser()); + leafsrl = ProtoIndexComponentSerialiser.get(ProtoIndexComponentSerialiser.FMT_DEFAULT, archiver); + if(lastUploadURI == null) { + try { + idxFreenet = new ProtoIndex(new FreenetURI("CHK@"), "test", null, null, 0L); + } catch (MalformedURLException e) { + throw new AssertionError(e); + } + // FIXME more hacks: It's essential that we use the + // same FreenetArchiver instance here. + leafsrl.setSerialiserFor(idxFreenet); + } else { + try { + PullTask pull = new PullTask(lastUploadURI); + System.out.println("Pulling previous index "+lastUploadURI+" so can update it."); + srl.pull(pull); + System.out.println("Pulled previous index "+lastUploadURI+" - updating..."); + idxFreenet = pull.data; + if(idxFreenet.getSerialiser().getLeafSerialiser() != archiver) + throw new IllegalStateException("Different serialiser: "+idxFreenet.getSerialiser()+" should be "+leafsrl); + } catch (TaskAbortException e) { + System.err.println("Failed to download previous index for spider update: "+e); + e.printStackTrace(); + return; + } + } + } + } + + /** Set up the on-disk cache, which keeps a copy of everything we + * upload to Freenet, so we won't need to re-download it, which + * can be very slow and doesn't always succeed. */ + private void setupFreenetCacheDir() { + File dir = new File(UploaderPaths.LIBRARY_CACHE); + dir.mkdir(); + } + + protected static SkeletonBTreeSet makeEntryTree(ProtoIndexComponentSerialiser leafsrl) { + SkeletonBTreeSet tree = new SkeletonBTreeSet(ProtoIndex.BTREE_NODE_MIN); + leafsrl.setSerialiserFor(tree); + return tree; + } +} diff --git a/uploader/src/freenet/library/uploader/DownloadAll.java b/uploader/src/freenet/library/uploader/DownloadAll.java new file mode 100644 index 00000000..f229c487 --- /dev/null +++ b/uploader/src/freenet/library/uploader/DownloadAll.java @@ -0,0 +1,39 @@ +/* + */ + +/* + * Log levels used: + * None/Warning: Serious events and small problems. + * FINE: Stats for fetches and overview of contents of fetched keys. Minor events. + * FINER: Queue additions, length, ETA, rotations. + * FINEST: Really minor events. + */ + +package freenet.library.uploader; + +import java.net.MalformedURLException; + +import plugins.Library.io.FreenetURI; + +/** + * Class to download the entire index. + */ +public class DownloadAll { + public static void main(String[] argv) { + if (argv.length > 1 && argv[0].equals("--move")) { + try { + new FetchAllOnce(new FreenetURI(argv[1])).doMove(); + } catch (MalformedURLException e) { + e.printStackTrace(); + System.exit(2); + } + } else { + try { + new FetchAllOnce(new FreenetURI(argv[0])).doDownload(); + } catch (MalformedURLException e) { + e.printStackTrace(); + System.exit(2); + } + } + } +} diff --git a/uploader/src/freenet/library/uploader/DownloadOneEdition.java b/uploader/src/freenet/library/uploader/DownloadOneEdition.java new file mode 100644 index 00000000..34064faa --- /dev/null +++ b/uploader/src/freenet/library/uploader/DownloadOneEdition.java @@ -0,0 +1,1348 @@ +/* + */ + +/* + * Log levels used: + * None/Warning: Serious events and small problems. + * FINE: Stats for fetches and overview of contents of fetched keys. Minor events. + * FINER: Queue additions, length, ETA, rotations. + * FINEST: Really minor events. + */ + +package freenet.library.uploader; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.net.MalformedURLException; +import java.nio.file.Files; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Date; +import java.util.Formatter; +import java.util.HashSet; +import java.util.Queue; +import java.util.Random; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import plugins.Library.io.FreenetURI; + +import net.pterodactylus.fcp.AllData; +import net.pterodactylus.fcp.ClientGet; +import net.pterodactylus.fcp.ClientPut; +import net.pterodactylus.fcp.FcpAdapter; +import net.pterodactylus.fcp.FcpConnection; +import net.pterodactylus.fcp.GetFailed; +import net.pterodactylus.fcp.Priority; +import net.pterodactylus.fcp.PutFailed; +import net.pterodactylus.fcp.PutSuccessful; +import net.pterodactylus.fcp.SubscribeUSK; +import net.pterodactylus.fcp.SubscribedUSKUpdate; +import net.pterodactylus.fcp.URIGenerated; +import net.pterodactylus.fcp.Verbosity; + +/** + * Class to download the entire index and save it. + * + * When a newer USK is seen, stop the processing immediately and exit. + * + * If a non-downloadable part is encountered upload it from the saved parts or + * attempt to download later. + */ +class DownloadOneEdition { + /** Logger. */ + private static final Logger logger = Logger.getLogger(DownloadOneEdition.class.getName()); + + private ScheduledExecutorService FCPexecutors; + private ScheduledExecutorService otherExecutors; + private FcpConnection connection; + private boolean closingDown = false; + private File directory; + private File morePagesDirectory; + private CleanupOldFiles cleanUp = null; + private Unfetchables unfetchables = new Unfetchables(); + private int getterCounter = 0; + private int uploadCounter = 0; + + private AdHocDataReader reader = new AdHocDataReader(); + + private static final long OPERATION_GIVE_UP_TIME = TimeUnit.HOURS.toMillis(2); + + class RotatingQueue extends LinkedBlockingQueue { + /** + * Serializeable. + */ + private static final long serialVersionUID = -9157586651059771247L; + + public RotatingQueue(Random r) { + random = r; + } + + @Override + public E poll() { + int toRotate = 0; + int s = size(); + if (s > 0) { + toRotate = random.nextInt(s); + } + while (true) { + E taken; + + taken = super.poll(); + if (taken == null) { + return null; + } + if (--toRotate > 0) { + offer(taken); // Ignoring impossible false status. + continue; + } + return taken; + } + } + + @Override + public E poll(long l, TimeUnit u) { + throw new IllegalStateException("Not implemented"); + } + + @Override + public E take() throws InterruptedException { + E taken = poll(); + if (taken == null) { + return super.take(); + } + return taken; + } + } + + + /** + * A class to keep track of the Pages we work with. + */ + private class Page { + private final long START_DEFER_TIME = TimeUnit.HOURS.toMillis(4); + + private FreenetURI uri; + private int level = 0; + Date nextFetchAttempt = new Date(); + StringBuffer logAttempts; + private long timeToNextFetchAttempt; + + Page(FreenetURI u, Page p) { + uri = u; + if (p != null) { + level = p.level + 1; + } + nextFetchAttempt = new Date(); + logAttempts = new StringBuffer(); + timeToNextFetchAttempt = START_DEFER_TIME; + } + + FreenetURI getURI() { + return uri; + } + + int getLevel() { + return level; + } + + File getFile() { + return new File(directory, getURI().toString().replace("/", "__")); + } + + private void calculateNextFetchAttempt() { + nextFetchAttempt = new Date(new Date().getTime() + timeToNextFetchAttempt); + } + + void fetchFailed() { + timeToNextFetchAttempt += 1 + 2 * random.nextInt(Long.valueOf(timeToNextFetchAttempt).intValue()) - timeToNextFetchAttempt / 2; + calculateNextFetchAttempt(); + logAttempts.append("Failed at ").append(new Date()).append(" and deferred to ").append(nextFetchAttempt).append("\n"); + } + + boolean fetchAvailable() { + return new Date().after(nextFetchAttempt); + } + + void fetchTimerReset() { + timeToNextFetchAttempt = START_DEFER_TIME; + logAttempts = new StringBuffer(); + logAttempts.append("Deferred to ").append(new Date()).append("\n"); + calculateNextFetchAttempt(); + } + } + + class AvoidRecentFetchesQueue extends RotatingQueue { + /** + * Serializeable. + */ + private static final long serialVersionUID = 7608442014226987011L; + + AvoidRecentFetchesQueue(Random r) { + super(r); + } + + public Page pollNotDeferred() { + int maxLaps = size(); + if (maxLaps > 10) { + maxLaps = 6; + } + do { + Page page = poll(); + if (page == null) { + return page; + } + if (page.fetchAvailable()) { + return page; + } + logger.finest("Skipped page deferred until " + page.nextFetchAttempt); + offer(page); // Ignored impossible false status + } while (maxLaps-- > 0); + logger.finest("Did not find any not deferred page"); + return null; + } + } + + + + private Random random = new Random(); + private RotatingQueue toParse = new RotatingQueue(random); + private RotatingQueue toFetch = new RotatingQueue(random); + private AvoidRecentFetchesQueue toRefetchUnfetchable = new AvoidRecentFetchesQueue(random); + private RotatingQueue toRefetch = new RotatingQueue(random); + private AvoidRecentFetchesQueue toUploadUnfetchable = new AvoidRecentFetchesQueue(random); + + private int counterParseSuccess = 0; + private int counterParseFailed = 0; + private int counterFetchSuccess = 0; + private int counterFetchFailed = 0; + private int counterRefetchUnfetchableSuccess = 0; + private int counterRefetchUnfetchableFailed = 0; + private int counterRefetchSuccess = 0; + private int counterRefetchFailed = 0; + private int counterUploadUnfetchableSuccess = 0; + private int counterUploadUnfetchableFailed = 0; + private int counterRefetchUploadSuccess = 0; + private int counterRefetchUploadFailed = 0; + + + private static String STATISTICS_FORMAT_PREFIX = "%-21s%7d%7d%7d"; + + public synchronized final void logStatistics() { + StringBuffer sb = new StringBuffer(); + sb.append(statisticsLine("toParse", counterParseSuccess, counterParseFailed, toParse)); + sb.append(statisticsLine("toFetch", counterFetchSuccess, counterFetchFailed, toFetch)); + sb.append(statisticsLine("toRefetchUnfetchable", + counterRefetchUnfetchableSuccess, counterRefetchUnfetchableFailed, + toRefetchUnfetchable)); + int counterRefetchUpload = counterRefetchUploadSuccess + counterRefetchUploadFailed; + if (counterRefetchUpload > 0) { + sb.append(new Formatter().format(STATISTICS_FORMAT_PREFIX, + "RefetchUpload", + counterRefetchUpload, + counterRefetchUploadSuccess, + counterRefetchUploadFailed)).append("\n"); + } + sb.append(statisticsLine("toRefetch", + counterRefetchSuccess, counterRefetchFailed, + toRefetch)); + sb.append(statisticsLine("toUploadUnfetchable", + counterUploadUnfetchableSuccess, counterUploadUnfetchableFailed, + toUploadUnfetchable)); + if (cleanUp != null) { + cleanUp.addLog(sb); + } + if (unfetchables.size() > 0) { + sb.append("Unfetchables from previous run: " + unfetchables.size() + "\n"); + } + logger.info("Statistics:\n" + sb.toString() + "End Statistics."); + } + + private static String STATISTICS_FORMAT = STATISTICS_FORMAT_PREFIX + "%6d%5d%5d%6d%6d%5d%5d\n"; + + public final String statisticsLine(String r, int success, int failed, RotatingQueue rqp) { + int counter = success + failed; + if (rqp.size() > 0 || counter > 0) { + int arr[] = new int[12]; + for (Page p : rqp) { + arr[p.level]++; + } + Formatter formatter = new Formatter(); + String line = formatter.format(STATISTICS_FORMAT, r, counter, success, failed, + rqp.size(), arr[0], arr[1], arr[2], arr[3], arr[4], arr[5]) + .toString(); + formatter.close(); + return line; + } + return ""; + } + + private boolean fetch(final Page page) { + int counter; + synchronized (this) { + counter = ++getterCounter; + } + final String token = "Getter" + counter; + final ClientGet getter = new ClientGet(page.getURI().toString(), token); + getter.setPriority(Priority.prefetch); + getter.setVerbosity(Verbosity.NONE); + final boolean[] results = new boolean[1]; + results[0] = false; + FcpAdapter listener = new FcpAdapter() { + @Override + public void receivedAllData(FcpConnection c, AllData ad) { + assert c == connection; + assert ad != null; + if (!token.equals(ad.getIdentifier())) { + return; + } + logger.entering(DownloadOneEdition.class.toString(), "receivedAllData", "receivedAllData for " + token); + try { + Files.copy(ad.getPayloadInputStream(), page.getFile().toPath(), + StandardCopyOption.REPLACE_EXISTING); + } catch (IOException ioe) { + page.getFile().delete(); + synchronized (getter) { + getter.notify(); + } + return; + } + results[0] = true; + synchronized (getter) { + getter.notify(); + } + } + + @Override + public void receivedGetFailed(FcpConnection c, GetFailed gf) { + assert c == connection; + assert gf != null; + if (!token.equals(gf.getIdentifier())) { + return; + } + synchronized (getter) { + getter.notify(); + } + logger.fine("receivedGetFailed for " + token + " (" + page.getURI() + ")."); + } + + @Override + public void receivedSimpleProgress(FcpConnection c, net.pterodactylus.fcp.SimpleProgress sp) { + assert c == connection; + assert sp != null; + if (!token.equals(sp.getIdentifier())) { + return; + } + logger.finest("Progress for " + token + " (" + sp.getSucceeded() + "/" + sp.getRequired() + "/" + + sp.getTotal() + ")."); + } + }; + connection.addFcpListener(listener); + try { + connection.sendMessage(getter); + } catch (IOException e) { + logger.log(Level.SEVERE, "Exception", e); + return false; + } + synchronized (getter) { + try { + getter.wait(OPERATION_GIVE_UP_TIME); + } catch (InterruptedException e) { + if (!closingDown) { + logger.log(Level.SEVERE, "Exception", e); + } + return false; + } + } + connection.removeFcpListener(listener); + + boolean result = results[0]; + if (result) { + page.fetchTimerReset(); + } else { + page.fetchFailed(); + } + + return result; + } + + private void parse(final Page page) { + try { + reader.readAndProcessYamlData(new FileInputStream(page.getFile()), new AdHocDataReader.UriProcessor() { + @Override + public FreenetURI getURI() { + return page.getURI(); + } + + @Override + public int getLevel() { + return page.getLevel(); + } + + Set seen = new HashSet(); + + @Override + public boolean processUri(FreenetURI uri) { + if (seen.contains(uri)) { + return false; + } + seen.add(uri); + handleNew(new Page(uri, page)); + return true; + } + + @Override + public void uriSeen() { + } + + @Override + public void stringSeen() { + } + + @Override + public void childrenSeen(int level, int foundChildren) { + } + + + }, page.getLevel()); + } catch (IOException ioe) { + page.getFile().delete(); + } + } + + private boolean upload(final Page page) { + final boolean[] successfuls = new boolean[1]; + successfuls[0] = false; + int counter; + synchronized (this) { + counter = ++uploadCounter; + } + final String identifier = "Upload" + counter; + final ClientPut putter = new ClientPut("CHK@", identifier); + putter.setEarlyEncode(true); + putter.setPriority(net.pterodactylus.fcp.Priority.bulkSplitfile); + putter.setVerbosity(Verbosity.NONE); + final long dataLength = page.getFile().length(); + putter.setDataLength(dataLength); + + final FcpAdapter listener = new FcpAdapter() { + @Override + public void receivedURIGenerated(FcpConnection c, URIGenerated uriGenerated) { + assert c == connection; + assert uriGenerated != null; + if (!identifier.equals(uriGenerated.getIdentifier())) { + return; + } + FreenetURI chk = page.getURI(); + FreenetURI generatedURI; + try { + generatedURI = new FreenetURI(uriGenerated.getURI()); + } catch (MalformedURLException e) { + logger.severe( + "Were supposed to resurrect " + chk + " but the URI calculated to " + uriGenerated.getURI() + + " that is not possible to convert to an URI. Will upload anyway."); + return; + } + if (!generatedURI.equals(chk)) { + logger.severe("Were supposed to resurrect " + chk + " but the URI calculated to " + + uriGenerated.getURI() + ". " + "Will upload anyway."); + } else { + logger.finest("Resurrecting " + chk); + } + } + + @Override + public void receivedPutSuccessful(FcpConnection c, PutSuccessful putSuccessful) { + assert c == connection; + assert putSuccessful != null; + if (!identifier.equals(putSuccessful.getIdentifier())) { + return; + } + FreenetURI chk = page.getURI(); + FreenetURI generatedURI = null; + try { + try { + generatedURI = new FreenetURI(putSuccessful.getURI()); + } catch (MalformedURLException e) { + logger.severe("Uploaded " + putSuccessful.getURI() + " that is not possible to convert to an URI."); + return; + } + if (!generatedURI.equals(chk)) { + logger.severe("Uploaded " + putSuccessful.getURI() + " while supposed to upload " + chk + ". "); + return; + } + logger.finer("Uploaded " + chk); + successfuls[0] = true; + } finally { + synchronized (putter) { + putter.notify(); + } + } + } + + @Override + public void receivedPutFailed(FcpConnection c, PutFailed putFailed) { + assert c == connection; + assert putFailed != null; + if (!identifier.equals(putFailed.getIdentifier())) { + return; + } + FreenetURI chk = page.getURI(); + logger.severe("Uploaded " + chk + " failed."); + synchronized (putter) { + putter.notify(); + } + } + }; + connection.addFcpListener(listener); + FileInputStream in; + try { + in = new FileInputStream(page.getFile()); + putter.setPayloadInputStream(in); + connection.sendMessage(putter); + synchronized (putter) { + putter.wait(OPERATION_GIVE_UP_TIME); + } + in.close(); + in = null; + } catch (IOException | NullPointerException e) { + logger.log(Level.WARNING, "Upload failed for " + page.getFile(), e); + } catch (InterruptedException e) { + if (!closingDown) { + logger.log(Level.WARNING, "Upload interrupted for " + page.getFile(), e); + } + return false; + } finally { + connection.removeFcpListener(listener); + } + return successfuls[0]; + } + + private boolean doRefetchUnfetchable(Page page) { + boolean result = fetch(page); + if (result) { + add(toParse, page); + counterRefetchUnfetchableSuccess++; + } else { + add(toRefetchUnfetchable, page); + counterRefetchUnfetchableFailed++; + } + return result; + } + + private boolean doRefetchToUpload(Page page) { + boolean result = fetch(page); + if (result) { + add(toRefetch, page); + counterRefetchUploadSuccess++; + } else { + add(toUploadUnfetchable, page); + counterRefetchUploadFailed++; + } + return result; + } + + private boolean doRefetch(Page page) { + boolean result = fetch(page); + if (result) { + add(toRefetch, page); + counterRefetchSuccess++; + } else { + handleUnfetchable(page); + counterRefetchFailed++; + } + return result; + } + + private void handleNew(Page page) { + if (page.getFile().exists()) { + page.getFile().setLastModified(System.currentTimeMillis()); + if (cleanUp != null) { + cleanUp.remove(page.getFile()); + } + add(toParse, page); + } else if (unfetchables.check(page.getURI())) { + add(toRefetchUnfetchable, page); + } else { + add(toFetch, page); + } + } + + private boolean doFetch(Page page) { + boolean result = fetch(page); + if (result) { + add(toParse, page); + counterFetchSuccess++; + } else { + handleUnfetchable(page); + counterFetchFailed++; + } + return result; + } + + private void doParse(Page page) { + parse(page); + if (unfetchables.check(page.getURI())) { + add(toUploadUnfetchable, page); + counterParseFailed++; + } else { + add(toRefetch, page); + counterParseSuccess++; + } + } + + private void handleUnfetchable(Page page) { + if (page.getFile().exists()) { + add(toUploadUnfetchable, page); + } else { + add(toRefetchUnfetchable, page); + } + } + + private boolean doUploadUnfetchable(Page page) { + boolean result = upload(page); + add(toRefetch, page); + if (result) { + counterUploadUnfetchableSuccess++; + } else { + counterUploadUnfetchableFailed++; + } + return result; + } + + private void doCopyAndUploadUnfetchable(Page page) { + File fromFile = new File(morePagesDirectory, page.getFile().getName()); + try { + Files.copy(fromFile.toPath(), page.getFile().toPath()); + boolean result = doUploadUnfetchable(page); + logger.finer("Uploaded Unfetchable" + (result ? "" : "failed") + "."); + } catch (UnsupportedOperationException uoe) { + logger.log(Level.SEVERE, "Could not copy file " + fromFile + " to " + page.getFile() + ".", uoe); + toRefetchUnfetchable.offer(page); + } catch (FileAlreadyExistsException faee) { + logger.log(Level.SEVERE, "Could not copy file " + fromFile + " to " + page.getFile() + ".", faee); + toUploadUnfetchable.offer(page); + } catch (IOException ioe) { + logger.log(Level.SEVERE, "Could not copy file " + fromFile + " to " + page.getFile() + ".", ioe); + if (page.getFile().exists()) { + page.getFile().delete(); + logger.info("Deleted partial copy " + page.getFile()); + } + toRefetchUnfetchable.offer(page); + } catch (SecurityException se) { + logger.log(Level.SEVERE, "Could not copy file " + fromFile + " to " + page.getFile() + ".", se); + toRefetchUnfetchable.offer(page); + } + } + + private void add(RotatingQueue whereto, Page p) { + whereto.offer(p); + } + + private class CleanupOldFiles implements Runnable { + private final Set allFiles = new HashSet(); + private ScheduledFuture handle = null; + private int count = 1; + + public CleanupOldFiles() { + allFiles.addAll(Arrays.asList(directory.listFiles())); + } + + public ScheduledFuture setHandle(ScheduledFuture h) { + handle = h; + return h; + } + + public void addLog(StringBuffer sb) { + if (allFiles.size() > 0) { + sb.append("Files left to remove: " + allFiles.size() + "\n"); + } + } + + public void remove(File f) { + allFiles.remove(f); + } + + public void run() { + if (toParse.size() > 0) { + // Don't delete anything if the parsing is not completed. + count = 1; + return; + } + if (toFetch.size() > 0) { + // Don't delete anything if the fetching is not completed. + count = 1; + return; + } + if (allFiles.size() == 0) { + if (handle != null) { + handle.cancel(true); + handle = null; + } + return; + } + // Sort in oldest order. + SortedSet toRemove = new TreeSet(new Comparator() { + @Override + public int compare(File o1, File o2) { + int l = Long.compare(o1.lastModified(), o2.lastModified()); + if (l != 0) { + return l; + } + return o1.getName().compareTo(o2.getName()); + } + }); + for (File f : allFiles) { + toRemove.add(f); + if (toRemove.size() > count) { + toRemove.remove(toRemove.last()); + } + } + for (File f : toRemove) { + allFiles.remove(f); + try { + unfetchables.check(new FreenetURI(f.getName())); + } catch (MalformedURLException e) { + logger.log(Level.WARNING, "File " + f + " strange filename.", e); + } + if (f.exists()) { + logger.fine("Removing file " + f); + f.delete(); + } + } + count += 1 + count / 7; + } + } + + /** + * Class to keep track of uploads from the previous run. + */ + private static class Unfetchables { + private Set fromPreviousRun = new HashSet(); + private final static String UNFETCHABLES_FILENAME = "unfetchables.saved"; + private File directory; + + @SuppressWarnings("unchecked") + private Set extracted(ObjectInputStream ois) throws IOException, ClassNotFoundException { + return (Set) ois.readObject(); + } + + void load(File dir) { + directory = dir; + File file = new File(directory, UNFETCHABLES_FILENAME); + if (file.exists()) { + logger.finest("Reading file " + file); + try { + FileInputStream f = new FileInputStream(file); + ObjectInputStream ois = new ObjectInputStream(f); + fromPreviousRun = extracted(ois); + ois.close(); + } catch (IOException e) { + logger.warning("Could not read the file " + file); + } catch (ClassCastException | ClassNotFoundException e) { + logger.warning("File " + file + " contains strange object"); + } finally { + file.delete(); + } + } else { + logger.finest("No file " + file); + } + } + + private void rotate() { + String OLD_FILENAME = UNFETCHABLES_FILENAME + ".old"; + File oldfile = new File(directory, OLD_FILENAME); + if (oldfile.exists()) { + oldfile.delete(); + } + File file = new File(directory, UNFETCHABLES_FILENAME); + if (file.exists()) { + file.renameTo(oldfile); + } + } + + synchronized void save(RotatingQueue toSave1, RotatingQueue toSave2) { + rotate(); + File file = new File(directory, UNFETCHABLES_FILENAME); + Set set = new HashSet(); + for (Page p : toSave1) { + set.add(p.getURI()); + } + if (toSave2 != null) { + for (Page p : toSave2) { + set.add(p.getURI()); + } + } + if (set.size() > 0) { + logger.finest("Writing file " + file); + try { + FileOutputStream f = new FileOutputStream(file); + ObjectOutputStream oos = new ObjectOutputStream(f); + oos.writeObject(set); + oos.close(); + f.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Problem writing file " + file, e); + file.delete(); + } + } else { + logger.finest("Nothing to write to file " + file); + } + } + + synchronized int size() { + return fromPreviousRun.size(); + } + + synchronized boolean check(FreenetURI uri) { + boolean retval = fromPreviousRun.contains(uri); + fromPreviousRun.remove(uri); + return retval; + } + } + + private abstract class ProcessSomething implements Runnable { + protected abstract void process(); + + public void run() { + try { + process(); + } catch (RejectedExecutionException e) { + // Do nothing. + if (!closingDown) { + logger.log(Level.SEVERE, "Confusion in the executor or queue full.", e); + } + } catch (Exception e) { + logger.log(Level.SEVERE, "Class " + this + " threw exception: " + e, e); + } + } + } + + private class ProcessParse extends ProcessSomething { + protected void process() { + Page page = toParse.poll(); + if (page != null) { + doParse(page); + otherExecutors.schedule(this, 200, TimeUnit.MILLISECONDS); + } else { + otherExecutors.schedule(this, 10, TimeUnit.SECONDS); + } + } + } + + private class ProcessUploadUnfetchable extends ProcessSomething { + protected void process() { + if (morePagesDirectory != null) { + Page page = toRefetchUnfetchable.poll(); + if (page != null) { + doCopyAndUploadUnfetchable(page); + return; + } + } + + Page page = toUploadUnfetchable.poll(); + if (page != null) { + boolean result = doUploadUnfetchable(page); + logger.finer("Uploaded Unfetchable" + (result ? "" : "failed") + "."); + return; + } + } + } + + /** + * This is the bulk of all fetches. + * + * Mostly Fetch, if any, but sometimes one of the refetches. + */ + private class ProcessFetches extends ProcessSomething { + protected void process() { + Page page = toFetch.poll(); + if (page != null) { + boolean result = doFetch(page); + logger.finest("Fetched Fetch" + (result ? "" : " failed") + "."); + return; + } + + page = toRefetchUnfetchable.pollNotDeferred(); + if (page != null) { + String log = page.logAttempts.toString(); + boolean result = doRefetchUnfetchable(page); + logger.finer(log + "Fetched RefetchUnfetchable" + (result ? "" : " failed") + "."); + return; + } + + page = toUploadUnfetchable.pollNotDeferred(); + if (page != null) { + String log = page.logAttempts.toString(); + boolean result = doRefetchToUpload(page); + logger.finer(log + "Fetched ToUpload" + (result ? "" : " failed") + "."); + return; + } + + page = toRefetch.poll(); + if (page != null) { + boolean result = doRefetch(page); + logger.finer("Fetched Refetch" + (result ? "" : " failed") + "."); + return; + } + } + } + + void shutdown(FcpSession session) { + waitTermination(TimeUnit.SECONDS.toMillis(1)); + closingDown = true; + logger.info("Shutdown."); + FCPexecutors.shutdown(); + otherExecutors.shutdown(); + unfetchables.save(toUploadUnfetchable, toRefetchUnfetchable); + if (!waitTermination(TimeUnit.MINUTES.toMillis(1) + OPERATION_GIVE_UP_TIME)) { + logger.info("Shutdown now (after long wait)."); + FCPexecutors.shutdownNow(); + otherExecutors.shutdownNow(); + session.close(); + if (!waitTermination(TimeUnit.MINUTES.toMillis(1))) { + logger.info("Shutdown now did not succeed to stop all jobs"); + } + } + FCPexecutors = null; + otherExecutors = null; + session.close(); + session = null; + logger.info("Shutdown completed."); + } + + private void run(FreenetURI u, File morePagesDir) { + morePagesDirectory = morePagesDir; + FCPexecutors = Executors.newScheduledThreadPool(10); + otherExecutors = Executors.newScheduledThreadPool(1); + directory = new File("library-download-all-once-db"); + if (directory.exists()) { + unfetchables.load(directory); + cleanUp = new CleanupOldFiles(); + cleanUp.setHandle(otherExecutors.scheduleWithFixedDelay(cleanUp, 500, 1, TimeUnit.MINUTES)); + } else { + directory.mkdir(); + } + + otherExecutors.scheduleAtFixedRate(new Runnable() { + public void run() { + logStatistics(); + } + }, 10, 30, TimeUnit.SECONDS); + for (int i = 0; i < 10; i++) { + FCPexecutors.scheduleWithFixedDelay(new ProcessFetches(), 20 + i, 4, TimeUnit.SECONDS); + } + for (int i = 0; i < 4; i++) { + FCPexecutors.scheduleWithFixedDelay(new ProcessUploadUnfetchable(), 40 + i, 1, TimeUnit.SECONDS); + } + otherExecutors.schedule(new ProcessParse(), 2000, TimeUnit.MILLISECONDS); + otherExecutors.scheduleWithFixedDelay(new Runnable() { + public void run() { + unfetchables.save(toUploadUnfetchable, toRefetchUnfetchable); + } + }, 100, 20, TimeUnit.MINUTES); + FcpSession session; + try { + session = new FcpSession("DownloadAllOnceFor" + u); + } catch (IllegalStateException | IOException e1) { + logger.log(Level.SEVERE, "Exception", e1); + return; + } + try { + startAndBlockUntilUpdate(session, u); + } finally { + shutdown(session); + } + } + + private class ParseQueues implements Runnable { + private long count = 0; + private long sum = 0; + + @Override + public void run() { + final Page page = toParse.poll(); + if (page != null) { + otherExecutors.execute(new Runnable() { + @Override + public void run() { + Date start = new Date(); + doParse(page); + count++; + sum += new Date().getTime() - start.getTime(); + } + }); + + otherExecutors.execute(this); + } else { + otherExecutors.schedule(this, 10, TimeUnit.SECONDS); + } + } + + long getMean() { + if (count == 0) { + return 1; + } + return sum / count; + } + } + + private class QueueQueues implements Runnable { + + private boolean startedFetch = false; + private boolean startedUpload = false; + + private Random random = new Random(); + + private long nextLong(long l) { + if (l <= 0) { + l = 1L; + } + return (random.nextLong() >>> 1) % l; + } + + private boolean shallUpload(int ql) { + if (!startedFetch) { + return true; + } + return nextLong(uploadTime.get() * (toFetch.size() + toRefetchUnfetchable.size()) / (1 + ql) + / fetchTime.get()) == 0; + } + + private class Mean { + private Queue whereToRead; + private long count; + private long sum; + + Mean(Queue w) { + whereToRead = w; + count = 0L; + sum = 0L; + } + + private void consume() { + boolean done = false; + do { + Long found = whereToRead.poll(); + if (found != null) { + count += 1; + sum += found.longValue(); + } else { + done = true; + } + } while (!done); + } + + long get() { + consume(); + if (count == 0) { + return 1; + } + return sum / count; + } + } + + private final Queue fetchTimes = new LinkedBlockingQueue(); + private final Mean fetchTime = new Mean(fetchTimes); + private final Queue uploadTimes = new LinkedBlockingQueue(); + private final Mean uploadTime = new Mean(uploadTimes); + + private class MeasureTime { + private Queue whereToPost; + private Date start; + + MeasureTime(Queue w) { + whereToPost = w; + start = new Date(); + } + + void done() { + whereToPost.offer(new Date().getTime() - start.getTime()); + } + } + + private void queueFetch() { + { + final Page page = toFetch.poll(); + if (page != null) { + FCPexecutors.execute(new Runnable() { + @Override + public void run() { + MeasureTime t = new MeasureTime(fetchTimes); + boolean result = doFetch(page); + t.done(); + logger.finest("Fetched Fetch" + (result ? "" : " failed") + "."); + } + }); + startedFetch = true; + } + } + + if (!startedFetch) { + final Page page = toRefetchUnfetchable.pollNotDeferred(); + if (page != null) { + FCPexecutors.execute(new Runnable() { + @Override + public void run() { + String log = page.logAttempts.toString(); + MeasureTime t = new MeasureTime(fetchTimes); + boolean result = doRefetchUnfetchable(page); + t.done(); + logger.finer(log + "Fetched RefetchUnfetchable" + (result ? "" : " failed") + "."); + } + }); + startedFetch = true; + } + } + + { + final Page page = toUploadUnfetchable.pollNotDeferred(); + if (page != null) { + FCPexecutors.execute(new Runnable() { + @Override + public void run() { + String log = page.logAttempts.toString(); + MeasureTime t = new MeasureTime(fetchTimes); + boolean result = doRefetchToUpload(page); + t.done(); + logger.finer(log + "Fetched ToUpload" + (result ? "" : " failed") + "."); + } + }); + startedFetch = true; + } + } + + if (!startedFetch) { + final Page page = toRefetch.poll(); + if (page != null) { + FCPexecutors.execute(new Runnable() { + @Override + public void run() { + MeasureTime t = new MeasureTime(fetchTimes); + boolean result = doRefetch(page); + t.done(); + logger.finer("Fetched Refetch" + (result ? "" : " failed") + "."); + } + }); + } + } + } + + private void queueUpload() { + if (morePagesDirectory != null && shallUpload(toRefetchUnfetchable.size())) { + final Page page = toRefetchUnfetchable.poll(); + if (page != null) { + FCPexecutors.execute(new Runnable() { + @Override + public void run() { + MeasureTime t = new MeasureTime(uploadTimes); + doCopyAndUploadUnfetchable(page); + t.done(); + } + }); + startedUpload = true; + } + } + + if (shallUpload(toUploadUnfetchable.size())) { + final Page page = toUploadUnfetchable.poll(); + if (page != null) { + FCPexecutors.execute(new Runnable() { + @Override + public void run() { + MeasureTime t = new MeasureTime(uploadTimes); + boolean result = doUploadUnfetchable(page); + t.done(); + logger.finer("Uploaded Unfetchable" + (result ? "" : "failed") + "."); + } + }); + startedUpload = true; + } + } + } + + @Override + public void run() { + startedFetch = false; + startedUpload = false; + + queueFetch(); + queueUpload(); + + if (startedFetch || startedUpload) { + FCPexecutors.execute(this); + } else { + FCPexecutors.schedule(this, 10, TimeUnit.SECONDS); + } + } + + public Object getFetchMean() { + return fetchTime.get(); + } + + public Object getUploadMean() { + return uploadTime.get(); + } + } + + private void run2(int numThreads, FreenetURI u, File morePagesDir) { + morePagesDirectory = morePagesDir; + FCPexecutors = Executors.newScheduledThreadPool(numThreads); + otherExecutors = Executors.newScheduledThreadPool(1); + directory = new File("library-download-all-once-db"); + if (directory.exists()) { + unfetchables.load(directory); + cleanUp = new CleanupOldFiles(); + cleanUp.setHandle(FCPexecutors.scheduleWithFixedDelay(cleanUp, 500, 1, TimeUnit.MINUTES)); + } else { + directory.mkdir(); + } + + final ParseQueues pq = new ParseQueues(); + otherExecutors.execute(pq); + final QueueQueues qq = new QueueQueues(); + otherExecutors.scheduleWithFixedDelay(new Runnable() { + public void run() { + logStatistics(); + logger.log(Level.INFO, "Parse time: {0} Fetch time: {1} Upload time: {2}", + new Object[] { + pq.getMean(), qq.getFetchMean(), qq.getUploadMean() + }); + } + }, 1, 1, TimeUnit.MINUTES); + FCPexecutors.schedule(qq, 2, TimeUnit.SECONDS); + FCPexecutors.scheduleWithFixedDelay(new Runnable() { + public void run() { + unfetchables.save(toUploadUnfetchable, toRefetchUnfetchable); + } + }, 100, 20, TimeUnit.MINUTES); + FcpSession session; + try { + session = new FcpSession("DownloadOneEditionFor" + u); + } catch (IllegalStateException | IOException e1) { + logger.log(Level.SEVERE, "Exception", e1); + return; + } + try { + startAndBlockUntilUpdate(session, u); + } finally { + shutdown(session); + } + } + + private void startAndBlockUntilUpdate(FcpSession session, FreenetURI uri) { + connection = session.getConnection(); + if (connection == null) { + throw new IllegalArgumentException("No connection."); + } + final SubscribeUSK subscriber = new SubscribeUSK(uri + "-1", "USK"); + subscriber.setActive(true); + final int[] editions = new int[1]; + final FreenetURI[] newUris = new FreenetURI[1]; + editions[0] = 0; + FcpAdapter listener = new FcpAdapter() { + @Override + public void receivedSubscribedUSKUpdate(FcpConnection fcpConnection, + SubscribedUSKUpdate subscribedUSKUpdate) { + assert fcpConnection == connection; + FreenetURI newUri; + try { + newUri = new FreenetURI(subscribedUSKUpdate.getURI()); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + if (/*FIXME subscribedUSKUpdate.isNewKnownGood() &&*/ !newUri.equals(newUris[0])) { + newUris[0] = newUri; + editions[0] = subscribedUSKUpdate.getEdition(); + synchronized (subscriber) { + subscriber.notify(); + } + } + } + }; + connection.addFcpListener(listener); + + synchronized (subscriber) { + try { + connection.sendMessage(subscriber); + subscriber.wait(); // Wait until found + handleNew(new Page(newUris[0], null)); + subscriber.wait(); // Work until next one found + logger.info("Next edition seen."); + } catch (InterruptedException e) { + throw new RuntimeException("Subscription interrupted."); + } catch (IOException e) { + throw new RuntimeException("Subscription can't write."); + } + } + } + + private boolean waitTermination(long ms) { + boolean t1 = false; + boolean t2 = false; + try { + t1 = FCPexecutors.awaitTermination(ms, TimeUnit.MILLISECONDS); + t2 = otherExecutors.awaitTermination(1 + ms / 10, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new RuntimeException("Waiting for jobs."); + } + return t1 && t2; + } + + public static void main(String[] argv) throws InterruptedException { + Integer numThreads; + try { + numThreads = new Integer(argv[0]); + } catch (NumberFormatException e) { + logger.log(Level.SEVERE, "First parameter must be a number, was " + argv[0] + ".", e); + System.exit(2); + return; + } + + FreenetURI u; + try { + u = new FreenetURI(argv[1]); + } catch (MalformedURLException e) { + logger.log(Level.SEVERE, "Exception", e); + System.exit(2); + return; + } + + File morePagesDir = null; + if (argv.length > 2) { + morePagesDir = new File(argv[2]); + if (!morePagesDir.exists()) { + logger.severe("Directory " + morePagesDir + " does not exist."); + System.exit(2); + return; + } + if (!morePagesDir.isDirectory()) { + logger.severe("File " + morePagesDir + " is not a directory."); + System.exit(2); + return; + } + } + + if (numThreads.intValue() == 0) { + new DownloadOneEdition().run(u, morePagesDir); + } else { + logger.info("Running with " + numThreads.intValue() + " threads."); + new DownloadOneEdition().run2(numThreads.intValue(), u, morePagesDir); + } + } +} diff --git a/uploader/src/freenet/library/uploader/FcpArchiver.java b/uploader/src/freenet/library/uploader/FcpArchiver.java new file mode 100644 index 00000000..211a115f --- /dev/null +++ b/uploader/src/freenet/library/uploader/FcpArchiver.java @@ -0,0 +1,439 @@ +package freenet.library.uploader; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import freenet.support.Base64; +import freenet.crypt.SHA256; + +import plugins.Library.Priority; +import plugins.Library.io.FreenetURI; +import plugins.Library.io.ObjectStreamReader; +import plugins.Library.io.ObjectStreamWriter; +import plugins.Library.io.serial.LiveArchiver; +import plugins.Library.util.exec.SimpleProgress; +import plugins.Library.util.exec.TaskAbortException; + +import net.pterodactylus.fcp.ClientPut; +import net.pterodactylus.fcp.FcpAdapter; +import net.pterodactylus.fcp.FcpConnection; +import net.pterodactylus.fcp.PutFailed; +import net.pterodactylus.fcp.PutFetchable; +import net.pterodactylus.fcp.PutSuccessful; +import net.pterodactylus.fcp.URIGenerated; +import net.pterodactylus.fcp.Verbosity; + +public class FcpArchiver + implements LiveArchiver { + private FcpConnection connection; + private File cacheDir; + private ObjectStreamReader reader; + private ObjectStreamWriter writer; + private int totalBlocksStillUploading = 0; + private Priority priorityLevel; + + /** + * Before synchronizing on stillRunning, be sure to synchronize + * connection! + */ + private Map stillRunning = new HashMap<>(); + private Thread cleanupThread; + + public FcpArchiver(FcpConnection fcpConnection, + File directory, + S rw, + String mime, int s, + Priority pl) { + connection = fcpConnection; + cacheDir = directory; + reader = rw; + writer = rw; + priorityLevel = pl; + } + + private net.pterodactylus.fcp.Priority getPriority() { + switch (priorityLevel) { + case Interactive: + return net.pterodactylus.fcp.Priority.interactive; + case Bulk: + return net.pterodactylus.fcp.Priority.bulkSplitfile; + } + return net.pterodactylus.fcp.Priority.bulkSplitfile; + } + + @Override + public void pull(plugins.Library.io.serial.Serialiser.PullTask task) + throws TaskAbortException { + pullLive(task, null); + } + + @Override + public void push(plugins.Library.io.serial.Serialiser.PushTask task) + throws TaskAbortException { + pushLive(task, null); + } + + /** + * Initial implementation, fetch everything from the cache. This means + * that we cannot take over someone else's index. + */ + @Override + public void pullLive(plugins.Library.io.serial.Serialiser.PullTask task, + SimpleProgress progress) throws TaskAbortException { + if (cacheDir.exists()) { + String cacheKey = null; + if (task.meta instanceof FreenetURI) { + cacheKey = task.meta.toString(); + } else if (task.meta instanceof String) { + cacheKey = (String) task.meta; + } else if (task.meta instanceof byte[]) { + cacheKey = Base64.encode(SHA256.digest((byte[]) task.meta)); + } + + try { + if(cacheDir != null && cacheDir.exists() && cacheDir.canRead()) { + File cached = new File(cacheDir, cacheKey); + if(cached.exists() && + cached.length() != 0 && + cached.canRead()) { + InputStream is = new FileInputStream(cached); + task.data = (T) reader.readObject(is); + is.close(); + } + } + + if (progress != null) { + progress.addPartKnown(0, true); + } + } catch (IOException e) { + System.out.println("IOException:"); + e.printStackTrace(); + throw new TaskAbortException("Failed to read content from local tempbucket", e, true); + } + return; + } + throw new UnsupportedOperationException( + "Cannot find the key " + + task.meta + + " in the cache."); + } + + private static long lastUriMillis = 0; + + private class PushAdapter extends FcpAdapter { + private ClientPut putter; + private String identifier; + private String token; + private FreenetURI uri; + private int progressTotal; + private int progressCompleted; + private boolean done; + private long started; + + public PushAdapter(ClientPut p, String i, String t) { + putter = p; + identifier = i; + token = t; + uri = null; + progressTotal = 0; + progressCompleted = 0; + synchronized (stillRunning) { + stillRunning.put(token, this); + printLeft(); + } + started = System.currentTimeMillis(); + } + + /** + * Show the amount of outstanding work. + */ + void printLeft() { + int total = 0; + int completed = 0; + synchronized (stillRunning) { + for (Map.Entry entry : stillRunning.entrySet()) { + total += entry.getValue().progressTotal; + completed += entry.getValue().progressCompleted; + } + totalBlocksStillUploading = total - completed; + System.out.println("Outstanding " + stillRunning.size() + " jobs " + + "(" + completed + "/" + total + ")"); + } + } + + private String at() { + return " took " + (System.currentTimeMillis() - started) + "ms"; + } + + @Override + public void receivedPutSuccessful(FcpConnection c, PutSuccessful ps) { + assert c == connection; + assert ps != null; + if (!identifier.equals(ps.getIdentifier())) + return; + System.out.println("receivedPutSuccessful for " + token + at()); + markDone(); + } + + @Override + public void receivedPutFetchable(FcpConnection c, PutFetchable pf) { + assert c == connection; + assert pf != null; + if (!identifier.equals(pf.getIdentifier())) + return; + System.out.println("receivedPutFetchable for " + token + at()); + synchronized (this) { + this.notifyAll(); + } + } + + + @Override + public void receivedPutFailed(FcpConnection c, PutFailed pf) { + assert c == connection; + assert pf != null; + if (!identifier.equals(pf.getIdentifier())) + return; + synchronized (putter) { + putter.notify(); + } + System.err.println("receivedPutFailed for " + token + at() + " aborting."); + markDone(); + System.exit(1); + } + + @Override + public void receivedSimpleProgress(FcpConnection c, + net.pterodactylus.fcp.SimpleProgress sp) { + assert c == connection; + assert sp != null; + if (!identifier.equals(sp.getIdentifier())) + return; + if (sp.getFailed() > 0 || + sp.getFatallyFailed() > 0) { + System.out.println(token + "failed - aborted."); + } + progressCompleted = sp.getSucceeded(); + progressTotal = sp.getTotal(); + printLeft(); + synchronized (stillRunning) { + stillRunning.notifyAll(); + } + } + + public void receivedURIGenerated(FcpConnection c, URIGenerated uriGenerated) { + assert c == connection; + assert uriGenerated != null; + if (!identifier.equals(uriGenerated.getIdentifier())) + return; + String sinceTime = ""; + if (lastUriMillis != 0) { + sinceTime = " (" + (System.currentTimeMillis() - lastUriMillis) + "ms since last URI)"; + } + System.out.println("receivedURIGenerated for " + token + at() + sinceTime); + try { + uri = new FreenetURI(uriGenerated.getURI()); + } catch (MalformedURLException e) { + System.err.println("receivedURIGenerated failed with URI: " + uriGenerated.getURI() + + " for " + token + at() + " aborting."); + markDone(); + System.exit(1); + } + synchronized (this) { + this.notifyAll(); + } + lastUriMillis = System.currentTimeMillis(); + } + + private void markDone() { + done = true; + synchronized (this) { + this.notifyAll(); + } + // Signal to the cleanup thread: + synchronized (stillRunning) { + stillRunning.notifyAll(); + } + } + + private void forgetAboutThis() { + assert done; + connection.removeFcpListener(this); + synchronized (stillRunning) { + stillRunning.remove(token); + stillRunning.notifyAll(); + printLeft(); + } + } + + boolean isDone() { + return done; + } + + FreenetURI getURI() { + return uri; + } + }; + + + private static int counter = 1; + + @Override + public void pushLive(plugins.Library.io.serial.Serialiser.PushTask task, + SimpleProgress progress) throws TaskAbortException { + // Slow down the build up of the queue. + try { + int stillRunningSize; + synchronized (stillRunning) { + stillRunningSize = stillRunning.size(); + } + final int uploading = totalBlocksStillUploading + stillRunningSize; + Thread.sleep(1 + 10 * uploading * uploading); + } catch (InterruptedException e1) { + throw new RuntimeException("Unexpected interrupt"); + } + if (connection == null) { + throw new IllegalArgumentException("No connection."); + } + final String identifier = "FcpArchiver" + counter; + final String token = "FcpArchiverPushLive" + counter; + counter++; + final ClientPut putter = new ClientPut("CHK@", identifier); + putter.setClientToken(token); + putter.setEarlyEncode(true); + putter.setPriority(getPriority()); + putter.setVerbosity(Verbosity.ALL); + + // Writing to file. + File file = new File(cacheDir, token); + FileOutputStream fileOut = null; + try { + fileOut = new FileOutputStream(file); + writer.writeObject(task.data, fileOut); + } catch (IOException e) { + throw new TaskAbortException("Cannot write to file " + file, e); + } finally { + try { + fileOut.close(); + } catch (IOException e) { + throw new TaskAbortException("Cannot close file " + file, e); + } + } + + final long dataLength = file.length(); + putter.setDataLength(dataLength); + + FileInputStream in; + try { + in = new FileInputStream(file); + } catch (FileNotFoundException e) { + throw new TaskAbortException("Cannot read from file " + file, e); + } + putter.setPayloadInputStream(in); + + PushAdapter putterListener = new PushAdapter(putter, identifier, token); + connection.addFcpListener(putterListener); + try { + if (progress != null) { + progress.addPartKnown(1, true); + } + connection.sendMessage(putter); + in.close(); + } catch (IOException e) { + throw new TaskAbortException("Cannot send message", e); + } + + // Wait for identifier + synchronized (putterListener) { + while (putterListener.getURI() == null) { + try { + putterListener.wait(); + } catch (InterruptedException e) { + throw new TaskAbortException("Iterrupted wait", e); + } + } + } + + if (progress != null) { + progress.addPartDone(); + } + task.meta = putterListener.getURI(); + + // Moving file. + file.renameTo(new File(cacheDir, putterListener.getURI().toString())); + + startCleanupThread(); + } + + private synchronized void startCleanupThread() { + if (cleanupThread == null) { + cleanupThread = new Thread( + new Runnable() { + public void run () { + boolean moreJobs = false; + do { + if (moreJobs) { + synchronized (stillRunning) { + try { + stillRunning.wait(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + Set copy; + synchronized (stillRunning) { + copy = new HashSet(stillRunning.values()); + } + for (PushAdapter pa : copy) { + if (pa.isDone()) { + pa.forgetAboutThis(); + } + } + } + synchronized (stillRunning) { + moreJobs = !stillRunning.isEmpty(); + } + } while (moreJobs); + removeCleanupThread(); + } + } + ); + cleanupThread.start(); + } + } + + private synchronized void removeCleanupThread() { + cleanupThread = null; + } + + @Override + public void waitForAsyncInserts() throws TaskAbortException { + boolean moreJobs = false; + do { + if (moreJobs) { + synchronized (stillRunning) { + try { + stillRunning.wait(TimeUnit.HOURS.toMillis(1)); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + synchronized (stillRunning) { + moreJobs = !stillRunning.isEmpty(); + } + } while (moreJobs); + } +} diff --git a/uploader/src/freenet/library/uploader/FcpSession.java b/uploader/src/freenet/library/uploader/FcpSession.java new file mode 100644 index 00000000..3d73b623 --- /dev/null +++ b/uploader/src/freenet/library/uploader/FcpSession.java @@ -0,0 +1,85 @@ +package freenet.library.uploader; + +import java.io.IOException; + +import net.pterodactylus.fcp.ClientHello; +import net.pterodactylus.fcp.CloseConnectionDuplicateClientName; +import net.pterodactylus.fcp.FcpAdapter; +import net.pterodactylus.fcp.FcpConnection; +import net.pterodactylus.fcp.FcpMessage; +import net.pterodactylus.fcp.NodeHello; + +public class FcpSession { + + private FcpAdapter closeListener; + private FcpConnection connection; + private int exitStatus; + + public FcpSession() throws IllegalStateException, IOException { + this("SpiderMerger"); + } + + public FcpSession(final String clientName) throws IllegalStateException, IOException { + exitStatus = 0; + + closeListener = new FcpAdapter() { + public void connectionClosed(FcpConnection fcpConnection, Throwable throwable) { + System.out.println("Connection Closed - Aborting."); + System.exit(1); + } + }; + + connection = new FcpConnection("127.0.0.1"); + connection.connect(); + final FcpMessage hello = new ClientHello(clientName); + FcpAdapter helloListener = new FcpAdapter() { + public void receivedNodeHello(FcpConnection c, NodeHello nh) { + synchronized (hello) { + hello.notify(); + } + } + + public void receivedCloseConnectionDuplicateClientName(FcpConnection fcpConnection, CloseConnectionDuplicateClientName closeConnectionDuplicateClientName) { + System.out.println("Another " + clientName + " connected - Aborting."); + System.exit(1); + } + }; + connection.addFcpListener(helloListener); + + connection.addFcpListener(closeListener); + + synchronized (hello) { + try { + connection.sendMessage(hello); + hello.wait(); + } catch (InterruptedException e) { + System.err.println("Waiting for connection interrupted."); + exitStatus = 1; + return; + } finally { + connection.removeFcpListener(helloListener); + } + } + helloListener = null; + System.out.println("Connected"); + } + + public void close() { + if (closeListener != null) { + connection.removeFcpListener(closeListener); + closeListener = null; + } + if (connection != null) { + connection.close(); + connection = null; + } + } + + public FcpConnection getConnection() { + return connection; + } + + public int getStatus() { + return exitStatus; + } +} diff --git a/uploader/src/freenet/library/uploader/FetchAllOnce.java b/uploader/src/freenet/library/uploader/FetchAllOnce.java new file mode 100644 index 00000000..aae8887d --- /dev/null +++ b/uploader/src/freenet/library/uploader/FetchAllOnce.java @@ -0,0 +1,1257 @@ +/* + */ + +/* + * Log levels used: + * None/Warning: Serious events and small problems. + * FINE: Stats for fetches and overview of contents of fetched keys. Minor events. + * FINER: Queue additions, length, ETA, rotations. + * FINEST: Really minor events. + */ + +package freenet.library.uploader; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.Formatter; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.Set; +import java.util.WeakHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import plugins.Library.io.FreenetURI; +import plugins.Library.io.YamlReaderWriter; +import plugins.Library.io.serial.Packer; +import plugins.Library.io.serial.Packer.BinInfo; + +import net.pterodactylus.fcp.AllData; +import net.pterodactylus.fcp.ClientGet; +import net.pterodactylus.fcp.ClientPut; +import net.pterodactylus.fcp.FcpAdapter; +import net.pterodactylus.fcp.FcpConnection; +import net.pterodactylus.fcp.GetFailed; +import net.pterodactylus.fcp.Priority; +import net.pterodactylus.fcp.PutFailed; +import net.pterodactylus.fcp.PutSuccessful; +import net.pterodactylus.fcp.SubscribeUSK; +import net.pterodactylus.fcp.SubscribedUSKUpdate; +import net.pterodactylus.fcp.URIGenerated; +import net.pterodactylus.fcp.Verbosity; + +/** + * Class to download the entire index. + */ +class FetchAllOnce extends AdHocDataReader { + private static final int PARALLEL_JOBS = 10; + private static final int PARALLEL_UPLOADS = 3; + + /** Logger. */ + private static final Logger logger = Logger.getLogger(FetchAllOnce.class.getName()); + + public final Map stillRunning = new HashMap(); + private FreenetURI uri; + private FreenetURI newUri; + private int edition; + private FcpConnection connection; + private static int getterCounter = 0; + private static int uploadCounter = 0; + private LinkedBlockingQueue objectQueue = + new LinkedBlockingQueue(); + private Thread cleanupThread; + private List roots = new ArrayList(); + + private ExecutorService uploadStarter = null; + private Map ongoingUploads = null; + + private int successful = 0; + private int successfulBlocks = 0; + private long successfulBytes = 0; + private int failed = 0; + private long uriUrisSeen = 0; + private long stringUrisSeen = 0; + private int recreated = 0; + private int failedRecreated = 0; + private int avoidFetching = 0; + private int uploadsStarted = 0; + private int avoidRecreate = 0; + private int wrongChkCounterForUpload = 0; + private int maxObjectQueueSize = 0; + + private Random rand = new Random(); + private Date started = new Date(); + + public FetchAllOnce(FreenetURI u) { + uri = u; + } + + public static class WeakHashSet + implements Set { + /** + * We just use the keys and let all values be TOKEN. + */ + private Map map = new WeakHashMap(); + private static Object TOKEN = new Object(); + + @Override + public boolean add(T arg0) { + if (map.containsKey(arg0)) { + return false; + } else { + map.put(arg0, TOKEN); + return true; + } + } + + @Override + public boolean addAll(Collection arg0) { + boolean retval = false; + for (T ele : arg0) { + if (add(ele)) { + retval = true; + } + } + return retval; + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public boolean contains(Object arg0) { + return map.containsKey(arg0); + } + + @Override + public boolean containsAll(Collection arg0) { + for (Object ele : arg0) { + if (!contains(ele)) { + return false; + } + } + return true; + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public Iterator iterator() { + return map.keySet().iterator(); + } + + @Override + public boolean remove(Object arg0) { + return map.remove(arg0) != null; + } + + @Override + public boolean removeAll(Collection arg0) { + boolean retval = true; + for (Object ele : arg0) { + if (!remove(ele)) { + retval = false; + } + } + return retval; + } + + @Override + public boolean retainAll(Collection arg0) { + boolean retval = false; + for (T ele : map.keySet()) { + if (!arg0.contains(ele)) { + if (map.remove(ele) != null) { + retval = true; + } + } + } + return retval; + } + + @Override + public int size() { + return map.size(); + } + + @Override + public Object[] toArray() { + return map.keySet().toArray(); + } + + @Override + public T[] toArray(T[] arg0) { + return map.keySet().toArray(arg0); + } + + } + + /** + * A class to keep track of what pages are fetched and how they are related + * to other fetched pages. The purpose of this is to avoid fetching stuff + * related only to "old" editions. + */ + private static class FetchedPage { + /** + * This is really a Set but there is no WeakSet so we use the keys + * and let all values be TOKEN. + */ + private Set parents = Collections.synchronizedSet(new WeakHashSet()); + private Set children = Collections.synchronizedSet(new HashSet()); + + private FreenetURI uri; + int level; + private boolean succeeded; + private boolean failed; + + FetchedPage(FreenetURI u) { + this(u, 0); + } + + FetchedPage(FreenetURI u, int l) { + uri = u; + level = l; + } + + void addParent(FetchedPage fp) { + parents.add(fp); + } + + void addChild(FetchedPage fp) { + children.add(fp); + } + + FetchedPage newChild(FreenetURI u) { + FetchedPage child = new FetchedPage(u, level + 1); + child.addParent(this); + addChild(child); + return child; + } + + FreenetURI getURI() { + return uri; + } + + boolean hasParent() { + return !parents.isEmpty(); + } + + private FetchedPage[] getParents() { + // Even though parents and children are synchronized we + // encountered some ConcurrentModificationException when + // fetching them through iterators so we avoid that. + return parents.toArray(new FetchedPage[0]); + } + + private FetchedPage[] getChildren() { + return children.toArray(new FetchedPage[0]); + } + + /** + * fetchedPage is an ancestor, any number of levels, to this + * page. + * + * @param fetchedPage the ancestor to search for. + * @return + */ + public boolean hasParent(FetchedPage fetchedPage) { + if (parents.contains(fetchedPage)) { + return true; + } + for (FetchedPage parent : getParents()) { + if (parent != null && parent.hasParent(fetchedPage)) { + return true; + } + } + return false; + } + + int getTreeSize() { + int size = 1; + for (FetchedPage child : getChildren()) { + size += child.getTreeSize(); + } + return size; + } + + void addPerLevel(Map result) { + if (!result.containsKey(level)) { + result.put(level, 0); + } + if (!succeeded && !failed) { + result.put(level, result.get(level) + 1); + } + for (FetchedPage child : children) { + child.addPerLevel(result); + } + } + + int getTreeSizeSucceeded() { + int size = succeeded ? 1 : 0; + for (FetchedPage child : getChildren()) { + size += child.getTreeSizeSucceeded(); + } + return size; + } + + int getTreeSizeFailed() { + int size = failed ? 1 : 0; + for (FetchedPage child : getChildren()) { + size += child.getTreeSizeFailed(); + } + return size; + } + + void didFail() { + failed = true; + } + + void didSucceed() { + failed = false; + succeeded = true; + } + + public FetchedPage findUri(FreenetURI u) { + if (u.equals(uri)) { + return this; + } + for (FetchedPage child : getChildren()) { + FetchedPage found = child.findUri(u); + if (found != null) { + return found; + } + } + return null; + } + } + + private class USKUpdateAdapter extends FcpAdapter { + + private boolean updated = false; + private Object subscriber; + + public USKUpdateAdapter(Object s) { + subscriber = s; + } + + @Override + public void receivedSubscribedUSKUpdate(FcpConnection fcpConnection, SubscribedUSKUpdate subscribedUSKUpdate) { + assert fcpConnection == connection; + if (/*FIXME subscribedUSKUpdate.isNewKnownGood() &&*/ + subscribedUSKUpdate.getEdition() > edition) { + updated = true; + try { + newUri = new FreenetURI(subscribedUSKUpdate.getURI()); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + edition = subscribedUSKUpdate.getEdition(); + synchronized (subscriber) { + subscriber.notify(); + } + } + } + + public void restart() { + if (updated) { + updated = false; + logger.info("Found: " + newUri + " Edition: " + edition); + FetchedPage rootPage = new FetchedPage(newUri); + synchronized (roots) { + roots.add(rootPage); + } + new GetAdapter(rootPage.newChild(newUri)); + } + } + } + + + class StatisticsAccumulator { + private int count = 0; + private int sum = 0; + + void addSample(int found) { + count++; + sum += found; + } + + double getMean() { + return 1.0 * sum / count; + } + + public String toString() { + return "" + getMean() + " (" + count + ")"; + } + } + + private Map statistics = new HashMap(); + private void addFoundChildren(int level, int foundChildren) { + if (!statistics.containsKey(level)) { + statistics.put(level, new StatisticsAccumulator()); + } + statistics.get(level).addSample(foundChildren); + } + + private double getEstimatedPagesLeft(FetchedPage page) { + double estimate = 0.0; + double extra = 0.0; + Map pagesPerLevel = new HashMap(); + page.addPerLevel(pagesPerLevel); + for (int level = 1; pagesPerLevel.containsKey(level); level++) { + if (!statistics.containsKey(level)) { + return Double.POSITIVE_INFINITY; + } + extra += pagesPerLevel.get(level); + estimate += extra; + extra = extra * statistics.get(level).getMean(); + } + return estimate; + } + + + private static class OngoingUpload { + private final Date started = new Date(); + private final FreenetURI freenetURI; + private final Runnable callback; + + public OngoingUpload(FreenetURI fname, Runnable cback) { + freenetURI = fname; + callback = cback; + } + + Date getStarted() { + return started; + } + + FreenetURI getKey() { + return freenetURI; + } + + void complete() { + final long millis = new Date().getTime() - started.getTime(); + final long seconds = millis / 1000; + final long minutes = seconds / 60; + final long hours = minutes / 60; + logger.log(Level.FINE, "Upload completed after {0,number}:{1,number,00}:{2,number,00}.", + new Object[] { + hours, + minutes % 60, + seconds % 60, + }); + callback.run(); + } + } + + /** + * Show the amount of outstanding work. + */ + void printLeft() { + if (logger.isLoggable(Level.FINEST)) { + int total = 0; + int required = 0; + int completed = 0; + synchronized (stillRunning) { + for (GetAdapter value : stillRunning.values()) { + total += value.progressTotal; + required += value.progressRequired; + completed += value.progressCompleted; + } + String ongoingUploadsMessage = ""; + if (logger.isLoggable(Level.FINEST) && ongoingUploadsSize() > 0) { + Date oldest = null; + synchronized (ongoingUploads) { + for (Map.Entry entry : ongoingUploads.entrySet()) { + if (oldest == null || oldest.compareTo(entry.getValue().getStarted()) > 0) { + oldest = entry.getValue().getStarted(); + } + } + } + ongoingUploadsMessage = " and " + ongoingUploadsSize() + " uploads"; + if (oldest != null && new Date().getTime() - oldest.getTime() > TimeUnit.HOURS.toMillis(5)) { + ongoingUploadsMessage += new MessageFormat(", oldest from {0,date,long}").format(new Object[] { oldest }); + } + ongoingUploadsMessage += "."; + } + logger.finest("Outstanding " + stillRunning.size() + " ClientGet jobs " + + "(" + completed + "/" + required + "/" + total + ")" + + ongoingUploadsMessage); + } + } + } + + private class GetAdapter extends FcpAdapter { + private ClientGet getter; + private String token; + private FetchedPage page; + private int progressTotal; + private int progressRequired; + private int progressCompleted; + private boolean done; + int waitingLaps; + public static final int WAITING_FACTOR = 50; + + public GetAdapter(FetchedPage u) { + page = u; + getterCounter ++; + token = "Getter" + getterCounter; + waitingLaps = 0; + getter = new ClientGet(page.getURI().toString(), token); + getter.setPriority(Priority.prefetch); + getter.setVerbosity(Verbosity.ALL); + + waitForSlot(); + connection.addFcpListener(this); + try { + connection.sendMessage(getter); + } catch (IOException e) { + e.printStackTrace(); + System.exit(1); + } + synchronized (stillRunning) { + stillRunning.put(page, this); + stillRunning.notifyAll(); + } + } + + /** + * Called when nothing has happened for a while with this request. + * @param key The page. + */ + public void hasBeenWaiting(FetchedPage key) { + waitingLaps++; + if (waitingLaps > WAITING_FACTOR * PARALLEL_JOBS) { + connection.removeFcpListener(this); + getter = null; + synchronized (stillRunning) { + stillRunning.remove(key); + } + if (key.hasParent()) { + logger.warning("Restarting fetch for " + key.getURI()); + new GetAdapter(key); + } else { + logger.finer("Avoid refetching " + key.getURI()); + } + } + } + + + private boolean processAnUri(FreenetURI uri) { + synchronized (roots) { + for (FetchedPage root : roots) { + FetchedPage foundChild = root.findUri(uri); + if (foundChild != null) { + page.addChild(foundChild); + foundChild.addParent(page); + return false; + } + } + } + objectQueue.offer(page.newChild(uri)); + return true; + } + + @Override + public void receivedAllData(FcpConnection c, AllData ad) { + assert c == connection; + assert ad != null; + if (!token.equals(ad.getIdentifier())) { + return; + } + final int objectQueueSize = objectQueue.size(); + if (objectQueueSize > maxObjectQueueSize) { + maxObjectQueueSize = objectQueueSize; + } + logger.entering(GetAdapter.class.toString(), + "receivedAllData", + "receivedAllData for " + token + + " adding to the " + objectQueueSize + " elements in the queue " + + "(max " + maxObjectQueueSize + ")."); + page.didSucceed(); + UriProcessor uriProcessor = new UriProcessor() { + @Override + public FreenetURI getURI() { + return page.getURI(); + } + + @Override + public int getLevel() { + return page.level; + } + + @Override + public boolean processUri(FreenetURI uri) { + return processAnUri(uri); + } + + @Override + public void uriSeen() { + uriUrisSeen++; + } + + @Override + public void stringSeen() { + stringUrisSeen++; + } + + @Override + public void childrenSeen(int level, int foundChildren) { + addFoundChildren(level, foundChildren); + } + + }; + final InputStream inputStream = ad.getPayloadInputStream(); + try { + readAndProcessYamlData(inputStream, uriProcessor, page.level); + } catch (IOException e) { + logger.log(Level.SEVERE, "Cannot unpack.", e); + e.printStackTrace(); + System.exit(1); + } catch (ClassCastException cce) { + logger.log(Level.SEVERE, "Cannot unpack.", cce); + cce.printStackTrace(); + System.exit(1); + } finally { + markDone(); + successful ++; + successfulBlocks += progressCompleted; + successfulBytes += ad.getDataLength(); + showProgress(); + } + } + + @Override + public void receivedGetFailed(FcpConnection c, GetFailed gf) { + assert c == connection; + assert gf != null; + if (!token.equals(gf.getIdentifier())) { + return; + } + synchronized (getter) { + getter.notify(); + } + logger.warning("receivedGetFailed for " + token + " (" + page.getURI() + ")."); + page.didFail(); + markDone(); + failed ++; + showProgress(); + upload(page, new Runnable() { + public void run() { + objectQueue.offer(page); + recreated ++; + } + }); + } + + /** + * We have detected that we cannot download a certain CHK. + * + * If this CHK is actually cached, lets upload it from + * the cache in an attempt to repair the index. + * + * @param page the URI to upload. + * @param callback when the file is successfully uploaded. + */ + public boolean upload(final FetchedPage page, final Runnable callback) { + final File dir = new File(".", UploaderPaths.LIBRARY_CACHE); + if (!dir.canRead()) { + return false; + } + final File file = new File(dir, page.getURI().toString()); + if (!file.canRead()) { + logger.warning("Cannot find " + file + " in the cache."); + return false; + } + if (uploadStarter == null) { + uploadStarter = Executors.newSingleThreadExecutor(); + uploadStarter.execute(new Runnable() { + public void run() { + connection.addFcpListener(new FcpAdapter() { + @Override + public void receivedURIGenerated(FcpConnection c, URIGenerated uriGenerated) { + assert c == connection; + assert uriGenerated != null; + String identifier = uriGenerated.getIdentifier(); + FreenetURI chk; + synchronized (ongoingUploads) { + chk = ongoingUploads.get(identifier).getKey(); + } + FreenetURI generatedURI; + try { + generatedURI = new FreenetURI(uriGenerated.getURI()); + } catch (MalformedURLException e) { + logger.severe("Were supposed to resurrect " + chk + + " but the URI calculated to " + uriGenerated.getURI() + + " that is not possible to convert to an URI. Will upload anyway."); + wrongChkCounterForUpload++; + return; + } + if (!generatedURI.equals(chk)) { + logger.severe("Were supposed to resurrect " + chk + + " but the URI calculated to " + uriGenerated.getURI() + ". " + + "Will upload anyway."); + wrongChkCounterForUpload++; + } else { + logger.finest("Resurrecting " + chk); + } + } + + @Override + public void receivedPutSuccessful(FcpConnection c, PutSuccessful putSuccessful) { + assert c == connection; + assert putSuccessful != null; + String identifier = putSuccessful.getIdentifier(); + OngoingUpload ongoingUpload; + synchronized (ongoingUploads) { + ongoingUpload = ongoingUploads.get(identifier); + } + final OngoingUpload foundUpload = ongoingUpload; + FreenetURI chk = foundUpload.getKey(); + FreenetURI generatedURI = null; + try { + generatedURI = new FreenetURI(putSuccessful.getURI()); + } catch (MalformedURLException e) { + logger.severe("Uploaded " + putSuccessful.getURI() + + " that is not possible to convert to an URI."); + } + if (generatedURI != null) { + if (!generatedURI.equals(chk)) { + logger.severe("Uploaded " + putSuccessful.getURI() + + " while supposed to upload " + chk + + ". "); + } else { + foundUpload.complete(); + } + } + synchronized (ongoingUploads) { + ongoingUploads.remove(identifier); + ongoingUploads.notifyAll(); + } + synchronized (stillRunning) { + stillRunning.notifyAll(); + } + }; + + @Override + public void receivedPutFailed(FcpConnection c, PutFailed putFailed) { + assert c == connection; + assert putFailed != null; + String identifier = putFailed.getIdentifier(); + OngoingUpload ongoingUpload; + synchronized (ongoingUploads) { + ongoingUpload = ongoingUploads.get(identifier); + } + final OngoingUpload foundUpload = ongoingUpload; + FreenetURI chk = foundUpload.getKey(); + logger.severe("Uploaded " + chk + " failed."); + failedRecreated++; + synchronized (ongoingUploads) { + ongoingUploads.remove(identifier); + ongoingUploads.notifyAll(); + } + synchronized (stillRunning) { + stillRunning.notifyAll(); + } + } + }); + } + }); + ongoingUploads = new HashMap(); + } + uploadsStarted++; + uploadStarter.execute(new Runnable() { + public void run() { + if (!page.hasParent()) { + avoidRecreate++; + return; + } + uploadCounter++; + final String identifier = "Upload" + uploadCounter; + synchronized (ongoingUploads) { + ongoingUploads.put(identifier, new OngoingUpload(page.getURI(), callback)); + ongoingUploads.notifyAll(); + } + final ClientPut putter = new ClientPut("CHK@", identifier); + putter.setEarlyEncode(true); + putter.setPriority(net.pterodactylus.fcp.Priority.bulkSplitfile); + putter.setVerbosity(Verbosity.NONE); + final long dataLength = file.length(); + putter.setDataLength(dataLength); + FileInputStream in; + try { + in = new FileInputStream(file); + putter.setPayloadInputStream(in); + connection.sendMessage(putter); + in.close(); + in = null; + } catch (IOException | NullPointerException e) { + e.printStackTrace(); + logger.warning("Upload failed for " + file); + } + while (true) { + synchronized (ongoingUploads) { + if (ongoingUploads.size() < PARALLEL_UPLOADS) { + break; + } + try { + ongoingUploads.wait(TimeUnit.SECONDS.toMillis(3)); + } catch (InterruptedException e) { + throw new RuntimeException("Waiting for upload slot terminated."); + } + } + } + } + }); + return true; + } + + @Override + public void receivedSimpleProgress(FcpConnection c, + net.pterodactylus.fcp.SimpleProgress sp) { + assert c == connection; + assert sp != null; + if (!token.equals(sp.getIdentifier())) { + return; + } + progressTotal = sp.getTotal(); + progressRequired = sp.getRequired(); + progressCompleted = sp.getSucceeded(); + printLeft(); + } + + + private void markDone() { + done = true; + synchronized (this) { + this.notifyAll(); + } + // Signal to the cleanup thread: + synchronized (stillRunning) { + stillRunning.notifyAll(); + } + } + + private void forgetAboutThis() { + assert done; + connection.removeFcpListener(this); + synchronized (stillRunning) { + stillRunning.remove(page); + // Signal to the + stillRunning.notifyAll(); + printLeft(); + } + } + + boolean isDone() { + return done; + } + }; + + private int uploadsWaiting() { + return uploadsStarted - uploadCounter - avoidRecreate; + } + + private void ageRunning() { + final HashSet> stillRunningCopy; + synchronized (stillRunning) { + stillRunningCopy = new HashSet>(stillRunning.entrySet()); + } + for (Entry entry : stillRunningCopy) { + entry.getValue().hasBeenWaiting(entry.getKey()); + } + } + + public void doDownload() { + FcpSession session; + try { + session = new FcpSession("DownloaderFor" + uri); + } catch (IllegalStateException | IOException e1) { + e1.printStackTrace(); + return; + } + try { + connection = session.getConnection(); + if (connection == null) { + throw new IllegalArgumentException("No connection."); + } + final SubscribeUSK subscriber = new SubscribeUSK(uri + "-1", "USK"); + subscriber.setActive(true); + + final USKUpdateAdapter subscriberListener = new USKUpdateAdapter(subscriber); + connection.addFcpListener(subscriberListener); + + synchronized (subscriber) { + try { + connection.sendMessage(subscriber); + subscriber.wait(); + } catch (InterruptedException e) { + throw new RuntimeException("Waiting for connection interrupted."); + } catch (IOException e) { + throw new RuntimeException("Hello cannot write."); + } + } + subscriberListener.restart(); + + boolean moreJobs = false; + do { + if (moreJobs) { + synchronized (stillRunning) { + try { + logger.fine("Queue empty. " + + "Still running " + + stillRunning.size() + "."); + stillRunning.wait(20000); + } catch (InterruptedException e) { + e.printStackTrace(); + System.exit(1); + } + } + } + + boolean empty = true; + do { + ageRunning(); + synchronized (roots) { + final int roots_size = roots.size(); + if (roots_size > 1) { + int roots_distance = roots_size - 1; + if (roots.get(1).getTreeSizeSucceeded() >= roots.get(0).getTreeSizeSucceeded() - roots_distance * roots_distance * roots_distance) { + roots.remove(0); + } + } + } + + FetchedPage lastRoot; + synchronized (roots) { + lastRoot = roots.get(roots.size() - 1); + } + + // Randomize the order by rotating the queue + int maxLaps = objectQueue.size(); + if (maxLaps == 0) { + maxLaps = 1; + } + int toRotate = rand.nextInt(maxLaps); + int rotated = 0; + int counted = 0; + + while (!objectQueue.isEmpty()) { + FetchedPage taken; + try { + taken = objectQueue.take(); + } catch (InterruptedException e) { + e.printStackTrace(); + System.exit(1); + continue; + } + if (!taken.hasParent()) { + logger.finer("Avoid fetching " + taken.getURI()); + taken = null; + avoidFetching++; + continue; + } + + counted += taken.level * taken.level * taken.level; + if (counted < toRotate) { + rotated++; + objectQueue.offer(taken); + continue; + } + + if (!taken.hasParent(lastRoot) && rand.nextInt(100) > 0) { + logger.finer("Defer fetching non-last " + taken.getURI()); + objectQueue.offer(taken); + continue; + } + + logger.finest("Rotated " + rotated + " (count to " + toRotate + ")."); + new GetAdapter(taken); + break; + } + subscriberListener.restart(); + empty = objectQueue.isEmpty(); + } while (!empty); + synchronized (stillRunning) { + moreJobs = !stillRunning.isEmpty(); + } + } while (moreJobs); + if (uploadStarter != null) { + uploadStarter.shutdown(); + try { + uploadStarter.awaitTermination(1, TimeUnit.HOURS); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + connection.removeFcpListener(subscriberListener); + } finally { + removeCleanupThread(); + session.close(); + connection = null; + } + showProgress(); + } + + + private void showProgress() { + String recreatedMessage = ""; + if (recreated > 0) { + recreatedMessage = " Recreated: " + recreated; + } + if (failedRecreated > 0) { + recreatedMessage += " Recreation failed: " + failedRecreated; + } + if (avoidRecreate > 0) { + recreatedMessage += " Recreation avoided: " + avoidRecreate; + } + String urisSeenMessage = ""; + if (uriUrisSeen > 0 || stringUrisSeen > 0) { + urisSeenMessage = " StringUrisSeen: " + stringUrisSeen + "/" + (uriUrisSeen + stringUrisSeen); + urisSeenMessage += new Formatter().format(" (%.1f%%)", 100.0 * stringUrisSeen / (uriUrisSeen + stringUrisSeen)); + } + String wrongChkCounterForUploadMessage = ""; + if (wrongChkCounterForUpload > 0) { + wrongChkCounterForUploadMessage = " WrongChkUploaded: " + wrongChkCounterForUpload; + } + logger.fine("Fetches: Successful: " + successful + + " blocks: " + successfulBlocks + + " bytes: " + successfulBytes + + " Failed: " + failed + + urisSeenMessage + + recreatedMessage + + wrongChkCounterForUploadMessage + + " Avoided: " + avoidFetching + "."); + + StringBuilder sb = new StringBuilder(); + List copiedRoots; + synchronized (roots) { + copiedRoots = new ArrayList(roots); + } + Collections.reverse(copiedRoots); + boolean first = true; + for (FetchedPage root : copiedRoots) { + if (sb.length() > 0) { + sb.append(", "); + } + long edition = root.getURI().getEdition(); + sb.append(edition); + int succeeded = root.getTreeSizeSucceeded(); + int failed = root.getTreeSizeFailed(); + if (failed > 0) { + sb.append(new Formatter().format(" FAILED: %.1f%%.", 100.0 * failed / (failed + succeeded))); + } + double estimate = getEstimatedPagesLeft(root); + if (estimate < Double.POSITIVE_INFINITY) { + final double fractionDone = 1.0 * succeeded / (estimate + succeeded); + sb.append(new Formatter().format(" Fetched: %.1f%%.", + 100.0 * fractionDone)); + if (first) { + logger.log(Level.FINER, "ETA: {0,date}, Started: {1,date}. Done {2,number,percent}.", + new Object[] { + new Date(new Double(1.0 / fractionDone * (new Date().getTime() - started.getTime())).longValue() + + started.getTime()), + started, + fractionDone, + }); + first = false; + } + } + sb.append(" ("); + sb.append(succeeded); + + if (failed > 0) { + sb.append(" and "); + sb.append(failed); + sb.append(" failed"); + } + + sb.append(")"); + } + + System.out.println("Editions: " + sb.toString()); + } + + /** + * 1. chdir to the directory with all the files. + * 2. Give parameters --move CHK/filename + * The CHK/filename is of the top file (in library.index.lastpushed.chk). + */ + public void doMove() { + int count = 0; + File toDirectory = new File("../" + UploaderPaths.LIBRARY_CACHE + ".new2"); + if (!toDirectory.mkdir()) { + System.err.println("Could not create the directory " + toDirectory); + System.exit(1); + } + final FetchedPage fetchedPage = new FetchedPage(uri); + roots.add(fetchedPage); + objectQueue.add(fetchedPage); + while (objectQueue.size() > 0) { + FetchedPage page; + try { + page = objectQueue.take(); + } catch (InterruptedException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + System.exit(1); + return; + } + final FetchedPage finalPage = page; + FileInputStream inputStream; + try { + Files.createLink(Paths.get(toDirectory.getPath(), page.uri.toString()), Paths.get(page.uri.toString())); + inputStream = new FileInputStream(page.uri.toString()); + count++; + System.out.println("Read file " + count + " in " + page.uri + " level " + page.level + " left: " + objectQueue.size()); + } catch (IOException e) { + System.out.println("Cannot find file " + page.uri); + e.printStackTrace(); + System.exit(1); + return; + } + try { + readAndProcessYamlData(inputStream, + new UriProcessor() { + @Override + public FreenetURI getURI() { + return finalPage.getURI(); + } + + @Override + public int getLevel() { + return 1; + } + + Set seen = new HashSet(); + @Override + public boolean processUri(FreenetURI uri) { + if (seen.contains(uri)) { + return false; + } + seen.add(uri); + objectQueue.offer(finalPage.newChild(uri)); + return true; + } + + @Override + public void uriSeen() {} + + @Override + public void stringSeen() {} + + @Override + public void childrenSeen(int level, int foundChildren) {} + + }, page.level); + } catch (IOException e) { + System.out.println("Cannot read file " + page.uri); + e.printStackTrace(); + System.exit(1); + return; + } + + } + } + + private int ongoingUploadsSize() { + if (ongoingUploads == null) { + return 0; + } + + synchronized (ongoingUploads) { + return ongoingUploads.size(); + } + } + + public void waitForSlot() { + startCleanupThread(); + synchronized (stillRunning) { + try { + for (int i = 0; i < uploadsWaiting() + ongoingUploadsSize() + stillRunning.size(); i++) { + stillRunning.wait(TimeUnit.SECONDS.toMillis(1 + uploadsWaiting() + uploadsWaiting())); + } + while (stillRunning.size() >= PARALLEL_JOBS) { + stillRunning.wait(1 + TimeUnit.MINUTES.toMillis(2)); + } + } catch (InterruptedException e) { + e.printStackTrace(); + System.exit(1); + } + } + } + + private synchronized void startCleanupThread() { + if (cleanupThread == null) { + cleanupThread = new Thread( + new Runnable() { + public void run () { + boolean moreJobs = false; + do { + if (moreJobs) { + synchronized (stillRunning) { + try { + stillRunning.wait(1234567); + } catch (InterruptedException e) { + e.printStackTrace(); + System.exit(1); + } + } + Set copy; + synchronized (stillRunning) { + copy = new HashSet(stillRunning.values()); + } + for (GetAdapter ga : copy) { + if (ga.isDone()) { + ga.forgetAboutThis(); + } + } + } + synchronized (stillRunning) { + moreJobs = !stillRunning.isEmpty(); + } + } while (moreJobs); + removeCleanupThread(); + } + } + ); + cleanupThread.start(); + } + } + + private synchronized void removeCleanupThread() { + cleanupThread = null; + + Set copy; + synchronized (stillRunning) { + copy = new HashSet(stillRunning.values()); + } + for (GetAdapter ga : copy) { + ga.markDone(); + ga.forgetAboutThis(); + } + } +} diff --git a/uploader/src/freenet/library/uploader/IndexPeeker.java b/uploader/src/freenet/library/uploader/IndexPeeker.java new file mode 100644 index 00000000..a86865c6 --- /dev/null +++ b/uploader/src/freenet/library/uploader/IndexPeeker.java @@ -0,0 +1,124 @@ +package freenet.library.uploader; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import plugins.Library.index.TermEntry; +import plugins.Library.io.YamlReaderWriter; +import plugins.Library.util.SkeletonBTreeMap; +import plugins.Library.util.SkeletonBTreeSet; + +class IndexPeeker { + private File directory; + private LinkedHashMap topTtab; + private Set topElements; + private List activeSections = null; + + private static final SkeletonBTreeMap> newtrees = + new SkeletonBTreeMap>(12); + + IndexPeeker(File dir) { + directory = dir; + String lastCHK = DirectoryUploader.readStringFrom(new File(directory, UploaderPaths.LAST_URL_FILENAME)); + + if (lastCHK == null) { // case when the index is created for the first time + topTtab = new LinkedHashMap<>(); + topElements = new HashSet<>(); + activeSections = new LinkedList<>(); + return; + } + + String rootFilename = directory + "/" + UploaderPaths.LIBRARY_CACHE + "/" + lastCHK; + try { + LinkedHashMap top = (LinkedHashMap) new YamlReaderWriter().readObject(new FileInputStream(new File(rootFilename))); + LinkedHashMap ttab = (LinkedHashMap) top.get("ttab"); + topTtab = (LinkedHashMap) ttab.get("entries"); + } catch (IOException e) { + e.printStackTrace(); + System.exit(1); + } + + System.out.println("topTtab.size: " + topTtab.size()); +// if (topTtab.size() < 1000) { +// // So far the growth of the index and the growth of the elements +// // in the top node has gone hand in hand keeping the amount of +// // pages to update for each merger low. When the amount of terms +// // will exceed 1500 x 2048 the B-tree index will suddenly be +// // rebuilt with just two entries on top that will share all the +// // terms between them. This means that this logic of splitting +// // on the top level only will split into two piles instead of +// // over a thousand and there is a risk that way too much will be +// // included in each update. This code needs to be improved to +// // handle this. FIXME +// throw new IllegalArgumentException("This version of the script does not handle multi-level tree."); +// } + + topElements = new HashSet(topTtab.keySet()); + activeSections = new LinkedList(); + } + + private static int compare(String a, String b) { + return SkeletonBTreeMap.compare(a, b, newtrees.comparator()); + } + + class ChoosenSection { + String before; + String after; + + ChoosenSection(String subj) { + System.out.println("Grouping around " + subj); + String previous = null; + String next = null; + for (String iter : topTtab.keySet()) { + next = iter; + if (compare(subj, next) < 0) { + break; + } + previous = iter; + next = null; + } + before = previous; + after = next; + } + + boolean include(String subj) { + if ((before == null || compare(before, subj) < 0) && + (after == null || compare(subj, after) < 0)) { + return true; + } + return false; + } + } + + /** + * If the subj is to be included. + * + * If subj is on top, include it. + * Let the first subj decide what part of the tree we match. + * Include subsequent terms if they are in the same part of the tree. + * + * @param subj The term to include. + * @return true if the term is included. + */ + boolean include(String subj) { + if (topElements.contains(subj)) { + return true; + } + for (ChoosenSection section : activeSections) { + if (section.include(subj)) { + return true; + } + } + if (activeSections.size() < 1) { + activeSections.add(new ChoosenSection(subj)); + return true; + } + return false; + } +} diff --git a/uploader/src/freenet/library/uploader/Merger.java b/uploader/src/freenet/library/uploader/Merger.java new file mode 100644 index 00000000..957dd79b --- /dev/null +++ b/uploader/src/freenet/library/uploader/Merger.java @@ -0,0 +1,428 @@ +/* This code is part of Freenet. It is distributed under the GNU General + * Public License, version 2 (or at your option any later version). See + * http://www.gnu.org/ for further details of the GPL. */ + +package freenet.library.uploader; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FilenameFilter; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +import net.pterodactylus.fcp.FcpConnection; // https://github.com/Bombe/jFCPlib + +import plugins.Library.FactoryRegister; +import plugins.Library.index.TermEntry; +import plugins.Library.util.exec.TaskAbortException; + +/** + * Standalone program to do the merging. + * + * The ambition is to avoid having the merger running in the freenet process, + * instead run it as a separate java program. + * + * It reads and removes the files created by the plugin (by Spider) and + * delivers data using FCP. + * + * Initially it is the same jar used as plugin and for the separate process. + * + * + * Initial logic: + * Run once to merge once. If more than one of these merge jobs are + * started at the time, the FcpConnection will fail to open since + * they use the same name. + *

    Check if there are directories to merge. If so, merge the first of them (order is not important). Done! + *
      If there are no files to merge. Done! + *
        Fetch the index top to get the top fan-out. + *
          Get the first term in the first and create an index with all + * the contents from all the files with all terms from the same index. + *
            While processing, group into some selected files. The rest goes + * into the filtered files. + *
              Merge that index. + *
                If there are selected files, get the first term from the first one of them instead. + *
                  If there is room for more in the same go, get the next selected file. + *
                    Rewrite all not filtered files getting the matched terms. + *
                      Done. + */ +final public class Merger { + + private static FcpSession session; + + private static final String SELECTED = UploaderPaths.BASE_FILENAME_DATA + "selected."; + private static final String FILTERED = UploaderPaths.BASE_FILENAME_DATA + "filtered."; + private static final String PROCESSED = UploaderPaths.BASE_FILENAME_DATA + "processed."; + + static final Comparator comparator = new StringNumberComparator(); + + static class StringNumberComparator implements Comparator { + @Override + public int compare(String a, String b) { + int ai; + int bi; + for (ai = 0, bi = 0; ai < a.length() && bi < b.length(); ai++, bi++) { + if (a.substring(ai, ai + 1).matches("[0-9]") + && a.substring(bi, bi + 1).matches("[0-9]")) { + int aii; + for (aii = ai + 1; aii < a.length(); aii++) { + if (!a.substring(aii, aii + 1).matches("[0-9]")) { + break; + } + } + int bii; + for (bii = bi + 1; bii < b.length(); bii++) { + if (!b.substring(bii, bii + 1).matches("[0-9]")) { + break; + } + } + try { + int ret = Integer.valueOf(a.substring(ai, aii)).compareTo( + Integer.valueOf(b.substring(bi, bii))); + if (ret != 0) { + return ret; + } + + ai = aii - 1; + bi = bii - 1; + continue; + } catch (NumberFormatException e) { + continue; + } + } + int ret = a.charAt(ai) - b.charAt(bi); + if (ret != 0) { + return ret; + } + } + if (ai < a.length()) { + return 1; + } + if (bi < b.length()) { + return -1; + } + return 0; + } + } + + /** + * Return an array with the filenames in order. + */ + static String[] getMatchingFiles(File directory, + final String baseFilename) { + String[] array = directory.list(new FilenameFilter() { + + public boolean accept(File arg0, String arg1) { + if (!(arg1.toLowerCase().startsWith(baseFilename))) { + return false; + } + File f = new File(arg0, arg1); + if (!f.isFile()) { + return false; + } + if (f.length() == 0) { + f.delete(); + return false; + } + return true; + } + }); + Arrays.sort(array, comparator); + return array; + } + + + public static void main(String[] argv) { + int exitStatus = 0; + + //if (!cwd.matches(".*/plugins")) { + // System.err.println("Should be started in the freenet directory."); + // System.exit(1); + //} + + // Now we are in the Freenet directory. + // The rest of the work is done here. + FcpConnection connection = null; + + try { + String[] dirsToMerge = null; + File directory = new File("."); + for (String arg : argv) { + if (new File(directory, arg).isDirectory()) { + dirsToMerge = new String[1]; + dirsToMerge[0] = arg; + } else { + System.out.println("No such directory " + arg); + } + break; + } + if (dirsToMerge == null) { + dirsToMerge = directory.list(new FilenameFilter() { + + public boolean accept(File arg0, String arg1) { + if(!(arg1.toLowerCase().startsWith(UploaderPaths.DISK_DIR_PREFIX))) return false; + return true; + } + + }); + } + + if (dirsToMerge.length > 0) { + System.out.println("Merging directory " + dirsToMerge[0]); + session = new FcpSession(); + connection = session.getConnection(); + UploaderLibrary.init(connection); + FactoryRegister.register(UploaderLibrary.getInstance()); + + File directoryToMerge = new File(directory, dirsToMerge[0]); + new DirectoryUploader(connection, directoryToMerge, false).run(); + System.out.println("Upload completed."); + return; + } + + createMergeDirectory(directory); + } catch (TaskAbortException | IllegalStateException | IOException e) { + e.printStackTrace(); + exitStatus = 1; + } finally { + if (session != null) { + session.close(); + if (exitStatus == 0) { + exitStatus = session.getStatus(); + } + } + } + System.exit(exitStatus); + } + + + private static void createMergeDirectory(File directory) throws TaskAbortException { + final String[] selectedFilesToMerge = getMatchingFiles(directory, SELECTED); + System.out.println("There is " + selectedFilesToMerge.length + " selected files."); + + final String [] filteredFilesToMerge = getMatchingFiles(directory, FILTERED); + System.out.println("There is " + filteredFilesToMerge.length + " filtered files."); + + final String [] processedFilesToMerge = getMatchingFiles(directory, PROCESSED); + System.out.println("There is " + processedFilesToMerge.length + " processed files."); + + final String[] newFilesToMerge = getMatchingFiles(directory, UploaderPaths.BASE_FILENAME_PUSH_DATA); + System.out.println("There is " + newFilesToMerge.length + " new files."); + + // Calculate the last number of filtered and processed files. + int lastFoundNumber = 0; + for (String filename : filteredFilesToMerge) { + int numberFound = Integer.parseInt(filename.substring(FILTERED.length())); + if (numberFound > lastFoundNumber) { + lastFoundNumber = numberFound; + } + } + for (String filename : processedFilesToMerge) { + int numberFound = Integer.parseInt(filename.substring(PROCESSED.length())); + if (numberFound > lastFoundNumber) { + lastFoundNumber = numberFound; + } + } + System.out.println("Last found: " + lastFoundNumber); + + int lastSelected = 0; + for (String filename : selectedFilesToMerge) { + int numberFound = Integer.parseInt(filename.substring(SELECTED.length())); + if (numberFound > lastSelected) { + lastSelected = numberFound; + } + } + + final DirectoryCreator creator = new DirectoryCreator(directory); + + Map writers = + new HashMap(); + IndexPeeker creatorPeeker = new IndexPeeker(directory); + + Set toBeRemoved = new HashSet(); + + /** + * All files to read through in the correct order. + */ + class ProcessedFilenames implements Iterator { + String restBase; + boolean createSelectedFiles = false; + boolean processingSelectedFile = false; + int movedTerms = 0; + private boolean doSelected = false; + private boolean doAllSelected = false; + private boolean doFiltered = false; + private boolean doProcessed = false; + private boolean doNew = true; + private int nextSelected = 0; + private int nextFiltered = 0; + private int nextProcessed = 0; + private int nextNew = 0; + + ProcessedFilenames() { + if (selectedFilesToMerge.length > 0) { + doSelected = true; + if (processedFilesToMerge.length > filteredFilesToMerge.length) { + createSelectedFiles = true; + doAllSelected = true; + doFiltered = true; + restBase = FILTERED; + } else { + restBase = PROCESSED; + } + } else { + createSelectedFiles = true; + doFiltered = true; + restBase = FILTERED; + } + doProcessed = true; + doNew = true; + } + + @Override + public boolean hasNext() { + if (doSelected && + nextSelected < selectedFilesToMerge.length) { + return true; + } + if (doAllSelected && nextSelected < selectedFilesToMerge.length) { + return true; + } + if (doFiltered && nextFiltered < filteredFilesToMerge.length) { + return true; + } + if (doProcessed && nextProcessed < processedFilesToMerge.length) { + return true; + } + if (doNew && nextNew < newFilesToMerge.length) { + return true; + } + return false; + } + + @Override + public String next() { + processingSelectedFile = false; + if (doSelected && + nextSelected < selectedFilesToMerge.length) { + processingSelectedFile = true; + doSelected = false; + return selectedFilesToMerge[nextSelected++]; + } else if (doAllSelected && nextSelected < selectedFilesToMerge.length) { + return selectedFilesToMerge[nextSelected++]; + } else if (doFiltered && nextFiltered < filteredFilesToMerge.length) { + return filteredFilesToMerge[nextFiltered++]; + } else if (doProcessed && nextProcessed < processedFilesToMerge.length) { + return processedFilesToMerge[nextProcessed++]; + } else if (doNew && nextNew < newFilesToMerge.length) { + return newFilesToMerge[nextNew++]; + } else { + throw new IllegalArgumentException("next() called after hasNext() returned false."); + } + } + + @Override + public void remove() { + throw new IllegalArgumentException("Not implemented"); + } + }; + final ProcessedFilenames processedFilenames = new ProcessedFilenames(); + TermEntryFileWriter notMerged = null; + + int totalTerms = 0; + + for (String s : new Iterable() { + @Override + public Iterator iterator() { + return processedFilenames; + } + }) { + System.out.println("File: " + s); + File file = new File(s); + FileInputStream fileInputStream; + try { + fileInputStream = new FileInputStream(file); + } catch (FileNotFoundException e) { + e.printStackTrace(); + return; + } + TermEntryReaderIterator teri = new TermEntryReaderIterator(new DataInputStream(fileInputStream)); + Iterator iterator = teri.iterator(); + while (iterator.hasNext()) { + TermEntry tt = iterator.next(); + if (tt.toBeDropped()) { + System.out.println("Ignoring term " + tt); + continue; + } + totalTerms ++; + if (creatorPeeker.include(tt.subj)) { + creator.putEntry(tt); + processedFilenames.movedTerms ++; + continue; + } + + if (processedFilenames.createSelectedFiles) { + // They are all to be sorted. + boolean found = false; + for (Map.Entry entry : writers.entrySet()) { + if (entry.getKey().include(tt.subj)) { + entry.getValue().write(tt); + found = true; + break; + } + } + if (found) { + continue; + } else if (writers.size() < 3 || writers.size() < 10 * (filteredFilesToMerge.length - 1)) { + lastSelected ++; + String selectedFilename = SELECTED + lastSelected; + IndexPeeker p = new IndexPeeker(directory); + TermEntryFileWriter t = new TermEntryFileWriter(teri.getHeader(), + new File(directory, selectedFilename)); + if (p.include(tt.subj)) { + writers.put(p, t); + t.write(tt); + } + continue; + } + } + if (notMerged == null) { + lastFoundNumber ++; + String restFilename = processedFilenames.restBase + lastFoundNumber; + notMerged = new TermEntryFileWriter(teri.getHeader(), new File(directory, restFilename)); + } + notMerged.write(tt); + if (notMerged.isFull()) { + notMerged.close(); + notMerged = null; + } + } + if (processedFilenames.processingSelectedFile) { + System.out.println("Items: " + processedFilenames.movedTerms + + " Entries: " + creator.size()); + } + toBeRemoved.add(file); + } + if (notMerged != null) { + notMerged.close(); + notMerged = null; + } + creator.done(); + for (File file : toBeRemoved) { + System.out.println("Removing file " + file); + file.delete(); + } + double percentage = new Double(processedFilenames.movedTerms).doubleValue() / new Double(totalTerms).doubleValue() * 100.0; + System.out.format("Processed %d/%d terms (%.2f%%).%n", + processedFilenames.movedTerms, + totalTerms, + percentage); + } +} diff --git a/uploader/src/freenet/library/uploader/TermEntryFileWriter.java b/uploader/src/freenet/library/uploader/TermEntryFileWriter.java new file mode 100644 index 00000000..01d6fbc9 --- /dev/null +++ b/uploader/src/freenet/library/uploader/TermEntryFileWriter.java @@ -0,0 +1,62 @@ +package freenet.library.uploader; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Map; +import java.util.Map.Entry; + +import plugins.Library.index.TermEntry; +import plugins.Library.index.TermEntryReaderWriter; + +class TermEntryFileWriter { + private DataOutputStream os; + int counter; + + public TermEntryFileWriter(Map params, + File file) { + counter = 0; + try { + os = new DataOutputStream(new FileOutputStream(file)); + } catch (FileNotFoundException e) { + e.printStackTrace(); + System.exit(1); + return; + } + try { + for (Entry entry : params.entrySet()) { + os.writeBytes(entry.getKey() + "=" + entry.getValue() + "\n"); + } + os.writeBytes("End\n"); + } catch (IOException e) { + e.printStackTrace(); + System.exit(1); + } + } + + void write(TermEntry tt) { + try { + TermEntryReaderWriter.getInstance().writeObject(tt, os); + } catch (IOException e) { + e.printStackTrace(); + System.exit(1); + } + counter ++; + } + + void close() { + try { + os.close(); + } catch (IOException e) { + e.printStackTrace(); + System.exit(1); + } + System.out.println("Written new file with " + counter + " entries."); + } + + public boolean isFull() { + return counter >= 1000000; + } +} diff --git a/uploader/src/freenet/library/uploader/TermEntryReaderIterator.java b/uploader/src/freenet/library/uploader/TermEntryReaderIterator.java new file mode 100644 index 00000000..2c0845e5 --- /dev/null +++ b/uploader/src/freenet/library/uploader/TermEntryReaderIterator.java @@ -0,0 +1,84 @@ +package freenet.library.uploader; + +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import plugins.Library.index.TermEntry; +import plugins.Library.index.TermEntryReaderWriter; + +class TermEntryReaderIterator implements Iterable { + private DataInputStream is; + private Map header; + + public TermEntryReaderIterator(DataInputStream s) { + is = s; + header = new HashMap(5); + String line = ""; + int laps = 0; + do { + try { + line = is.readLine(); + String[] parts = line.split("=", 2); + if (parts.length >= 2) { + header.put(parts[0], parts[1]); + } + } catch (IOException e) { + System.err.println("Error: Not closed header."); + System.exit(1); + } + if (laps > 100) { + System.err.println("Error: Cannot get out of file header."); + System.exit(1); + } + } while (!"End".equals(line)); + } + + @Override + public Iterator iterator() { + return new Iterator() { + TermEntry lastRead = null; + + @Override + public boolean hasNext() { + if (lastRead != null) { + return true; + } + lastRead = next(); + return lastRead != null; + } + + @Override + public TermEntry next() { + if (lastRead != null) { + TermEntry t = lastRead; + lastRead = null; + return t; + } + try { + return TermEntryReaderWriter.getInstance().readObject(is); + } catch (EOFException e) { + return null; + } catch (IOException e) { + System.out.println("Cannot understand read file:"); + e.printStackTrace(); + return null; + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + }; + } + + Map getHeader() { + return Collections.unmodifiableMap(header); + } +} \ No newline at end of file diff --git a/uploader/src/freenet/library/uploader/UploaderLibrary.java b/uploader/src/freenet/library/uploader/UploaderLibrary.java new file mode 100644 index 00000000..2ab62494 --- /dev/null +++ b/uploader/src/freenet/library/uploader/UploaderLibrary.java @@ -0,0 +1,110 @@ +/* This code is part of Freenet. It is distributed under the GNU General + * Public License, version 2 (or at your option any later version). See + * http://www.gnu.org/ for further details of the GPL. */ +package freenet.library.uploader; + +import java.io.File; +import java.security.MessageDigest; + +import plugins.Library.ArchiverFactory; +import plugins.Library.Priority; +import plugins.Library.io.ObjectStreamReader; +import plugins.Library.io.ObjectStreamWriter; +import plugins.Library.io.serial.LiveArchiver; +import plugins.Library.util.exec.SimpleProgress; + +import net.pterodactylus.fcp.FcpConnection; + +/** + * Library class is the api for others to use search facilities, it is used by the interfaces + * @author MikeB + */ +final public class UploaderLibrary implements ArchiverFactory { + + public static final String BOOKMARK_PREFIX = "bookmark:"; + public static final String DEFAULT_INDEX_SITE = BOOKMARK_PREFIX + "liberty-of-information" + " " + BOOKMARK_PREFIX + "free-market-free-people" + " " + + BOOKMARK_PREFIX + "gotcha" + " " + BOOKMARK_PREFIX + "wanna" + " " + BOOKMARK_PREFIX + "wanna.old" + " " + BOOKMARK_PREFIX + "gogo"; + private static int version = 36; + public static final String plugName = "Library " + getVersion(); + + public static String getPlugName() { + return plugName; + } + + public static long getVersion() { + return version; + } + + /** + ** Library singleton. + */ + private static UploaderLibrary lib; + public static UploaderLibrary getInstance() { + if (lib == null) { + lib = new UploaderLibrary(); + } + return lib; + } + + public static FcpConnection fcpConnection; + + public synchronized static void init(FcpConnection connection) { + fcpConnection = connection; + } + + + public static String convertToHex(byte[] data) { + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < data.length; i++) { + int halfbyte = (data[i] >>> 4) & 0x0F; + int two_halfs = 0; + do { + if ((0 <= halfbyte) && (halfbyte <= 9)) + buf.append((char) ('0' + halfbyte)); + else + buf.append((char) ('a' + (halfbyte - 10))); + halfbyte = data[i] & 0x0F; + } while (two_halfs++ < 1); + } + return buf.toString(); + } + + //this function will return the String representation of the MD5 hash for the input string + public static String MD5(String text) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] b = text.getBytes("UTF-8"); + md.update(b, 0, b.length); + byte[] md5hash = md.digest(); + return convertToHex(md5hash); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public + LiveArchiver + newArchiver(S rw, String mime, int size, + Priority priorityLevel) { + return new FcpArchiver(fcpConnection, + new File(UploaderPaths.LIBRARY_CACHE), + rw, + mime, size, priorityLevel); + } + + @Override + public + LiveArchiver + newArchiver(S rw, String mime, int size, + LiveArchiver archiver) { + Priority priorityLevel = Priority.Bulk; + /* + if (archiver != null && + archiver isinstance ??) { + priorityLevel = ((??) archiver).getPriorityLevel(); + } + */ + return newArchiver(rw, mime, size, priorityLevel); + } +} diff --git a/uploader/src/freenet/library/uploader/UploaderPaths.java b/uploader/src/freenet/library/uploader/UploaderPaths.java new file mode 100644 index 00000000..5be870e2 --- /dev/null +++ b/uploader/src/freenet/library/uploader/UploaderPaths.java @@ -0,0 +1,44 @@ +package freenet.library.uploader; + +public class UploaderPaths { + static final int MAX_HANDLING_COUNT = 5; + // When pushing is broken, allow max handling to reach this level before stalling forever to prevent running out of disk space. + + + /** idxDisk gets merged into idxFreenet this long after the last merge completed. */ + static final long MAX_TIME = 24*60*60*1000L; + + /** idxDisk gets merged into idxFreenet after this many incoming updates from Spider. */ + static final int MAX_UPDATES = 16; + + /** idxDisk gets merged into idxFreenet after it has grown to this many terms. + * Note that the entire main tree of terms (not the sub-trees with the positions and urls in) must + * fit into memory during the merge process. */ + static final int MAX_TERMS = 100*1000; + + /** idxDisk gets merged into idxFreenet after it has grown to this many terms. + * Note that the entire main tree of terms (not the sub-trees with the positions and urls in) must + * fit into memory during the merge process. */ + static final int MAX_TERMS_NOT_UPLOADED = 10*1000; + + /** Maximum size of a single entry, in TermPageEntry count, on disk. If we exceed this we force an + * insert-to-freenet and move on to a new disk index. The problem is that the merge to Freenet has + * to keep the whole of each entry in RAM. This is only true for the data being merged in - the + * on-disk index - and not for the data on Freenet, which is pulled on demand. SCALABILITY */ + static final int MAX_DISK_ENTRY_SIZE = 10000; + + /** Like pushNumber, the number of the current disk dir, used to create idxDiskDir. */ + static final String DISK_DIR_PREFIX = "library-temp-index-"; + + static final String LAST_URL_FILENAME = "library.index.lastpushed.chk"; + static final String PRIV_URI_FILENAME = "library.index.privkey"; + static final String PUB_URI_FILENAME = "library.index.pubkey"; + static final String EDITION_FILENAME = "library.index.next-edition"; + + static final String LAST_DISK_FILENAME = "library.index.lastpushed.disk"; + + static final String BASE_FILENAME_DATA = "library.index."; + static final String BASE_FILENAME_PUSH_DATA = BASE_FILENAME_DATA + "data."; + + static final String LIBRARY_CACHE = "library-spider-pushed-data-cache"; +} diff --git a/uploader/src/freenet/library/uploader/finest_logging.properties b/uploader/src/freenet/library/uploader/finest_logging.properties new file mode 100644 index 00000000..dd78149c --- /dev/null +++ b/uploader/src/freenet/library/uploader/finest_logging.properties @@ -0,0 +1,48 @@ +############################################################ +# Global properties +############################################################ + +# "handlers" specifies a comma separated list of log Handler +# classes. These handlers will be installed during VM startup. +# Note that these classes must be on the system classpath. +# By default we only configure a ConsoleHandler, which will only +# show messages at the INFO and above levels. +handlers= java.util.logging.ConsoleHandler + +# Default global logging level. +# This specifies which kinds of events are logged across +# all loggers. For any given facility this global level +# can be overriden by a facility specific level +# Note that the ConsoleHandler also has a separate level +# setting to limit messages printed to the console. +.level= FINEST + +############################################################ +# Handler specific properties. +# Describes specific configuration info for Handlers. +############################################################ + +# default file output is in user's home directory. +java.util.logging.FileHandler.pattern = %h/java%u.log +java.util.logging.FileHandler.limit = 50000 +java.util.logging.FileHandler.count = 1 +java.util.logging.FileHandler.formatter = java.util.logging.XMLFormatter + +# Limit the message that are printed on the console to INFO and above. +java.util.logging.ConsoleHandler.level = FINEST +java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter + +# Example to customize the SimpleFormatter output format +# to print one-line log message like this: +# : [] +# +# java.util.logging.SimpleFormatter.format=%4$s: %5$s [%1$tc]%n + +############################################################ +# Facility specific properties. +# Provides extra control for each logger. +############################################################ + +# For example, set the com.xyz.foo logger to only log SEVERE +# messages: +# com.xyz.foo.level = SEVERE diff --git a/uploader/test/freenet/library/uploader/MergerComparatorTest.java b/uploader/test/freenet/library/uploader/MergerComparatorTest.java new file mode 100644 index 00000000..2102b66d --- /dev/null +++ b/uploader/test/freenet/library/uploader/MergerComparatorTest.java @@ -0,0 +1,30 @@ +package freenet.library.uploader; + +import junit.framework.TestCase; + +public class MergerComparatorTest extends TestCase { + + public void testComparator() { + assertTrue(Merger.comparator.compare("a", "b") < 0); + assertTrue(Merger.comparator.compare("b", "a") > 0); + assertTrue(Merger.comparator.compare("a", "a") == 0); + + assertTrue(Merger.comparator.compare("3", "5") < 0); + assertTrue(Merger.comparator.compare("7", "5") > 0); + assertTrue(Merger.comparator.compare("4", "4") == 0); + + assertTrue(Merger.comparator.compare("a", "ab") < 0); + assertTrue(Merger.comparator.compare("ab", "a") > 0); + + assertTrue(Merger.comparator.compare("abc4", "abc00004") == 0); + assertTrue(Merger.comparator.compare("abc4", "abc00005") < 0); + assertTrue(Merger.comparator.compare("abc5", "abc00004") > 0); + assertTrue(Merger.comparator.compare("abc00003", "abc4") < 0); + assertTrue(Merger.comparator.compare("abc00004", "abc3") > 0); + + assertTrue(Merger.comparator.compare("abc4a", "abc00004a") == 0); + assertTrue(Merger.comparator.compare("abc4a", "abc00004b") < 0); + assertTrue(Merger.comparator.compare("abc4b", "abc00004a") > 0); + } + +}