Revert store

.gitignore

@@ -1,7 +1,11 @@
+*.db
+dbs
 .DS_Store
 target
 *.iws
 *.iml
 *.ipr
 .idea
-
+dependency-reduced-pom.xml
+.shell_history
+neo4j.conf

api/pom.xml

@@ -0,0 +1,16 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.neo4j.util</groupId>
+        <artifactId>store-util-parent</artifactId>
+        <version>3.2.0</version>
+    </parent>
+
+    <artifactId>store-util-api</artifactId>
+    <version>3.2.0</version>
+    <packaging>jar</packaging>
+
+    <name>store-util-api</name>
+</project>

api/src/main/java/org/neo4j/tool/api/NodeInfo.java

@@ -0,0 +1,17 @@
+package org.neo4j.tool.api;
+
+import java.util.Map;
+
+/**
+* @author mh
+* @since 10.06.14
+*/
+public class NodeInfo {
+    public long id;
+    public Map<String,Object> data;
+    public String[] labels;
+
+    public NodeInfo(long id) {
+        this.id = id;
+    }
+}

api/src/main/java/org/neo4j/tool/api/RelInfo.java

@@ -0,0 +1,23 @@
+package org.neo4j.tool.api;
+
+import java.util.Map;
+
+/**
+* @author mh
+* @since 10.06.14
+*/
+public class RelInfo {
+    public long id = -1;
+    public long from;
+    public long to;
+    public Map<String,Object> data;
+    public String type;
+
+    public RelInfo(long id) {
+        this.id = id;
+    }
+
+    public String toString() {
+        return from + "-[:" + type + "]" + "->" + to;
+    }
+}

api/src/main/java/org/neo4j/tool/api/StoreHandler.java

@@ -0,0 +1,13 @@
+package org.neo4j.tool.api;
+
+import java.io.IOException;
+
+/**
+* @author mh
+* @since 10.06.14
+*/
+public interface StoreHandler {
+    void init(String dir, String pageCache) throws IOException;
+    void shutdown();
+    void flush();
+}

api/src/main/java/org/neo4j/tool/api/StoreReader.java

@@ -0,0 +1,13 @@
+package org.neo4j.tool.api;
+
+/**
+* @author mh
+* @since 10.06.14
+*/
+public interface StoreReader extends StoreHandler {
+    NodeInfo readNode(long id);
+    RelInfo readRel(long id);
+    boolean nodeExists(long node);
+    long highestNodeId();
+    long highestRelId();
+}

api/src/main/java/org/neo4j/tool/api/StoreWriter.java

@@ -0,0 +1,10 @@
+package org.neo4j.tool.api;
+
+/**
+* @author mh
+* @since 10.06.14
+*/
+public interface StoreWriter extends StoreHandler {
+    void createNode(NodeInfo node);
+    void createRelationship(RelInfo rel);
+}

copy-store.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+EDITION=${1-community}
+shift
+SRC=${1%%:*}
+SVER=${1##*:}
+DST=${2%%:*}
+DVER=${2##*:}
+SKIP_RELS=$3
+SKIP_PROPS=$4
+SKIP_LABELS=$5
+DELETE_NODES=$6
+KEEP_NODE_IDS=$7
+HEAP=4G
+CACHE=2G
+CACHE_SRC=1G
+#$CACHE
+echo "Usage: copy-store.sh [community|enterprise] source.db:version target.db:version [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip] [Labels,To,Delete,Nodes]"
+
+if [[ "$EDITION" != "enterprise" && "$EDITION" != "community" ]]
+then
+    echo "ATTENTION: The parameter '$EDITION' you passed in for the edition is neither 'community' nor 'enterprise'. Aborting."
+    exit
+fi
+if [[ "$SRC" = "" || "$DST" = "" ]]
+then
+    echo "ATTENTION: Source '$SRC' or target '$DST' not provided. Aborting."
+    exit
+fi
+
+if [[ ! -d $SRC ]]
+then
+    echo "ATTENTION: Source '$SRC' is not a directory. Aborting."
+    exit
+fi
+
+if [[ -d $DST ]]
+then
+    echo "ATTENTION: Target '$DST' already exists. Aborting."
+    exit
+fi
+
+echo "Using: Heap $HEAP Pagecache $CACHE Edition '$EDITION' from '$SRC' (version $SVER) to '$DST' (version $DVER) skipping labels: '$SKIP_LABELS', removing nodes with labels: '$DELETE_NODES' rels: '$SKIP_RELS' props '$SKIP_PROPS' Keeping Node Ids: $KEEP_NODE_IDS"
+echo
+echo "Please note that you will need this memory ($CACHE + $CACHE_SRC + $HEAP) as it opens 2 databases one for reading and one for writing."
+# heap config
+
+echo "Building binary packages (jars) for all versions"
+mvn install -P${EDITION} 2>&1 | grep -v '\[\(INFO\|WARNING\)\]'
+
+pushd `pwd`
+
+cd runner
+
+export MAVEN_OPTS="-Xmx$HEAP -Xms$HEAP -XX:+UseG1GC"
+mvn exec:java -P${EDITION} -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=$CACHE -Ddbms.pagecache.memory.source=$CACHE_SRC \
+      -Dexec.args="$SRC:$SVER $DST:$DVER $SKIP_RELS $SKIP_PROPS $SKIP_LABELS $DELETE_NODES $KEEP_NODE_IDS"
+popd

neo4j22/pom.xml

@@ -0,0 +1,71 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.neo4j.util</groupId>
+        <artifactId>store-util-parent</artifactId>
+        <version>3.2.0</version>
+    </parent>
+
+    <artifactId>store-util-impl-2.2</artifactId>
+    <version>3.2.0</version>
+    <packaging>jar</packaging>
+
+    <name>store-util-impl-2.2</name>
+
+    <properties>
+        <neo4j.version>2.2.10</neo4j.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.neo4j</groupId>
+            <artifactId>neo4j-kernel</artifactId>
+            <version>${neo4j.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.neo4j</groupId>
+            <artifactId>neo4j-lucene-index</artifactId>
+            <version>${neo4j.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.neo4j.util</groupId>
+            <artifactId>store-util-api</artifactId>
+            <version>3.2.0</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.0.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <filters>
+                        <filter>
+                            <artifact>*:*</artifact>
+                            <excludes>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                            </excludes>
+                        </filter>
+                    </filters>
+                    <transformers>
+                        <transformer
+                                implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                    </transformers>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

neo4j22/src/main/java/org/neo4j/tool/impl/StoreBatchHandlerImpl.java

@@ -0,0 +1,96 @@
+package org.neo4j.tool.impl;
+
+import org.neo4j.graphdb.DynamicLabel;
+import org.neo4j.graphdb.DynamicRelationshipType;
+import org.neo4j.graphdb.Label;
+import org.neo4j.graphdb.RelationshipType;
+import org.neo4j.helpers.collection.MapUtil;
+import org.neo4j.tool.api.NodeInfo;
+import org.neo4j.tool.api.RelInfo;
+import org.neo4j.tool.api.StoreHandler;
+import org.neo4j.unsafe.batchinsert.*;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * @author mh
+ * @since 09.06.14
+ */
+public class StoreBatchHandlerImpl implements StoreHandler {
+
+    static Label[] NO_LABELS = new Label[0];
+    public static final String[] NO_LABELS_NAMES = new String[0];
+    private Runnable flusher;
+
+    protected String[] toLabelStrings(long id) {
+        Iterator<Label> labels = batchInserter.getNodeLabels(id).iterator();
+        if (labels == null || !labels.hasNext()) return NO_LABELS_NAMES;
+        String[] result = new String[50];
+        int i = 0;
+        while (labels.hasNext()) {
+            Label next = labels.next();
+            result[i++] = next.name();
+        }
+        return Arrays.copyOf(result, i);
+    }
+
+    public Label[] labels(NodeInfo node) {
+        if (node.labels == null || node.labels.length == 0) return NO_LABELS;
+        Label[] result = new Label[node.labels.length];
+        for (int i = 0; i < result.length; i++) { // todo cache
+            result[i] = DynamicLabel.label(node.labels[i]);
+        }
+        return result;
+    }
+
+    public RelationshipType type(RelInfo rel) {
+        return DynamicRelationshipType.withName(rel.type);
+    }
+
+    protected BatchInserter batchInserter;
+    protected String dir;
+
+    @Override
+    public void init(String dir, String pageCache) throws IOException {
+        this.dir = dir;
+        batchInserter = BatchInserters.inserter(dir, MapUtil.stringMap("dbms.pagecache.memory", pageCache));
+        flusher = getFlusher(batchInserter);
+    }
+
+    @Override
+    public void flush() {
+        flusher.run();
+    }
+
+    @Override
+    public void shutdown() {
+        batchInserter.shutdown();
+    }
+
+    private static Runnable getFlusher(final BatchInserter db) {
+        try {
+            Field field = BatchInserterImpl.class.getDeclaredField("recordAccess");
+            field.setAccessible(true);
+            final DirectRecordAccessSet recordAccessSet = (DirectRecordAccessSet) field.get(db);
+            final Field cacheField = DirectRecordAccess.class.getDeclaredField("batch");
+            cacheField.setAccessible(true);
+            return new Runnable() {
+                @Override public void run() {
+                    try {
+                        ((Map) cacheField.get(recordAccessSet.getNodeRecords())).clear();
+                        ((Map) cacheField.get(recordAccessSet.getRelRecords())).clear();
+                        ((Map) cacheField.get(recordAccessSet.getPropertyRecords())).clear();
+                    } catch (IllegalAccessException e) {
+                        throw new RuntimeException("Error clearing cache "+cacheField,e);
+                    }
+                }
+            };
+        } catch (IllegalAccessException | NoSuchFieldException e) {
+            throw new RuntimeException("Error accessing cache field ", e);
+        }
+    }
+}

neo4j22/src/main/java/org/neo4j/tool/impl/StoreBatchReaderImpl.java

@@ -0,0 +1,51 @@
+package org.neo4j.tool.impl;
+
+import org.neo4j.kernel.impl.store.NodeStore;
+import org.neo4j.tool.api.NodeInfo;
+import org.neo4j.tool.api.RelInfo;
+import org.neo4j.tool.api.StoreReader;
+import org.neo4j.unsafe.batchinsert.BatchRelationship;
+
+import java.io.File;
+
+/**
+ * @author mh
+ * @since 09.06.14
+ */
+public class StoreBatchReaderImpl extends StoreBatchHandlerImpl implements StoreReader {
+
+
+    @Override
+    public NodeInfo readNode(long id) {
+        NodeInfo node = new NodeInfo(id);
+        node.data = batchInserter.getNodeProperties(id);
+        node.labels = toLabelStrings(id);
+        return node;
+    }
+
+    @Override
+    public RelInfo readRel(long id) {
+        RelInfo rel = new RelInfo(id);
+        rel.data = batchInserter.getRelationshipProperties(id);
+        BatchRelationship batchRelationship = batchInserter.getRelationshipById(id);
+        rel.from = batchRelationship.getStartNode();
+        rel.to = batchRelationship.getEndNode();
+        rel.type = batchRelationship.getType().name();
+        return rel;
+    }
+
+    @Override
+    public boolean nodeExists(long node) {
+        return batchInserter.nodeExists(node);
+    }
+
+    @Override
+    public long highestNodeId() {
+        return new File(dir, "neostore.nodestore.db").length() /  NodeStore.RECORD_SIZE;
+    }
+
+    @Override
+    public long highestRelId() {
+        return new File(dir, "neostore.relationshipstore.db").length() / NodeStore.RECORD_SIZE;
+    }
+}