diff --git a/.gitignore b/.gitignore index 8068abf..2e8d1ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ +*.db +dbs .DS_Store target *.iws *.iml *.ipr .idea - +dependency-reduced-pom.xml +.shell_history +neo4j.conf diff --git a/api/pom.xml b/api/pom.xml new file mode 100644 index 0000000..fd42edf --- /dev/null +++ b/api/pom.xml @@ -0,0 +1,16 @@ + + 4.0.0 + + + org.neo4j.util + store-util-parent + 3.2.0 + + + store-util-api + 3.2.0 + jar + + store-util-api + diff --git a/api/src/main/java/org/neo4j/tool/api/NodeInfo.java b/api/src/main/java/org/neo4j/tool/api/NodeInfo.java new file mode 100644 index 0000000..b944c5f --- /dev/null +++ b/api/src/main/java/org/neo4j/tool/api/NodeInfo.java @@ -0,0 +1,17 @@ +package org.neo4j.tool.api; + +import java.util.Map; + +/** +* @author mh +* @since 10.06.14 +*/ +public class NodeInfo { + public long id; + public Map data; + public String[] labels; + + public NodeInfo(long id) { + this.id = id; + } +} diff --git a/api/src/main/java/org/neo4j/tool/api/RelInfo.java b/api/src/main/java/org/neo4j/tool/api/RelInfo.java new file mode 100644 index 0000000..07a62a1 --- /dev/null +++ b/api/src/main/java/org/neo4j/tool/api/RelInfo.java @@ -0,0 +1,23 @@ +package org.neo4j.tool.api; + +import java.util.Map; + +/** +* @author mh +* @since 10.06.14 +*/ +public class RelInfo { + public long id = -1; + public long from; + public long to; + public Map data; + public String type; + + public RelInfo(long id) { + this.id = id; + } + + public String toString() { + return from + "-[:" + type + "]" + "->" + to; + } +} diff --git a/api/src/main/java/org/neo4j/tool/api/StoreHandler.java b/api/src/main/java/org/neo4j/tool/api/StoreHandler.java new file mode 100644 index 0000000..64eff5b --- /dev/null +++ b/api/src/main/java/org/neo4j/tool/api/StoreHandler.java @@ -0,0 +1,13 @@ +package org.neo4j.tool.api; + +import java.io.IOException; + +/** +* @author mh +* @since 10.06.14 +*/ +public interface StoreHandler { + void init(String dir, String pageCache) throws IOException; + void shutdown(); + void flush(); +} diff --git a/api/src/main/java/org/neo4j/tool/api/StoreReader.java b/api/src/main/java/org/neo4j/tool/api/StoreReader.java new file mode 100644 index 0000000..888ba09 --- /dev/null +++ b/api/src/main/java/org/neo4j/tool/api/StoreReader.java @@ -0,0 +1,13 @@ +package org.neo4j.tool.api; + +/** +* @author mh +* @since 10.06.14 +*/ +public interface StoreReader extends StoreHandler { + NodeInfo readNode(long id); + RelInfo readRel(long id); + boolean nodeExists(long node); + long highestNodeId(); + long highestRelId(); +} diff --git a/api/src/main/java/org/neo4j/tool/api/StoreWriter.java b/api/src/main/java/org/neo4j/tool/api/StoreWriter.java new file mode 100644 index 0000000..c56a0f5 --- /dev/null +++ b/api/src/main/java/org/neo4j/tool/api/StoreWriter.java @@ -0,0 +1,10 @@ +package org.neo4j.tool.api; + +/** +* @author mh +* @since 10.06.14 +*/ +public interface StoreWriter extends StoreHandler { + void createNode(NodeInfo node); + void createRelationship(RelInfo rel); +} diff --git a/copy-store.sh b/copy-store.sh new file mode 100755 index 0000000..60ce18a --- /dev/null +++ b/copy-store.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +EDITION=${1-community} +shift +SRC=${1%%:*} +SVER=${1##*:} +DST=${2%%:*} +DVER=${2##*:} +SKIP_RELS=$3 +SKIP_PROPS=$4 +SKIP_LABELS=$5 +DELETE_NODES=$6 +KEEP_NODE_IDS=$7 +HEAP=4G +CACHE=2G +CACHE_SRC=1G +#$CACHE +echo "Usage: copy-store.sh [community|enterprise] source.db:version target.db:version [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip] [Labels,To,Delete,Nodes]" + +if [[ "$EDITION" != "enterprise" && "$EDITION" != "community" ]] +then + echo "ATTENTION: The parameter '$EDITION' you passed in for the edition is neither 'community' nor 'enterprise'. Aborting." + exit +fi +if [[ "$SRC" = "" || "$DST" = "" ]] +then + echo "ATTENTION: Source '$SRC' or target '$DST' not provided. Aborting." + exit +fi + +if [[ ! -d $SRC ]] +then + echo "ATTENTION: Source '$SRC' is not a directory. Aborting." + exit +fi + +if [[ -d $DST ]] +then + echo "ATTENTION: Target '$DST' already exists. Aborting." + exit +fi + +echo "Using: Heap $HEAP Pagecache $CACHE Edition '$EDITION' from '$SRC' (version $SVER) to '$DST' (version $DVER) skipping labels: '$SKIP_LABELS', removing nodes with labels: '$DELETE_NODES' rels: '$SKIP_RELS' props '$SKIP_PROPS' Keeping Node Ids: $KEEP_NODE_IDS" +echo +echo "Please note that you will need this memory ($CACHE + $CACHE_SRC + $HEAP) as it opens 2 databases one for reading and one for writing." +# heap config + +echo "Building binary packages (jars) for all versions" +mvn install -P${EDITION} 2>&1 | grep -v '\[\(INFO\|WARNING\)\]' + +pushd `pwd` + +cd runner + +export MAVEN_OPTS="-Xmx$HEAP -Xms$HEAP -XX:+UseG1GC" +mvn exec:java -P${EDITION} -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=$CACHE -Ddbms.pagecache.memory.source=$CACHE_SRC \ + -Dexec.args="$SRC:$SVER $DST:$DVER $SKIP_RELS $SKIP_PROPS $SKIP_LABELS $DELETE_NODES $KEEP_NODE_IDS" +popd diff --git a/neo4j22/pom.xml b/neo4j22/pom.xml new file mode 100644 index 0000000..ae39aba --- /dev/null +++ b/neo4j22/pom.xml @@ -0,0 +1,71 @@ + + 4.0.0 + + + org.neo4j.util + store-util-parent + 3.2.0 + + + store-util-impl-2.2 + 3.2.0 + jar + + store-util-impl-2.2 + + + 2.2.10 + + + + + org.neo4j + neo4j-kernel + ${neo4j.version} + + + org.neo4j + neo4j-lucene-index + ${neo4j.version} + + + org.neo4j.util + store-util-api + 3.2.0 + + + + + + + maven-shade-plugin + 3.0.0 + + + package + + shade + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + diff --git a/neo4j22/src/main/java/org/neo4j/tool/impl/StoreBatchHandlerImpl.java b/neo4j22/src/main/java/org/neo4j/tool/impl/StoreBatchHandlerImpl.java new file mode 100644 index 0000000..3c3b7a5 --- /dev/null +++ b/neo4j22/src/main/java/org/neo4j/tool/impl/StoreBatchHandlerImpl.java @@ -0,0 +1,96 @@ +package org.neo4j.tool.impl; + +import org.neo4j.graphdb.DynamicLabel; +import org.neo4j.graphdb.DynamicRelationshipType; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.helpers.collection.MapUtil; +import org.neo4j.tool.api.NodeInfo; +import org.neo4j.tool.api.RelInfo; +import org.neo4j.tool.api.StoreHandler; +import org.neo4j.unsafe.batchinsert.*; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; + +/** + * @author mh + * @since 09.06.14 + */ +public class StoreBatchHandlerImpl implements StoreHandler { + + static Label[] NO_LABELS = new Label[0]; + public static final String[] NO_LABELS_NAMES = new String[0]; + private Runnable flusher; + + protected String[] toLabelStrings(long id) { + Iterator