From c5d76cf03e5adf73582d18288d29ed538ac307ca Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Sun, 28 Sep 2014 22:57:31 +0200
Subject: [PATCH 01/33] update to 2.1.4

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 112b95a..cea89c3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>2.0.1</version>
+    <version>2.1.4</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>

From e81fde1831052380168337a483b7726247e2dda8 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Sun, 28 Sep 2014 22:58:13 +0200
Subject: [PATCH 02/33] Update readme.md

---
 readme.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readme.md b/readme.md
index d28ab0b..b72dd06 100644
--- a/readme.md
+++ b/readme.md
@@ -7,7 +7,7 @@ Ignores broken nodes and relationships.
 Also useful to skip no longer wanted properties or relationships with a certain type. Good for store compaction as it
 rewrites the store file reclaiming space that is sitting empty.
 
-Change the Neo4j version in pom.xml before running. (Currently 1.9.5)
+Change the Neo4j version in pom.xml before running. (Currently 2.1.4)
 
 ### Store Copy
 

From a7aed40154988fc8091ebf830f1179c5f883a18e Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Thu, 9 Oct 2014 01:21:25 +0200
Subject: [PATCH 03/33] fixed batch-caches in BatchInserterImpl filling up
 endlessly

---
 src/main/java/org/neo4j/tool/StoreCopy.java | 36 ++++++++++++++++++---
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index 95fe717..27dd09d 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -37,7 +37,8 @@ public static Map<String, String> config() {
                 "neostore.propertystore.db.arrays.mapped_memory", "300M",
                 "neostore.propertystore.db.index.keys.mapped_memory", "100M",
                 "neostore.propertystore.db.index.mapped_memory", "100M",
-                "cache_type", "weak"
+                "use_memory_mapped_buffers","true",
+                "cache_type", "none"
         );
     }
 
@@ -74,8 +75,9 @@ private static void copyStore(String sourceDir, String targetDir, Set<String> ig
         BatchInserter sourceDb = BatchInserters.inserter(source.getAbsolutePath(), config());
         logs = new PrintWriter(new FileWriter(new File(target, "store-copy.log")));
 
+        long firstNode = firstNode(sourceDb, highestIds.first());
         copyNodes(sourceDb, targetDb, ignoreProperties, ignoreLabels, highestIds.first());
-        copyRelationships(sourceDb, targetDb, ignoreRelTypes, ignoreProperties, highestIds.other());
+        copyRelationships(sourceDb, targetDb, ignoreRelTypes, ignoreProperties, highestIds.other(), firstNode);
 
         targetDb.shutdown();
         sourceDb.shutdown();
@@ -103,7 +105,7 @@ private static void copyIndex(File source, File target) throws IOException {
         }
     }
 
-    private static void copyRelationships(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreRelTypes, Set<String> ignoreProperties, long highestRelId) {
+    private static void copyRelationships(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreRelTypes, Set<String> ignoreProperties, long highestRelId, long firstNode) {
         long time = System.currentTimeMillis();
         long relId = 0;
         long notFound = 0;
@@ -117,12 +119,25 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
             }
             if (ignoreRelTypes.contains(rel.getType().name().toLowerCase())) continue;
             createRelationship(targetDb, sourceDb, rel, ignoreProperties);
-            if (relId % 1000 == 0) System.out.print(".");
-            if (relId % 100000 == 0) System.out.println(" " + rel);
+            if (relId % 1000 == 0) {
+				System.out.print(".");
+			}
+            if (relId % 100000 == 0) {
+                flushCache(sourceDb, firstNode);
+                System.out.println(" " + rel.getId());
+            }
         }
         System.out.println("\n copying of "+relId+" relationships took "+(System.currentTimeMillis()-time)+" ms. Not found "+notFound);
     }
 
+    private static long firstNode(BatchInserter sourceDb, long highestNodeId) {
+        int node = -1;
+        while (++node <= highestNodeId) {
+            if (sourceDb.nodeExists(node)) return node;
+        }
+        return -1;
+    }
+
     private static void createRelationship(BatchInserter targetDb, BatchInserter sourceDb, BatchRelationship rel, Set<String> ignoreProperties) {
         long startNodeId = rel.getStartNode();
         long endNodeId = rel.getEndNode();
@@ -142,6 +157,7 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
             targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node,ignoreLabels));
             if (node % 1000 == 0) System.out.print(".");
             if (node % 100000 == 0) {
+                flushCache(sourceDb, node);
                 logs.flush();
                 System.out.println(" " + node);
             }
@@ -149,6 +165,16 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
         System.out.println("\n copying of " + node + " nodes took " + (System.currentTimeMillis() - time) + " ms.");
     }
 
+    private static void flushCache(BatchInserter sourceDb, long node) {
+        Map<String, Object> nodeProperties = sourceDb.getNodeProperties(node);
+        Iterator<Map.Entry<String, Object>> iterator = nodeProperties.entrySet().iterator();
+        if (iterator.hasNext()) {
+            Map.Entry<String, Object> firstProp = iterator.next();
+            sourceDb.setNodeProperty(node, firstProp.getKey(), firstProp.getValue()); // force flush
+            System.out.print("F");
+        }
+    }
+
     private static Label[] labelsArray(BatchInserter db, long node, Set<String> ignoreLabels) {
         Collection<Label> labels = IteratorUtil.asCollection(db.getNodeLabels(node));
         if (labels.isEmpty()) return NO_LABELS;

From 4063e2d81365353a6ec6ac98fc65f023ed5cb712 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Fri, 10 Oct 2014 02:25:43 +0200
Subject: [PATCH 04/33] another fix for memory buildup in the batch-inserter

---
 src/main/java/org/neo4j/tool/StoreCopy.java | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index 27dd09d..bd76ad7 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -31,10 +31,10 @@ public class StoreCopy {
     public static Map<String, String> config() {
         return (Map) MapUtil.map(
                 "neostore.nodestore.db.mapped_memory", "100M",
-                "neostore.relationshipstore.db.mapped_memory", "500M",
-                "neostore.propertystore.db.mapped_memory", "300M",
-                "neostore.propertystore.db.strings.mapped_memory", "1G",
-                "neostore.propertystore.db.arrays.mapped_memory", "300M",
+                "neostore.relationshipstore.db.mapped_memory", "100M",
+                "neostore.propertystore.db.mapped_memory", "100M",
+                "neostore.propertystore.db.strings.mapped_memory", "100m",
+                "neostore.propertystore.db.arrays.mapped_memory", "100M",
                 "neostore.propertystore.db.index.keys.mapped_memory", "100M",
                 "neostore.propertystore.db.index.mapped_memory", "100M",
                 "use_memory_mapped_buffers","true",
@@ -170,6 +170,7 @@ private static void flushCache(BatchInserter sourceDb, long node) {
         Iterator<Map.Entry<String, Object>> iterator = nodeProperties.entrySet().iterator();
         if (iterator.hasNext()) {
             Map.Entry<String, Object> firstProp = iterator.next();
+            sourceDb.nodeHasProperty(node,firstProp.getKey());
             sourceDb.setNodeProperty(node, firstProp.getKey(), firstProp.getValue()); // force flush
             System.out.print("F");
         }

From 3b85b0de30c06593ee4621e00954c6c88f70d104 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Fri, 10 Oct 2014 02:26:57 +0200
Subject: [PATCH 05/33] another fix for memory buildup in the batch-inserter

---
 src/main/java/org/neo4j/tool/StoreCopy.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index bd76ad7..e8415fd 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -31,10 +31,10 @@ public class StoreCopy {
     public static Map<String, String> config() {
         return (Map) MapUtil.map(
                 "neostore.nodestore.db.mapped_memory", "100M",
-                "neostore.relationshipstore.db.mapped_memory", "100M",
-                "neostore.propertystore.db.mapped_memory", "100M",
-                "neostore.propertystore.db.strings.mapped_memory", "100m",
-                "neostore.propertystore.db.arrays.mapped_memory", "100M",
+                "neostore.relationshipstore.db.mapped_memory", "500M",
+                "neostore.propertystore.db.mapped_memory", "300M",
+                "neostore.propertystore.db.strings.mapped_memory", "1G",
+                "neostore.propertystore.db.arrays.mapped_memory", "300M",
                 "neostore.propertystore.db.index.keys.mapped_memory", "100M",
                 "neostore.propertystore.db.index.mapped_memory", "100M",
                 "use_memory_mapped_buffers","true",

From e4e89362d482bcea76248b4cf701fcb6c9fb4da9 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Wed, 15 Apr 2015 16:04:31 +0200
Subject: [PATCH 06/33] update to 2.1.8

---
 pom.xml                                     | 2 +-
 src/main/java/org/neo4j/tool/StoreCopy.java | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index cea89c3..1245ee3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>2.1.4</version>
+    <version>2.1.8</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>
diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index e8415fd..2a0d40c 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -32,6 +32,7 @@ public static Map<String, String> config() {
         return (Map) MapUtil.map(
                 "neostore.nodestore.db.mapped_memory", "100M",
                 "neostore.relationshipstore.db.mapped_memory", "500M",
+                "neostore.relationshipgroupstore.db.mapped_memory", "10M",
                 "neostore.propertystore.db.mapped_memory", "300M",
                 "neostore.propertystore.db.strings.mapped_memory", "1G",
                 "neostore.propertystore.db.arrays.mapped_memory", "300M",

From 05a138aa7042241ef27c6558be8b786f9798173f Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Wed, 20 May 2015 10:37:41 +0200
Subject: [PATCH 07/33] update to Neo4j 2.2.1

---
 pom.xml                                       |  7 +++-
 .../java/org/neo4j/tool/StoreComparer.java    |  3 +-
 src/main/java/org/neo4j/tool/StoreCopy.java   | 35 +++++++++----------
 3 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/pom.xml b/pom.xml
index 1245ee3..9005d1a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>2.1.8</version>
+    <version>2.2.1</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>
@@ -20,6 +20,11 @@
             <version>4.11</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.neo4j</groupId>
+            <artifactId>neo4j-io</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.neo4j</groupId>
             <artifactId>neo4j-kernel</artifactId>
diff --git a/src/main/java/org/neo4j/tool/StoreComparer.java b/src/main/java/org/neo4j/tool/StoreComparer.java
index b790ff5..de57389 100644
--- a/src/main/java/org/neo4j/tool/StoreComparer.java
+++ b/src/main/java/org/neo4j/tool/StoreComparer.java
@@ -4,8 +4,7 @@
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
 import org.neo4j.helpers.collection.IteratorUtil;
 import org.neo4j.helpers.collection.MapUtil;
-import org.neo4j.kernel.EmbeddedGraphDatabase;
-import org.neo4j.kernel.impl.util.FileUtils;
+import org.neo4j.io.fs.FileUtils;
 
 import java.io.File;
 import java.io.IOException;
diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index 2a0d40c..a95e90b 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -5,10 +5,10 @@
 import org.neo4j.helpers.Pair;
 import org.neo4j.helpers.collection.IteratorUtil;
 import org.neo4j.helpers.collection.MapUtil;
+import org.neo4j.io.fs.FileUtils;
 import org.neo4j.kernel.GraphDatabaseAPI;
-import org.neo4j.kernel.impl.core.NodeManager;
-import org.neo4j.kernel.impl.nioneo.store.InvalidRecordException;
-import org.neo4j.kernel.impl.util.FileUtils;
+import org.neo4j.kernel.IdGeneratorFactory;
+import org.neo4j.kernel.IdType;
 import org.neo4j.unsafe.batchinsert.BatchInserter;
 import org.neo4j.unsafe.batchinsert.BatchInserters;
 import org.neo4j.unsafe.batchinsert.BatchRelationship;
@@ -88,9 +88,9 @@ private static void copyStore(String sourceDir, String targetDir, Set<String> ig
 
     private static Pair<Long, Long> getHighestNodeId(File source) {
         GraphDatabaseAPI api = (GraphDatabaseAPI) new GraphDatabaseFactory().newEmbeddedDatabase(source.getAbsolutePath());
-        NodeManager nodeManager = api.getDependencyResolver().resolveDependency(NodeManager.class);
-        long highestNodeId = nodeManager.getHighestPossibleIdInUse(Node.class);
-        long highestRelId = nodeManager.getHighestPossibleIdInUse(Relationship.class);
+        IdGeneratorFactory idGenerators = api.getDependencyResolver().resolveDependency(IdGeneratorFactory.class);
+        long highestNodeId = idGenerators.get(IdType.NODE).getHighestPossibleIdInUse();
+        long highestRelId = idGenerators.get(IdType.RELATIONSHIP).getHighestPossibleIdInUse();
         api.shutdown();
         return Pair.of(highestNodeId, highestRelId);
     }
@@ -114,18 +114,17 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
             BatchRelationship rel = null;
             try {
                 rel = sourceDb.getRelationshipById(relId++);
-            } catch (InvalidRecordException nfe) {
+                if (ignoreRelTypes.contains(rel.getType().name().toLowerCase())) continue;
+                createRelationship(targetDb, sourceDb, rel, ignoreProperties);
+                if (relId % 1000 == 0) {
+                    System.out.print(".");
+                }
+                if (relId % 100000 == 0) {
+                    flushCache(sourceDb, firstNode);
+                    System.out.println(" " + rel.getId());
+                }
+            } catch (Exception nfe) {
                 notFound++;
-                continue;
-            }
-            if (ignoreRelTypes.contains(rel.getType().name().toLowerCase())) continue;
-            createRelationship(targetDb, sourceDb, rel, ignoreProperties);
-            if (relId % 1000 == 0) {
-				System.out.print(".");
-			}
-            if (relId % 100000 == 0) {
-                flushCache(sourceDb, firstNode);
-                System.out.println(" " + rel.getId());
             }
         }
         System.out.println("\n copying of "+relId+" relationships took "+(System.currentTimeMillis()-time)+" ms. Not found "+notFound);
@@ -145,7 +144,7 @@ private static void createRelationship(BatchInserter targetDb, BatchInserter sou
         final RelationshipType type = rel.getType();
         try {
             targetDb.createRelationship(startNodeId, endNodeId, type, getProperties(sourceDb.getRelationshipProperties(rel.getId()), ignoreProperties));
-        } catch (InvalidRecordException ire) {
+        } catch (Exception ire) {
             addLog(rel, "create Relationship: " + startNodeId + "-[:" + type + "]" + "->" + endNodeId, ire.getMessage());
         }
     }

From 57704244736bdb4a669ce3adff4fc5c8acf8a7ee Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Fri, 22 May 2015 23:45:14 +0200
Subject: [PATCH 08/33] Update readme.md

---
 readme.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readme.md b/readme.md
index b72dd06..5b3b60c 100644
--- a/readme.md
+++ b/readme.md
@@ -7,7 +7,7 @@ Ignores broken nodes and relationships.
 Also useful to skip no longer wanted properties or relationships with a certain type. Good for store compaction as it
 rewrites the store file reclaiming space that is sitting empty.
 
-Change the Neo4j version in pom.xml before running. (Currently 2.1.4)
+Change the Neo4j version in pom.xml before running. (Currently 2.2.1)
 
 ### Store Copy
 

From 2ffa872662d1e7b143098c227395609fa27fbc9a Mon Sep 17 00:00:00 2001
From: Mike Benza <mbenza@alertlogic.com>
Date: Mon, 24 Aug 2015 15:06:02 -0500
Subject: [PATCH 09/33] Clean up StoreCopy output formatting

The StoreCopy tool's output formatting doesn't behave reliably when
there's a lot of compaction going on.  A lot of the '.'s are skipped and
the line-ending summaries are skipped when a node or relationship
with the current node ID or rel ID doesn't exist (because that
information is outputted after getting the node / relationship).  This
resolves that so that every line is approximately the same length and
there's a line per every 500000 nodes/relationships.

This change also ups the number of nodes per '.' and per line.
---
 src/main/java/org/neo4j/tool/StoreCopy.java | 24 +++++++++++----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index a95e90b..b0a8be9 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -113,16 +113,16 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
         while (relId <= highestRelId) {
             BatchRelationship rel = null;
             try {
-                rel = sourceDb.getRelationshipById(relId++);
-                if (ignoreRelTypes.contains(rel.getType().name().toLowerCase())) continue;
-                createRelationship(targetDb, sourceDb, rel, ignoreProperties);
-                if (relId % 1000 == 0) {
+                if (relId % 10000 == 0) {
                     System.out.print(".");
                 }
-                if (relId % 100000 == 0) {
+                if (relId % 500000 == 0) {
                     flushCache(sourceDb, firstNode);
-                    System.out.println(" " + rel.getId());
+                    System.out.println(" " + relId + " / " + highestRelId + " (" + 100 *((float)relId / highestRelId) + "%)");
                 }
+                rel = sourceDb.getRelationshipById(relId++);
+                if (ignoreRelTypes.contains(rel.getType().name().toLowerCase())) continue;
+                createRelationship(targetDb, sourceDb, rel, ignoreProperties);
             } catch (Exception nfe) {
                 notFound++;
             }
@@ -153,14 +153,16 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
         long time = System.currentTimeMillis();
         int node = -1;
         while (++node <= highestNodeId) {
-            if (!sourceDb.nodeExists(node)) continue;
-            targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node,ignoreLabels));
-            if (node % 1000 == 0) System.out.print(".");
-            if (node % 100000 == 0) {
+            if (node % 10000 == 0) {
+                System.out.print(".");
+            }
+            if (node % 500000 == 0) {
                 flushCache(sourceDb, node);
                 logs.flush();
-                System.out.println(" " + node);
+                System.out.println(" " + node + " / " + highestNodeId + " (" + 100 *((float)node / highestNodeId) + "%)");
             }
+            if (!sourceDb.nodeExists(node)) continue;
+            targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node,ignoreLabels));
         }
         System.out.println("\n copying of " + node + " nodes took " + (System.currentTimeMillis() - time) + " ms.");
     }

From 16dc9650a8f2db2e22c8645b5a8cbba05aeb653f Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Tue, 25 Aug 2015 10:34:15 +0200
Subject: [PATCH 10/33] Update to Neo4j 2.2.4

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 9005d1a..97d516b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>2.2.1</version>
+    <version>2.2.4</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>

From f219585bd60c408da7333f6fbe1d7b5839e07b83 Mon Sep 17 00:00:00 2001
From: ahmet kizilay <ahmet.kizilay@gmail.com>
Date: Tue, 25 Aug 2015 20:41:24 +0300
Subject: [PATCH 11/33] added try-catch block to the copyNodes method

similar to copyRelationships method, try-catch block is added
to skip faulty node records, and continue with copying.
---
 src/main/java/org/neo4j/tool/StoreCopy.java | 25 +++++++++++++--------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index b0a8be9..892b66c 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -152,19 +152,26 @@ private static void createRelationship(BatchInserter targetDb, BatchInserter sou
     private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreProperties, Set<String> ignoreLabels, long highestNodeId) {
         long time = System.currentTimeMillis();
         int node = -1;
+        long notFound = 0;
         while (++node <= highestNodeId) {
-            if (node % 10000 == 0) {
-                System.out.print(".");
+            try {
+              if (node % 10000 == 0) {
+                  System.out.print(".");
+              }
+              if (node % 500000 == 0) {
+                  flushCache(sourceDb, node);
+                  logs.flush();
+                  System.out.println(" " + node + " / " + highestNodeId + " (" + 100 *((float)node / highestNodeId) + "%)");
+              }
+
+              if (!sourceDb.nodeExists(node)) continue;
+              targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node,ignoreLabels));
             }
-            if (node % 500000 == 0) {
-                flushCache(sourceDb, node);
-                logs.flush();
-                System.out.println(" " + node + " / " + highestNodeId + " (" + 100 *((float)node / highestNodeId) + "%)");
+            catch (Exception exp) {
+              notFound += 1;
             }
-            if (!sourceDb.nodeExists(node)) continue;
-            targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node,ignoreLabels));
         }
-        System.out.println("\n copying of " + node + " nodes took " + (System.currentTimeMillis() - time) + " ms.");
+        System.out.println("\n copying of " + node + " nodes took " + (System.currentTimeMillis() - time) + " ms. Not found " + notFound);
     }
 
     private static void flushCache(BatchInserter sourceDb, long node) {

From eaeba107a0d7cbaf7190ec994d2b5c00c6eab69c Mon Sep 17 00:00:00 2001
From: ahmet kizilay <ahmet.kizilay@gmail.com>
Date: Tue, 25 Aug 2015 20:55:07 +0300
Subject: [PATCH 12/33] updated current version info in readme

---
 readme.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readme.md b/readme.md
index 5b3b60c..11c82fd 100644
--- a/readme.md
+++ b/readme.md
@@ -7,7 +7,7 @@ Ignores broken nodes and relationships.
 Also useful to skip no longer wanted properties or relationships with a certain type. Good for store compaction as it
 rewrites the store file reclaiming space that is sitting empty.
 
-Change the Neo4j version in pom.xml before running. (Currently 2.2.1)
+Change the Neo4j version in pom.xml before running. (Currently 2.2.4)
 
 ### Store Copy
 

From 2b8e104d1a89bf806739550fee3f920a1955a58f Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Thu, 29 Oct 2015 23:23:14 +0100
Subject: [PATCH 13/33] Update to 2.2.6

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 97d516b..f646c79 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>2.2.4</version>
+    <version>2.2.6</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>

From 0d708071b419a7005265a9dd4e2fa08319e9d922 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Thu, 29 Oct 2015 23:24:28 +0100
Subject: [PATCH 14/33] Update to 2.3.0

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index f646c79..7bbe0ee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>2.2.6</version>
+    <version>2.3.0</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>

From ad00f8130445902929f2b8b87671fbd6bbe5e8ee Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Mon, 2 Nov 2015 22:17:14 +0100
Subject: [PATCH 15/33] better error handling

---
 src/main/java/org/neo4j/tool/StoreCopy.java | 34 +++++++++++++++------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index 892b66c..6fad205 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -112,6 +112,7 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
         long notFound = 0;
         while (relId <= highestRelId) {
             BatchRelationship rel = null;
+            String type = null;
             try {
                 if (relId % 10000 == 0) {
                     System.out.print(".");
@@ -121,13 +122,18 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
                     System.out.println(" " + relId + " / " + highestRelId + " (" + 100 *((float)relId / highestRelId) + "%)");
                 }
                 rel = sourceDb.getRelationshipById(relId++);
-                if (ignoreRelTypes.contains(rel.getType().name().toLowerCase())) continue;
+                type = rel.getType().name();
+                if (ignoreRelTypes.contains(type.toLowerCase())) continue;
                 createRelationship(targetDb, sourceDb, rel, ignoreProperties);
-            } catch (Exception nfe) {
-                notFound++;
+            } catch (Exception e) {
+                if (e instanceof org.neo4j.kernel.impl.store.InvalidRecordException && e.getMessage().endsWith("not in use")) {
+                   notFound++;
+                } else {
+                   addLog(rel, "copy Relationship: " + (relId - 1) + "-[:" + type + "]" + "->?", e.getMessage());
+                }
             }
         }
-        System.out.println("\n copying of "+relId+" relationships took "+(System.currentTimeMillis()-time)+" ms. Not found "+notFound);
+        System.out.println("\n copying of "+relId+" relationship records took "+(System.currentTimeMillis()-time)+" ms. Unused Records "+notFound);
     }
 
     private static long firstNode(BatchInserter sourceDb, long highestNodeId) {
@@ -143,9 +149,11 @@ private static void createRelationship(BatchInserter targetDb, BatchInserter sou
         long endNodeId = rel.getEndNode();
         final RelationshipType type = rel.getType();
         try {
-            targetDb.createRelationship(startNodeId, endNodeId, type, getProperties(sourceDb.getRelationshipProperties(rel.getId()), ignoreProperties));
-        } catch (Exception ire) {
-            addLog(rel, "create Relationship: " + startNodeId + "-[:" + type + "]" + "->" + endNodeId, ire.getMessage());
+            Map<String, Object> props = getProperties(sourceDb.getRelationshipProperties(rel.getId()), ignoreProperties);
+//            if (props.isEmpty()) props = Collections.<String,Object>singletonMap("old_id",rel.getId()); else props.put("old_id",rel.getId());
+            targetDb.createRelationship(startNodeId, endNodeId, type, props);
+        } catch (Exception e) {
+            addLog(rel, "create Relationship: " + startNodeId + "-[:" + type + "]" + "->" + endNodeId, e.getMessage());
         }
     }
 
@@ -167,11 +175,13 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
               if (!sourceDb.nodeExists(node)) continue;
               targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node,ignoreLabels));
             }
-            catch (Exception exp) {
-              notFound += 1;
+            catch (Exception e) {
+                if (e instanceof org.neo4j.kernel.impl.store.InvalidRecordException && e.getMessage().endsWith("not in use")) {
+                    notFound += 1;
+                } else addLog(node, e.getMessage());
             }
         }
-        System.out.println("\n copying of " + node + " nodes took " + (System.currentTimeMillis() - time) + " ms. Not found " + notFound);
+        System.out.println("\n copying of " + node + " node records took " + (System.currentTimeMillis() - time) + " ms. Unused Records " + notFound);
     }
 
     private static void flushCache(BatchInserter sourceDb, long node) {
@@ -208,6 +218,10 @@ private static void addLog(BatchRelationship rel, String property, String messag
         logs.append(String.format("%s.%s %s%n", rel, property, message));
     }
 
+    private static void addLog(long node, String message) {
+        logs.append(String.format("Node: %s %s%n", node, message));
+    }
+
     private static void addLog(PropertyContainer pc, String property, String message) {
         logs.append(String.format("%s.%s %s%n", pc, property, message));
     }

From 20533408f5f0af3574c61ab3e96f6ab2fe82a591 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Mon, 2 Nov 2015 22:17:14 +0100
Subject: [PATCH 16/33] better error handling

---
 src/main/java/org/neo4j/tool/StoreCopy.java | 34 +++++++++++++++------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index 892b66c..6fad205 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -112,6 +112,7 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
         long notFound = 0;
         while (relId <= highestRelId) {
             BatchRelationship rel = null;
+            String type = null;
             try {
                 if (relId % 10000 == 0) {
                     System.out.print(".");
@@ -121,13 +122,18 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
                     System.out.println(" " + relId + " / " + highestRelId + " (" + 100 *((float)relId / highestRelId) + "%)");
                 }
                 rel = sourceDb.getRelationshipById(relId++);
-                if (ignoreRelTypes.contains(rel.getType().name().toLowerCase())) continue;
+                type = rel.getType().name();
+                if (ignoreRelTypes.contains(type.toLowerCase())) continue;
                 createRelationship(targetDb, sourceDb, rel, ignoreProperties);
-            } catch (Exception nfe) {
-                notFound++;
+            } catch (Exception e) {
+                if (e instanceof org.neo4j.kernel.impl.store.InvalidRecordException && e.getMessage().endsWith("not in use")) {
+                   notFound++;
+                } else {
+                   addLog(rel, "copy Relationship: " + (relId - 1) + "-[:" + type + "]" + "->?", e.getMessage());
+                }
             }
         }
-        System.out.println("\n copying of "+relId+" relationships took "+(System.currentTimeMillis()-time)+" ms. Not found "+notFound);
+        System.out.println("\n copying of "+relId+" relationship records took "+(System.currentTimeMillis()-time)+" ms. Unused Records "+notFound);
     }
 
     private static long firstNode(BatchInserter sourceDb, long highestNodeId) {
@@ -143,9 +149,11 @@ private static void createRelationship(BatchInserter targetDb, BatchInserter sou
         long endNodeId = rel.getEndNode();
         final RelationshipType type = rel.getType();
         try {
-            targetDb.createRelationship(startNodeId, endNodeId, type, getProperties(sourceDb.getRelationshipProperties(rel.getId()), ignoreProperties));
-        } catch (Exception ire) {
-            addLog(rel, "create Relationship: " + startNodeId + "-[:" + type + "]" + "->" + endNodeId, ire.getMessage());
+            Map<String, Object> props = getProperties(sourceDb.getRelationshipProperties(rel.getId()), ignoreProperties);
+//            if (props.isEmpty()) props = Collections.<String,Object>singletonMap("old_id",rel.getId()); else props.put("old_id",rel.getId());
+            targetDb.createRelationship(startNodeId, endNodeId, type, props);
+        } catch (Exception e) {
+            addLog(rel, "create Relationship: " + startNodeId + "-[:" + type + "]" + "->" + endNodeId, e.getMessage());
         }
     }
 
@@ -167,11 +175,13 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
               if (!sourceDb.nodeExists(node)) continue;
               targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node,ignoreLabels));
             }
-            catch (Exception exp) {
-              notFound += 1;
+            catch (Exception e) {
+                if (e instanceof org.neo4j.kernel.impl.store.InvalidRecordException && e.getMessage().endsWith("not in use")) {
+                    notFound += 1;
+                } else addLog(node, e.getMessage());
             }
         }
-        System.out.println("\n copying of " + node + " nodes took " + (System.currentTimeMillis() - time) + " ms. Not found " + notFound);
+        System.out.println("\n copying of " + node + " node records took " + (System.currentTimeMillis() - time) + " ms. Unused Records " + notFound);
     }
 
     private static void flushCache(BatchInserter sourceDb, long node) {
@@ -208,6 +218,10 @@ private static void addLog(BatchRelationship rel, String property, String messag
         logs.append(String.format("%s.%s %s%n", rel, property, message));
     }
 
+    private static void addLog(long node, String message) {
+        logs.append(String.format("Node: %s %s%n", node, message));
+    }
+
     private static void addLog(PropertyContainer pc, String property, String message) {
         logs.append(String.format("%s.%s %s%n", pc, property, message));
     }

From b6954dac5851bffbe0761c8096e035eddd19d0da Mon Sep 17 00:00:00 2001
From: Johannes Mockenhaupt <git@jotomo.de>
Date: Tue, 13 Oct 2015 17:20:51 +0200
Subject: [PATCH 17/33] StoreComparer: add missing transaction wrapping.

---
 src/main/java/org/neo4j/tool/StoreComparer.java | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/neo4j/tool/StoreComparer.java b/src/main/java/org/neo4j/tool/StoreComparer.java
index de57389..81faed6 100644
--- a/src/main/java/org/neo4j/tool/StoreComparer.java
+++ b/src/main/java/org/neo4j/tool/StoreComparer.java
@@ -57,9 +57,12 @@ private static void compareStore(String sourceDir, String targetDir, Set<String>
         GraphDatabaseService targetDb = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(target.getAbsolutePath()).setConfig(config()).newGraphDatabase();
         GraphDatabaseService sourceDb = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(sourceDir).setConfig(config()).newGraphDatabase();
 
-        compareCounts(sourceDb, targetDb, ignoreRelTypes, ignoreProperties);
-        compareNodes(sourceDb, targetDb, ignoreProperties);
-        compareRelationships(sourceDb, targetDb, ignoreRelTypes, ignoreProperties);
+        try (Transaction srcDbTx = sourceDb.beginTx();
+             Transaction targetDbTx = targetDb.beginTx()) {
+            compareCounts(sourceDb, targetDb, ignoreRelTypes, ignoreProperties);
+            compareNodes(sourceDb, targetDb, ignoreProperties);
+            compareRelationships(sourceDb, targetDb, ignoreRelTypes, ignoreProperties);
+        }
 
         targetDb.shutdown();
         sourceDb.shutdown();

From 54f83594657212ae75290642f55cb5e092a4f674 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Tue, 24 Nov 2015 10:38:45 +0100
Subject: [PATCH 18/33] flushing without writing to the source-store

---
 copy-store.sh                               |   8 ++
 src/main/java/org/neo4j/tool/StoreCopy.java | 103 +++++++++++---------
 2 files changed, 64 insertions(+), 47 deletions(-)
 create mode 100755 copy-store.sh

diff --git a/copy-store.sh b/copy-store.sh
new file mode 100755
index 0000000..3c8fc14
--- /dev/null
+++ b/copy-store.sh
@@ -0,0 +1,8 @@
+SRC=$1
+DST=$2
+RELS=$3
+PROPS=$4
+LABELS=$5
+
+mvn compile exec:java -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=2G \
+      -Dexec.args="$SRC $DST $RELS $PROPS $LABELS"
diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index 6fad205..53f59c5 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -10,13 +10,14 @@
 import org.neo4j.kernel.IdGeneratorFactory;
 import org.neo4j.kernel.IdType;
 import org.neo4j.unsafe.batchinsert.BatchInserter;
+import org.neo4j.unsafe.batchinsert.BatchInserterImpl;
 import org.neo4j.unsafe.batchinsert.BatchInserters;
 import org.neo4j.unsafe.batchinsert.BatchRelationship;
 
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.*;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.*;
 
 import static java.util.Arrays.asList;
@@ -30,15 +31,7 @@ public class StoreCopy {
     @SuppressWarnings("unchecked")
     public static Map<String, String> config() {
         return (Map) MapUtil.map(
-                "neostore.nodestore.db.mapped_memory", "100M",
-                "neostore.relationshipstore.db.mapped_memory", "500M",
-                "neostore.relationshipgroupstore.db.mapped_memory", "10M",
-                "neostore.propertystore.db.mapped_memory", "300M",
-                "neostore.propertystore.db.strings.mapped_memory", "1G",
-                "neostore.propertystore.db.arrays.mapped_memory", "300M",
-                "neostore.propertystore.db.index.keys.mapped_memory", "100M",
-                "neostore.propertystore.db.index.mapped_memory", "100M",
-                "use_memory_mapped_buffers","true",
+                "dbms.pagecache.memory", System.getProperty("dbms.pagecache.memory","2G"),
                 "cache_type", "none"
         );
     }
@@ -62,6 +55,9 @@ private static Set<String> splitOptionIfExists(String[] args, final int index) {
         return new HashSet<String>(asList(args[index].toLowerCase().split(",")));
     }
 
+    interface Flusher {
+        void flush();
+    }
     private static void copyStore(String sourceDir, String targetDir, Set<String> ignoreRelTypes, Set<String> ignoreProperties, Set<String> ignoreLabels) throws Exception {
         final File target = new File(targetDir);
         final File source = new File(sourceDir);
@@ -74,18 +70,39 @@ private static void copyStore(String sourceDir, String targetDir, Set<String> ig
         Pair<Long, Long> highestIds = getHighestNodeId(source);
         BatchInserter targetDb = BatchInserters.inserter(target.getAbsolutePath(), config());
         BatchInserter sourceDb = BatchInserters.inserter(source.getAbsolutePath(), config());
-        logs = new PrintWriter(new FileWriter(new File(target, "store-copy.log")));
+        Flusher flusher = getFlusher(sourceDb);
 
-        long firstNode = firstNode(sourceDb, highestIds.first());
-        copyNodes(sourceDb, targetDb, ignoreProperties, ignoreLabels, highestIds.first());
-        copyRelationships(sourceDb, targetDb, ignoreRelTypes, ignoreProperties, highestIds.other(), firstNode);
+        logs = new PrintWriter(new FileWriter(new File(target, "store-copy.log")));
 
+        copyNodes(sourceDb, targetDb, ignoreProperties, ignoreLabels, highestIds.first(),flusher);
+        copyRelationships(sourceDb, targetDb, ignoreRelTypes, ignoreProperties, highestIds.other(), flusher);
         targetDb.shutdown();
         sourceDb.shutdown();
         logs.close();
         copyIndex(source, target);
     }
 
+    private static Flusher getFlusher(BatchInserter db) {
+        try {
+            Field field = BatchInserterImpl.class.getDeclaredField("flushStrategy");
+            field.setAccessible(true);
+            final Object flushStrategy = field.get(db);
+            final Method forceFlush = flushStrategy.getClass().getDeclaredMethod("forceFlush");
+            forceFlush.setAccessible(true);
+            return new Flusher() {
+                @Override public void flush() {
+                    try {
+                        forceFlush.invoke(flushStrategy);
+                    } catch (IllegalAccessException | InvocationTargetException e) {
+                        throw new RuntimeException("Error invoking flush strategy",e);
+                    }
+                }
+            };
+        } catch (NoSuchMethodException | IllegalAccessException | NoSuchFieldException e) {
+            throw new RuntimeException("Error accessing flush strategy", e);
+        }
+    }
+
     private static Pair<Long, Long> getHighestNodeId(File source) {
         GraphDatabaseAPI api = (GraphDatabaseAPI) new GraphDatabaseFactory().newEmbeddedDatabase(source.getAbsolutePath());
         IdGeneratorFactory idGenerators = api.getDependencyResolver().resolveDependency(IdGeneratorFactory.class);
@@ -106,7 +123,7 @@ private static void copyIndex(File source, File target) throws IOException {
         }
     }
 
-    private static void copyRelationships(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreRelTypes, Set<String> ignoreProperties, long highestRelId, long firstNode) {
+    private static void copyRelationships(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreRelTypes, Set<String> ignoreProperties, long highestRelId, Flusher flusher) {
         long time = System.currentTimeMillis();
         long relId = 0;
         long notFound = 0;
@@ -114,17 +131,19 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
             BatchRelationship rel = null;
             String type = null;
             try {
+                rel = sourceDb.getRelationshipById(relId);
+                type = rel.getType().name();
+                relId++;
+                if (!ignoreRelTypes.contains(type.toLowerCase())) {
+                    createRelationship(targetDb, sourceDb, rel, ignoreProperties);
+                }
                 if (relId % 10000 == 0) {
                     System.out.print(".");
                 }
                 if (relId % 500000 == 0) {
-                    flushCache(sourceDb, firstNode);
+                    flusher.flush();
                     System.out.println(" " + relId + " / " + highestRelId + " (" + 100 *((float)relId / highestRelId) + "%)");
                 }
-                rel = sourceDb.getRelationshipById(relId++);
-                type = rel.getType().name();
-                if (ignoreRelTypes.contains(type.toLowerCase())) continue;
-                createRelationship(targetDb, sourceDb, rel, ignoreProperties);
             } catch (Exception e) {
                 if (e instanceof org.neo4j.kernel.impl.store.InvalidRecordException && e.getMessage().endsWith("not in use")) {
                    notFound++;
@@ -157,23 +176,24 @@ private static void createRelationship(BatchInserter targetDb, BatchInserter sou
         }
     }
 
-    private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreProperties, Set<String> ignoreLabels, long highestNodeId) {
+    private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreProperties, Set<String> ignoreLabels, long highestNodeId, Flusher flusher) {
         long time = System.currentTimeMillis();
-        int node = -1;
+        int node = 0;
         long notFound = 0;
-        while (++node <= highestNodeId) {
+        while (node <= highestNodeId) {
             try {
-              if (node % 10000 == 0) {
-                  System.out.print(".");
-              }
-              if (node % 500000 == 0) {
-                  flushCache(sourceDb, node);
-                  logs.flush();
-                  System.out.println(" " + node + " / " + highestNodeId + " (" + 100 *((float)node / highestNodeId) + "%)");
+              if (sourceDb.nodeExists(node)) {
+                 targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node, ignoreLabels));
               }
-
-              if (!sourceDb.nodeExists(node)) continue;
-              targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node,ignoreLabels));
+              node++;
+                if (node % 10000 == 0) {
+                    System.out.print(".");
+                }
+                if (node % 500000 == 0) {
+                    flusher.flush();
+                    logs.flush();
+                    System.out.println(" " + node + " / " + highestNodeId + " (" + 100 *((float)node / highestNodeId) + "%)");
+                }
             }
             catch (Exception e) {
                 if (e instanceof org.neo4j.kernel.impl.store.InvalidRecordException && e.getMessage().endsWith("not in use")) {
@@ -184,17 +204,6 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
         System.out.println("\n copying of " + node + " node records took " + (System.currentTimeMillis() - time) + " ms. Unused Records " + notFound);
     }
 
-    private static void flushCache(BatchInserter sourceDb, long node) {
-        Map<String, Object> nodeProperties = sourceDb.getNodeProperties(node);
-        Iterator<Map.Entry<String, Object>> iterator = nodeProperties.entrySet().iterator();
-        if (iterator.hasNext()) {
-            Map.Entry<String, Object> firstProp = iterator.next();
-            sourceDb.nodeHasProperty(node,firstProp.getKey());
-            sourceDb.setNodeProperty(node, firstProp.getKey(), firstProp.getValue()); // force flush
-            System.out.print("F");
-        }
-    }
-
     private static Label[] labelsArray(BatchInserter db, long node, Set<String> ignoreLabels) {
         Collection<Label> labels = IteratorUtil.asCollection(db.getNodeLabels(node));
         if (labels.isEmpty()) return NO_LABELS;

From a18cfea9d12a9f5942f2c11e1dcc6afee2e4862f Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Tue, 24 Nov 2015 22:11:29 +0100
Subject: [PATCH 19/33] Updated docs, shell script and version to 2.3.1

---
 .gitignore       |  2 ++
 copy-store.sh    | 16 ++++++++++++----
 neo4j.properties |  4 ++++
 pom.xml          |  9 +++++----
 readme.md        | 41 ++++++++++++++++++++++++++++-------------
 5 files changed, 51 insertions(+), 21 deletions(-)
 create mode 100644 neo4j.properties

diff --git a/.gitignore b/.gitignore
index 8068abf..0dea21b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,4 +4,6 @@ target
 *.iml
 *.ipr
 .idea
+*.db*
+.shell_history
 
diff --git a/copy-store.sh b/copy-store.sh
index 3c8fc14..d9374af 100755
--- a/copy-store.sh
+++ b/copy-store.sh
@@ -1,8 +1,16 @@
 SRC=$1
 DST=$2
-RELS=$3
-PROPS=$4
-LABELS=$5
+SKIP_RELS=$3
+SKIP_PROPS=$4
+SKIP_LABELS=$5
+
+echo "Usage: copy-store.sh source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]"
+echo "Database config is read from neo4j.properties file in current directory if it exists"
+
+# heap config
+export MAVEN_OPTS="-Xmx4G -Xms4G -Xmn1G -XX:+UseG1GC"
 
 mvn compile exec:java -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=2G \
-      -Dexec.args="$SRC $DST $RELS $PROPS $LABELS"
+      -Dexec.args="$SRC $DST $SKIP_RELS $SKIP_PROPS $SKIP_LABELS"
+
+#-Dneo4j.version=2.3.0
diff --git a/neo4j.properties b/neo4j.properties
new file mode 100644
index 0000000..47cdef4
--- /dev/null
+++ b/neo4j.properties
@@ -0,0 +1,4 @@
+dbms.pagecache.memory=2G
+cache_type=none
+#allow_store_upgrade=true
+
diff --git a/pom.xml b/pom.xml
index 7bbe0ee..c9f61a9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,13 +4,14 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>2.3.0</version>
+    <version>2.3.1</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <neo4j.version>${project.version}</neo4j.version>
     </properties>
 
     <dependencies>
@@ -23,17 +24,17 @@
         <dependency>
             <groupId>org.neo4j</groupId>
             <artifactId>neo4j-io</artifactId>
-            <version>${project.version}</version>
+            <version>${neo4j.version}</version>
         </dependency>
         <dependency>
             <groupId>org.neo4j</groupId>
             <artifactId>neo4j-kernel</artifactId>
-            <version>${project.version}</version>
+            <version>${neo4j.version}</version>
         </dependency>
         <dependency>
             <groupId>org.neo4j</groupId>
             <artifactId>neo4j-lucene-index</artifactId>
-            <version>${project.version}</version>
+            <version>${neo4j.version}</version>
         </dependency>
     </dependencies>
 
diff --git a/readme.md b/readme.md
index 11c82fd..a02c887 100644
--- a/readme.md
+++ b/readme.md
@@ -1,22 +1,37 @@
-## Tools to copy and compare Neo4j Stores
+## Tools to copy Neo4j Stores
 
-Uses the GraphDatabaseService to read a store and the batch-inserter API to write the target store keeping the node-ids.
-Copies the index-files as is.
-Ignores broken nodes and relationships.
+Uses the BatchInserterImpl to read a store and write the target store keeping the node-ids.
+Copies the manual (legacy) index-files as is, please note it performs no index upgrade!
 
-Also useful to skip no longer wanted properties or relationships with a certain type. Good for store compaction as it
-rewrites the store file reclaiming space that is sitting empty.
+Ignores broken nodes and relationships and records them in `target/store-copy.log`
 
-Change the Neo4j version in pom.xml before running. (Currently 2.2.4)
+Also useful to skip no longer wanted properties or relationships with a certain type.
+Good for store compaction and reorganization of relationships and properties as
+it rewrites the store file reclaiming space that is sitting empty.
 
 ### Store Copy
 
-Usage:
+    copy-store.sh source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]"
 
-    mvn compile exec:java -Dexec.mainClass="org.neo4j.tool.StoreCopy" \
-      -Dexec.args="source-dir target-dir [rel,types,to,ignore] [properties,to,ignore] [labels,to,ignore]"
+Database config is read from `neo4j.properties` file in current directory if it exists.
+The provided one contains:
+
+    dbms.pagecache.memory=2G
+    cache_type=none
+
+Heap config is in the shell-script, default is:
+
+    export MAVEN_OPTS="-Xmx4G -Xms4G -Xmn1G -XX:+UseG1GC"
+
+**Please adapt the settings as needed for your store.**
 
-# Store Compare
+Change the Neo4j version in pom.xml before running as needed. (Currently 2.3.1)
+Optionally changeable from the outside with `-Dneo4j.version=2.3.0` on the `mvn` invocation.
+
+### Internally
+
+Note: It calls under the hood:
+
+    mvn compile exec:java -Dexec.mainClass="org.neo4j.tool.StoreCopy" \
+      -Dexec.args="source-dir target-dir [REL,TYPES,TO,IGNORE] [properties,to,ignore] [Labels,To,Ignore]"
 
-    mvn compile exec:java -Dexec.mainClass="org.neo4j.tool.StoreComparer" \
-      -Dexec.args="source-dir target-dir [rel,types,to,ignore] [properties,to,ignore]"

From 4ce6967e6ac6b487bd2f4f3df3782f3705cab0a3 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Tue, 24 Nov 2015 22:15:51 +0100
Subject: [PATCH 20/33] minimal readme fixes

---
 readme.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/readme.md b/readme.md
index a02c887..ccf3130 100644
--- a/readme.md
+++ b/readme.md
@@ -11,7 +11,7 @@ it rewrites the store file reclaiming space that is sitting empty.
 
 ### Store Copy
 
-    copy-store.sh source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]"
+    copy-store.sh source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]
 
 Database config is read from `neo4j.properties` file in current directory if it exists.
 The provided one contains:
@@ -26,6 +26,7 @@ Heap config is in the shell-script, default is:
 **Please adapt the settings as needed for your store.**
 
 Change the Neo4j version in pom.xml before running as needed. (Currently 2.3.1)
+
 Optionally changeable from the outside with `-Dneo4j.version=2.3.0` on the `mvn` invocation.
 
 ### Internally

From 0a084b4d95f7a42581d9215e083a0d0efd6811bb Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Mon, 7 Dec 2015 10:04:34 +0100
Subject: [PATCH 21/33] Fixed progress in case of errors, fixed memory usage in
 2.3

---
 src/main/java/org/neo4j/tool/StoreCopy.java | 67 ++++++++++++---------
 1 file changed, 39 insertions(+), 28 deletions(-)

diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index 53f59c5..b0551a9 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -70,7 +70,7 @@ private static void copyStore(String sourceDir, String targetDir, Set<String> ig
         Pair<Long, Long> highestIds = getHighestNodeId(source);
         BatchInserter targetDb = BatchInserters.inserter(target.getAbsolutePath(), config());
         BatchInserter sourceDb = BatchInserters.inserter(source.getAbsolutePath(), config());
-        Flusher flusher = getFlusher(sourceDb);
+        Flusher flusher = getFlusher(sourceDb, firstNode(sourceDb,highestIds.first()));
 
         logs = new PrintWriter(new FileWriter(new File(target, "store-copy.log")));
 
@@ -82,7 +82,7 @@ private static void copyStore(String sourceDir, String targetDir, Set<String> ig
         copyIndex(source, target);
     }
 
-    private static Flusher getFlusher(BatchInserter db) {
+    private static Flusher getFlusher(final BatchInserter db, final long nodeWithProps) {
         try {
             Field field = BatchInserterImpl.class.getDeclaredField("flushStrategy");
             field.setAccessible(true);
@@ -92,6 +92,7 @@ private static Flusher getFlusher(BatchInserter db) {
             return new Flusher() {
                 @Override public void flush() {
                     try {
+                        flushCache(db,nodeWithProps);
                         forceFlush.invoke(flushStrategy);
                     } catch (IllegalAccessException | InvocationTargetException e) {
                         throw new RuntimeException("Error invoking flush strategy",e);
@@ -131,26 +132,26 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
             BatchRelationship rel = null;
             String type = null;
             try {
-                rel = sourceDb.getRelationshipById(relId);
+                rel = sourceDb.getRelationshipById(relId++);
                 type = rel.getType().name();
-                relId++;
                 if (!ignoreRelTypes.contains(type.toLowerCase())) {
                     createRelationship(targetDb, sourceDb, rel, ignoreProperties);
                 }
-                if (relId % 10000 == 0) {
-                    System.out.print(".");
-                }
-                if (relId % 500000 == 0) {
-                    flusher.flush();
-                    System.out.println(" " + relId + " / " + highestRelId + " (" + 100 *((float)relId / highestRelId) + "%)");
-                }
             } catch (Exception e) {
                 if (e instanceof org.neo4j.kernel.impl.store.InvalidRecordException && e.getMessage().endsWith("not in use")) {
-                   notFound++;
+                    notFound++;
                 } else {
-                   addLog(rel, "copy Relationship: " + (relId - 1) + "-[:" + type + "]" + "->?", e.getMessage());
+                    addLog(rel, "copy Relationship: " + (relId - 1) + "-[:" + type + "]" + "->?", e.getMessage());
                 }
             }
+            if (relId % 10000 == 0) {
+                System.out.print(".");
+                logs.flush();
+            }
+            if (relId % 500000 == 0) {
+                flusher.flush();
+                System.out.printf(" %d / %d (%.0f%%) unused %d%n", relId, highestRelId, 100 * ((float) relId / highestRelId), notFound);
+            }
         }
         System.out.println("\n copying of "+relId+" relationship records took "+(System.currentTimeMillis()-time)+" ms. Unused Records "+notFound);
     }
@@ -158,11 +159,22 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
     private static long firstNode(BatchInserter sourceDb, long highestNodeId) {
         int node = -1;
         while (++node <= highestNodeId) {
-            if (sourceDb.nodeExists(node)) return node;
+            if (sourceDb.nodeExists(node) && !sourceDb.getNodeProperties(node).isEmpty()) return node;
         }
         return -1;
     }
 
+    private static void flushCache(BatchInserter sourceDb, long node) {
+        Map<String, Object> nodeProperties = sourceDb.getNodeProperties(node);
+        Iterator<Map.Entry<String, Object>> iterator = nodeProperties.entrySet().iterator();
+        if (iterator.hasNext()) {
+            Map.Entry<String, Object> firstProp = iterator.next();
+            sourceDb.nodeHasProperty(node,firstProp.getKey());
+            sourceDb.setNodeProperty(node, firstProp.getKey(), firstProp.getValue()); // force flush
+            System.out.print(" flush");
+        }
+    }
+
     private static void createRelationship(BatchInserter targetDb, BatchInserter sourceDb, BatchRelationship rel, Set<String> ignoreProperties) {
         long startNodeId = rel.getStartNode();
         long endNodeId = rel.getEndNode();
@@ -182,24 +194,23 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
         long notFound = 0;
         while (node <= highestNodeId) {
             try {
-              if (sourceDb.nodeExists(node)) {
-                 targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node, ignoreLabels));
-              }
-              node++;
-                if (node % 10000 == 0) {
-                    System.out.print(".");
-                }
-                if (node % 500000 == 0) {
-                    flusher.flush();
-                    logs.flush();
-                    System.out.println(" " + node + " / " + highestNodeId + " (" + 100 *((float)node / highestNodeId) + "%)");
+                if (sourceDb.nodeExists(node)) {
+                    targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node, ignoreLabels));
                 }
-            }
-            catch (Exception e) {
+            } catch (Exception e) {
                 if (e instanceof org.neo4j.kernel.impl.store.InvalidRecordException && e.getMessage().endsWith("not in use")) {
-                    notFound += 1;
+                    notFound++;
                 } else addLog(node, e.getMessage());
             }
+            node++;
+            if (node % 10000 == 0) {
+                System.out.print(".");
+            }
+            if (node % 500000 == 0) {
+                flusher.flush();
+                logs.flush();
+                System.out.printf(" %d / %d (%.0f%%)%n unused %d", node, highestNodeId, 100 * ((float) node / highestNodeId), notFound);
+            }
         }
         System.out.println("\n copying of " + node + " node records took " + (System.currentTimeMillis() - time) + " ms. Unused Records " + notFound);
     }

From 163f93cb695e2c715b8c9aebf0dbcdf6b358b900 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Mon, 7 Dec 2015 21:57:48 +0100
Subject: [PATCH 22/33] changed flushing to clear caches directly

---
 copy-store.sh                               |  7 +--
 src/main/java/org/neo4j/tool/StoreCopy.java | 50 ++++++++++++---------
 2 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/copy-store.sh b/copy-store.sh
index d9374af..9438bb6 100755
--- a/copy-store.sh
+++ b/copy-store.sh
@@ -3,14 +3,15 @@ DST=$2
 SKIP_RELS=$3
 SKIP_PROPS=$4
 SKIP_LABELS=$5
-
+HEAP=4G
+CACHE=2G
 echo "Usage: copy-store.sh source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]"
 echo "Database config is read from neo4j.properties file in current directory if it exists"
 
 # heap config
-export MAVEN_OPTS="-Xmx4G -Xms4G -Xmn1G -XX:+UseG1GC"
+export MAVEN_OPTS="-Xmx$HEAP -Xms$HEAP -Xmn1G -XX:+UseG1GC"
 
-mvn compile exec:java -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=2G \
+mvn compile exec:java -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=$CACHE \
       -Dexec.args="$SRC $DST $SKIP_RELS $SKIP_PROPS $SKIP_LABELS"
 
 #-Dneo4j.version=2.3.0
diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index b0551a9..c29dfe2 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -9,15 +9,10 @@
 import org.neo4j.kernel.GraphDatabaseAPI;
 import org.neo4j.kernel.IdGeneratorFactory;
 import org.neo4j.kernel.IdType;
-import org.neo4j.unsafe.batchinsert.BatchInserter;
-import org.neo4j.unsafe.batchinsert.BatchInserterImpl;
-import org.neo4j.unsafe.batchinsert.BatchInserters;
-import org.neo4j.unsafe.batchinsert.BatchRelationship;
+import org.neo4j.unsafe.batchinsert.*;
 
 import java.io.*;
 import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
 import java.util.*;
 
 import static java.util.Arrays.asList;
@@ -70,7 +65,7 @@ private static void copyStore(String sourceDir, String targetDir, Set<String> ig
         Pair<Long, Long> highestIds = getHighestNodeId(source);
         BatchInserter targetDb = BatchInserters.inserter(target.getAbsolutePath(), config());
         BatchInserter sourceDb = BatchInserters.inserter(source.getAbsolutePath(), config());
-        Flusher flusher = getFlusher(sourceDb, firstNode(sourceDb,highestIds.first()));
+        Flusher flusher = getFlusher(sourceDb);
 
         logs = new PrintWriter(new FileWriter(new File(target, "store-copy.log")));
 
@@ -82,25 +77,26 @@ private static void copyStore(String sourceDir, String targetDir, Set<String> ig
         copyIndex(source, target);
     }
 
-    private static Flusher getFlusher(final BatchInserter db, final long nodeWithProps) {
+    private static Flusher getFlusher(final BatchInserter db) {
         try {
-            Field field = BatchInserterImpl.class.getDeclaredField("flushStrategy");
+            Field field = BatchInserterImpl.class.getDeclaredField("recordAccess");
             field.setAccessible(true);
-            final Object flushStrategy = field.get(db);
-            final Method forceFlush = flushStrategy.getClass().getDeclaredMethod("forceFlush");
-            forceFlush.setAccessible(true);
+            final DirectRecordAccessSet recordAccessSet = (DirectRecordAccessSet) field.get(db);
+            final Field cacheField = DirectRecordAccess.class.getDeclaredField("batch");
+            cacheField.setAccessible(true);
             return new Flusher() {
                 @Override public void flush() {
                     try {
-                        flushCache(db,nodeWithProps);
-                        forceFlush.invoke(flushStrategy);
-                    } catch (IllegalAccessException | InvocationTargetException e) {
-                        throw new RuntimeException("Error invoking flush strategy",e);
+                        ((Map) cacheField.get(recordAccessSet.getNodeRecords())).clear();
+                        ((Map) cacheField.get(recordAccessSet.getRelRecords())).clear();
+                        ((Map) cacheField.get(recordAccessSet.getPropertyRecords())).clear();
+                    } catch (IllegalAccessException e) {
+                        throw new RuntimeException("Error clearing cache "+cacheField,e);
                     }
                 }
             };
-        } catch (NoSuchMethodException | IllegalAccessException | NoSuchFieldException e) {
-            throw new RuntimeException("Error accessing flush strategy", e);
+        } catch (IllegalAccessException | NoSuchFieldException e) {
+            throw new RuntimeException("Error accessing cache field ", e);
         }
     }
 
@@ -150,10 +146,16 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
             }
             if (relId % 500000 == 0) {
                 flusher.flush();
-                System.out.printf(" %d / %d (%.0f%%) unused %d%n", relId, highestRelId, 100 * ((float) relId / highestRelId), notFound);
+                System.out.printf(" %d / %d (%d%%) unused %d%n", relId, highestRelId, percent(relId,highestRelId), notFound);
             }
         }
-        System.out.println("\n copying of "+relId+" relationship records took "+(System.currentTimeMillis()-time)+" ms. Unused Records "+notFound);
+        time = (System.currentTimeMillis() - time) / 1000;
+        System.out.printf("%n copying of %d relationship records took %d seconds (%d rec/s). Unused Records %d (%d%%)%n",
+                relId, time, relId/time, notFound, percent(notFound,relId));
+    }
+
+    private static int percent(Number part, Number total) {
+        return (int) (100 * part.floatValue() / total.floatValue());
     }
 
     private static long firstNode(BatchInserter sourceDb, long highestNodeId) {
@@ -196,6 +198,8 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
             try {
                 if (sourceDb.nodeExists(node)) {
                     targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node, ignoreLabels));
+                } else {
+                    notFound++;
                 }
             } catch (Exception e) {
                 if (e instanceof org.neo4j.kernel.impl.store.InvalidRecordException && e.getMessage().endsWith("not in use")) {
@@ -209,10 +213,12 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
             if (node % 500000 == 0) {
                 flusher.flush();
                 logs.flush();
-                System.out.printf(" %d / %d (%.0f%%)%n unused %d", node, highestNodeId, 100 * ((float) node / highestNodeId), notFound);
+                System.out.printf(" %d / %d (%d%%)%n unused %d", node, highestNodeId, percent(node,highestNodeId), notFound);
             }
         }
-        System.out.println("\n copying of " + node + " node records took " + (System.currentTimeMillis() - time) + " ms. Unused Records " + notFound);
+        time = (System.currentTimeMillis() - time)/1000;
+        System.out.printf("%n copying of %d node records took %d seconds (%d rec/s). Unused Records %d (%d%%)%n",
+                node, time, node/time, notFound, percent(notFound,node));
     }
 
     private static Label[] labelsArray(BatchInserter db, long node, Set<String> ignoreLabels) {

From 3730ae49f160c5688613f96af1acee084aeddcbc Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Fri, 8 Jan 2016 00:37:29 +0100
Subject: [PATCH 23/33] fixed zero time

---
 src/main/java/org/neo4j/tool/StoreCopy.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index c29dfe2..ab2fd70 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -149,7 +149,7 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
                 System.out.printf(" %d / %d (%d%%) unused %d%n", relId, highestRelId, percent(relId,highestRelId), notFound);
             }
         }
-        time = (System.currentTimeMillis() - time) / 1000;
+        time = Math.max(1,(System.currentTimeMillis() - time)/1000);
         System.out.printf("%n copying of %d relationship records took %d seconds (%d rec/s). Unused Records %d (%d%%)%n",
                 relId, time, relId/time, notFound, percent(notFound,relId));
     }
@@ -216,7 +216,7 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
                 System.out.printf(" %d / %d (%d%%)%n unused %d", node, highestNodeId, percent(node,highestNodeId), notFound);
             }
         }
-        time = (System.currentTimeMillis() - time)/1000;
+        time = Math.max(1,(System.currentTimeMillis() - time)/1000);
         System.out.printf("%n copying of %d node records took %d seconds (%d rec/s). Unused Records %d (%d%%)%n",
                 node, time, node/time, notFound, percent(notFound,node));
     }

From 2d3632836ded58ab87b0bbc213a582afab288d10 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Sat, 2 Apr 2016 02:00:44 +0200
Subject: [PATCH 24/33] Upgrade to Neo4j 3.0.0-M05

---
 copy-store.sh                                 |  8 +-
 neo4j.properties                              |  2 +-
 pom.xml                                       | 14 +++-
 readme.md                                     | 11 +--
 .../java/org/neo4j/tool/DomainAnalyzer.java   |  1 -
 .../java/org/neo4j/tool/GraphGenerator.java   |  6 +-
 .../java/org/neo4j/tool/PropertyAnalyzer.java |  1 -
 .../tool/SingleRelationshipDeletion.java      | 82 -------------------
 src/main/java/org/neo4j/tool/StoreCopy.java   |  8 +-
 9 files changed, 32 insertions(+), 101 deletions(-)
 delete mode 100644 src/main/java/org/neo4j/tool/SingleRelationshipDeletion.java

diff --git a/copy-store.sh b/copy-store.sh
index 9438bb6..a3eabfd 100755
--- a/copy-store.sh
+++ b/copy-store.sh
@@ -1,3 +1,5 @@
+EDITION=${1-community}
+shift
 SRC=$1
 DST=$2
 SKIP_RELS=$3
@@ -5,13 +7,13 @@ SKIP_PROPS=$4
 SKIP_LABELS=$5
 HEAP=4G
 CACHE=2G
-echo "Usage: copy-store.sh source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]"
+echo "Usage: copy-store.sh [community|enterprise] source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]"
 echo "Database config is read from neo4j.properties file in current directory if it exists"
-
+echo "Using: Heap $HEAP Pagecache $CACHE Edition $EDITION from $SRC to $DST skipping labels: $SKIP_LABELS rels: $SKIP_RELS props $SKIP_PROPS"
 # heap config
 export MAVEN_OPTS="-Xmx$HEAP -Xms$HEAP -Xmn1G -XX:+UseG1GC"
 
-mvn compile exec:java -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=$CACHE \
+mvn clean compile exec:java -P${EDITION} -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=$CACHE \
       -Dexec.args="$SRC $DST $SKIP_RELS $SKIP_PROPS $SKIP_LABELS"
 
 #-Dneo4j.version=2.3.0
diff --git a/neo4j.properties b/neo4j.properties
index 47cdef4..fdd3598 100644
--- a/neo4j.properties
+++ b/neo4j.properties
@@ -1,4 +1,4 @@
 dbms.pagecache.memory=2G
 cache_type=none
-#allow_store_upgrade=true
+allow_store_upgrade=true
 
diff --git a/pom.xml b/pom.xml
index c9f61a9..0522489 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>2.3.1</version>
+    <version>3.0.0-M05</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>
@@ -14,6 +14,18 @@
         <neo4j.version>${project.version}</neo4j.version>
     </properties>
 
+    <profiles>
+        <profile>
+            <id>enterprise</id>
+            <dependencies>
+                <dependency>
+                    <groupId>org.neo4j</groupId>
+                    <artifactId>neo4j-enterprise</artifactId>
+                    <version>${neo4j.version}</version>
+                </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
     <dependencies>
         <dependency>
             <groupId>junit</groupId>
diff --git a/readme.md b/readme.md
index ccf3130..43a823c 100644
--- a/readme.md
+++ b/readme.md
@@ -9,15 +9,16 @@ Also useful to skip no longer wanted properties or relationships with a certain
 Good for store compaction and reorganization of relationships and properties as
 it rewrites the store file reclaiming space that is sitting empty.
 
+NOTE: With Neo4j 3.0 there are two different store formats, so you have to provide "enterprise" or "community" as first argument of the call!
+
 ### Store Copy
 
-    copy-store.sh source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]
+    copy-store.sh [enterprise|community] source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]
 
 Database config is read from `neo4j.properties` file in current directory if it exists.
 The provided one contains:
 
     dbms.pagecache.memory=2G
-    cache_type=none
 
 Heap config is in the shell-script, default is:
 
@@ -25,14 +26,14 @@ Heap config is in the shell-script, default is:
 
 **Please adapt the settings as needed for your store.**
 
-Change the Neo4j version in pom.xml before running as needed. (Currently 2.3.1)
+Change the Neo4j version in pom.xml before running as needed. (Currently 3.0.0-M05)
 
-Optionally changeable from the outside with `-Dneo4j.version=2.3.0` on the `mvn` invocation.
+Optionally changeable from the outside with `-Dneo4j.version=3.0.0` on the `mvn` invocation.
 
 ### Internally
 
 Note: It calls under the hood:
 
-    mvn compile exec:java -Dexec.mainClass="org.neo4j.tool.StoreCopy" \
+    mvn compile exec:java -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Penterprise \
       -Dexec.args="source-dir target-dir [REL,TYPES,TO,IGNORE] [properties,to,ignore] [Labels,To,Ignore]"
 
diff --git a/src/main/java/org/neo4j/tool/DomainAnalyzer.java b/src/main/java/org/neo4j/tool/DomainAnalyzer.java
index 1ce7f26..28a3b9c 100644
--- a/src/main/java/org/neo4j/tool/DomainAnalyzer.java
+++ b/src/main/java/org/neo4j/tool/DomainAnalyzer.java
@@ -4,7 +4,6 @@
 import org.neo4j.graphdb.Node;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
 import org.neo4j.helpers.collection.IteratorUtil;
-import org.neo4j.kernel.EmbeddedGraphDatabase;
 
 import java.util.*;
 
diff --git a/src/main/java/org/neo4j/tool/GraphGenerator.java b/src/main/java/org/neo4j/tool/GraphGenerator.java
index e54fe11..5402f9e 100644
--- a/src/main/java/org/neo4j/tool/GraphGenerator.java
+++ b/src/main/java/org/neo4j/tool/GraphGenerator.java
@@ -5,7 +5,6 @@
 import org.neo4j.graphdb.Relationship;
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
-import org.neo4j.kernel.EmbeddedGraphDatabase;
 
 import java.util.Arrays;
 
@@ -39,14 +38,15 @@ public static void createDatabase(GraphDatabaseService graphdb) {
                     System.out.print(".");
                     if ((i % 10000) == 0) {
                         tx.success();
-                        tx.finish();
+                        tx.close();
                         System.out.println(" " + i);
                         tx = graphdb.beginTx();
                     }
                 }
             }
         } finally {
-            tx.finish();
+            tx.success();
+            tx.close();
         }
         System.out.println();
         long delta = (System.currentTimeMillis() - cpuTime);
diff --git a/src/main/java/org/neo4j/tool/PropertyAnalyzer.java b/src/main/java/org/neo4j/tool/PropertyAnalyzer.java
index abef9ff..fbbad4b 100644
--- a/src/main/java/org/neo4j/tool/PropertyAnalyzer.java
+++ b/src/main/java/org/neo4j/tool/PropertyAnalyzer.java
@@ -3,7 +3,6 @@
 import org.neo4j.graphdb.*;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
 import org.neo4j.helpers.collection.MapUtil;
-import org.neo4j.kernel.EmbeddedGraphDatabase;
 
 import java.lang.reflect.Array;
 import java.util.*;
diff --git a/src/main/java/org/neo4j/tool/SingleRelationshipDeletion.java b/src/main/java/org/neo4j/tool/SingleRelationshipDeletion.java
deleted file mode 100644
index b596b26..0000000
--- a/src/main/java/org/neo4j/tool/SingleRelationshipDeletion.java
+++ /dev/null
@@ -1,82 +0,0 @@
-package org.neo4j.tool;
-
-/**
- * @author mh
- * @since 12.08.11
- */
-
-import org.neo4j.graphdb.GraphDatabaseService;
-import org.neo4j.graphdb.NotFoundException;
-import org.neo4j.graphdb.Relationship;
-import org.neo4j.graphdb.Transaction;
-import org.neo4j.graphdb.factory.GraphDatabaseFactory;
-import org.neo4j.kernel.EmbeddedGraphDatabase;
-
-import java.io.File;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-
-/**
- * @author mh
- * @since 11.08.11
- */
-public class SingleRelationshipDeletion {
-
-    public static final int COUNT = 1000;
-    private static final int ROUNDS = 50;
-
-    public static void main(String[] args) {
-        final File dir = new File("target/data");
-        boolean mustCreate = !dir.exists();
-        final GraphDatabaseService graphdb = new GraphDatabaseFactory().newEmbeddedDatabase(dir.getAbsolutePath());
-        if (mustCreate) GraphGenerator.createDatabase(graphdb);
-
-        for (int round = 0; round < ROUNDS; round++) {
-            long[] relIds = createRelIds(COUNT);
-            int success = 0, fail = 0;
-            long cpuTime = System.currentTimeMillis();
-            final Transaction tx = graphdb.beginTx();
-            try {
-                for (int i = 0; i < COUNT; i++) {
-                    final Relationship rel;
-                    try {
-                        rel = graphdb.getRelationshipById(relIds[i]);
-                    } catch (NotFoundException nfe) {
-                        fail++;
-                        continue;
-                    }
-                    try {
-                        rel.delete();
-                        tx.success();
-                        success++;
-                    } catch (NotFoundException nfe) {
-                        fail++;
-//                    tx.failure();
-                    }
-                }
-            } finally {
-                tx.finish();
-            }
-
-            long delta = (System.currentTimeMillis() - cpuTime);
-            System.out.printf("round %d delete %d relationships time = %d ms, succ %d failed %d%n", round, COUNT, delta, success, fail);
-        }
-        graphdb.shutdown();
-    }
-
-    private static long[] createRelIds(final int count) {
-        Random random = new Random(System.currentTimeMillis());
-        Set<Integer> values = new HashSet<Integer>();
-        do {
-            values.add(random.nextInt(GraphGenerator.MILLION));
-        } while (values.size() < count);
-        final long[] result = new long[count];
-        int i = 0;
-        for (Integer value : values) {
-            result[i++] = value.longValue();
-        }
-        return result;
-    }
-
-}
diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index ab2fd70..d2d1f43 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -2,13 +2,13 @@
 
 import org.neo4j.graphdb.*;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
-import org.neo4j.helpers.Pair;
 import org.neo4j.helpers.collection.IteratorUtil;
 import org.neo4j.helpers.collection.MapUtil;
+import org.neo4j.helpers.collection.Pair;
 import org.neo4j.io.fs.FileUtils;
-import org.neo4j.kernel.GraphDatabaseAPI;
-import org.neo4j.kernel.IdGeneratorFactory;
-import org.neo4j.kernel.IdType;
+import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
+import org.neo4j.kernel.impl.store.id.IdType;
+import org.neo4j.kernel.internal.GraphDatabaseAPI;
 import org.neo4j.unsafe.batchinsert.*;
 
 import java.io.*;

From dd2c9054624b960a047dc57aefeea8076bee4a5c Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Wed, 15 Jun 2016 23:23:49 +0200
Subject: [PATCH 25/33] Upgrade to 3.0.3

---
 pom.xml                                            |  6 +++---
 src/main/java/org/neo4j/tool/DomainAnalyzer.java   |  7 ++++---
 src/main/java/org/neo4j/tool/GraphGenerator.java   |  3 ++-
 src/main/java/org/neo4j/tool/PropertyAnalyzer.java |  3 ++-
 src/main/java/org/neo4j/tool/StoreComparer.java    | 13 ++++++-------
 src/main/java/org/neo4j/tool/StoreCopy.java        | 11 ++++++-----
 src/main/java/org/neo4j/tool/StoreUpdate.java      |  3 ++-
 7 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/pom.xml b/pom.xml
index 0522489..781de76 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>3.0.0-M05</version>
+    <version>3.0.3</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>
@@ -57,8 +57,8 @@
                 <artifactId>maven-compiler-plugin</artifactId>
                 <version>2.3.2</version>
                 <configuration>
-                    <source>1.7</source>
-                    <target>1.7</target>
+                    <source>1.8</source>
+                    <target>1.8</target>
                     <compilerArgument>-Xlint:all</compilerArgument>
                     <showWarnings>true</showWarnings>
                     <showDeprecation>false</showDeprecation>
diff --git a/src/main/java/org/neo4j/tool/DomainAnalyzer.java b/src/main/java/org/neo4j/tool/DomainAnalyzer.java
index 28a3b9c..489810e 100644
--- a/src/main/java/org/neo4j/tool/DomainAnalyzer.java
+++ b/src/main/java/org/neo4j/tool/DomainAnalyzer.java
@@ -3,8 +3,9 @@
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.Node;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
-import org.neo4j.helpers.collection.IteratorUtil;
+import org.neo4j.helpers.collection.Iterables;
 
+import java.io.File;
 import java.util.*;
 
 /**
@@ -73,13 +74,13 @@ private String toString(Object value) {
         }
     }
 	public static void main(String[] args) {
-        graphDb = new GraphDatabaseFactory().newEmbeddedDatabase(args[0]);
+        graphDb = new GraphDatabaseFactory().newEmbeddedDatabase(new File(args[0]));
 
         long time = System.currentTimeMillis();
         Map<Set<String>,Sample> statistics = new HashMap<Set<String>, Sample>();
         int count = 0;
         for (Node node : graphDb.getAllNodes()) {
-            final HashSet<String> keys = IteratorUtil.addToCollection(node.getPropertyKeys(), new HashSet<String>());
+            final HashSet<String> keys = Iterables.addToCollection(node.getPropertyKeys(), new HashSet<String>());
             Sample sample = statistics.get(keys);
             if (sample==null) {
                 sample = new Sample(node);
diff --git a/src/main/java/org/neo4j/tool/GraphGenerator.java b/src/main/java/org/neo4j/tool/GraphGenerator.java
index 5402f9e..fa4ecfd 100644
--- a/src/main/java/org/neo4j/tool/GraphGenerator.java
+++ b/src/main/java/org/neo4j/tool/GraphGenerator.java
@@ -6,6 +6,7 @@
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
 
+import java.io.File;
 import java.util.Arrays;
 
 /**
@@ -16,7 +17,7 @@ public class GraphGenerator {
     public static final int MILLION = 1000 * 1000;
 
     public static void main(String[] args) {
-        final GraphDatabaseService gdb = new GraphDatabaseFactory().newEmbeddedDatabase("target/data");
+        final GraphDatabaseService gdb = new GraphDatabaseFactory().newEmbeddedDatabase(new File("target/data"));
         createDatabase(gdb);
         gdb.shutdown();
     }
diff --git a/src/main/java/org/neo4j/tool/PropertyAnalyzer.java b/src/main/java/org/neo4j/tool/PropertyAnalyzer.java
index fbbad4b..f582c81 100644
--- a/src/main/java/org/neo4j/tool/PropertyAnalyzer.java
+++ b/src/main/java/org/neo4j/tool/PropertyAnalyzer.java
@@ -4,6 +4,7 @@
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
 import org.neo4j.helpers.collection.MapUtil;
 
+import java.io.File;
 import java.lang.reflect.Array;
 import java.util.*;
 
@@ -95,7 +96,7 @@ public static Map<String,String> config() {
     }
 
     public static void main(String[] args) {
-        final GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(args[0]).setConfig(config()).newGraphDatabase();
+        final GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(new File(args[0])).setConfig(config()).newGraphDatabase();
         int withoutProps=0, nodes = 0, rels = 0;
         Map<String,PropertyInfo> props=new HashMap<String, PropertyInfo>();
         for (Node node : db.getAllNodes()) {
diff --git a/src/main/java/org/neo4j/tool/StoreComparer.java b/src/main/java/org/neo4j/tool/StoreComparer.java
index 81faed6..90ce4d9 100644
--- a/src/main/java/org/neo4j/tool/StoreComparer.java
+++ b/src/main/java/org/neo4j/tool/StoreComparer.java
@@ -2,7 +2,7 @@
 
 import org.neo4j.graphdb.*;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
-import org.neo4j.helpers.collection.IteratorUtil;
+import org.neo4j.helpers.collection.Iterables;
 import org.neo4j.helpers.collection.MapUtil;
 import org.neo4j.io.fs.FileUtils;
 
@@ -11,7 +11,6 @@
 import java.util.*;
 
 import static java.util.Arrays.asList;
-import static java.util.Arrays.toString;
 import static java.util.Collections.emptySet;
 
 public class StoreComparer {
@@ -54,8 +53,8 @@ private static void compareStore(String sourceDir, String targetDir, Set<String>
         if (!target.exists()) throw new IllegalArgumentException("Target Directory does not exists " + target);
         if (!source.exists()) throw new IllegalArgumentException("Source Database does not exist " + source);
 
-        GraphDatabaseService targetDb = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(target.getAbsolutePath()).setConfig(config()).newGraphDatabase();
-        GraphDatabaseService sourceDb = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(sourceDir).setConfig(config()).newGraphDatabase();
+        GraphDatabaseService targetDb = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(target).setConfig(config()).newGraphDatabase();
+        GraphDatabaseService sourceDb = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(source).setConfig(config()).newGraphDatabase();
 
         try (Transaction srcDbTx = sourceDb.beginTx();
              Transaction targetDbTx = targetDb.beginTx()) {
@@ -100,7 +99,7 @@ private static Statistics count(GraphDatabaseService db, Set<String> ignoreRelTy
     }
 
     private static int countProperties(Set<String> ignoreProperties, PropertyContainer node) {
-        final Collection<String> keys = IteratorUtil.addToCollection(node.getPropertyKeys(), new HashSet<String>());
+        final Collection<String> keys = Iterables.addToCollection(node.getPropertyKeys(), new HashSet<String>());
         keys.removeAll(ignoreProperties);
         return keys.size();
     }
@@ -144,8 +143,8 @@ private static Relationship getTargetRel(GraphDatabaseService gdb,long startNode
     }
 
     private static void compareProperties(PropertyContainer pc1, PropertyContainer pc2, Set<String> ignoreProperties) {
-        final Collection<String> keys1 = IteratorUtil.addToCollection(pc1.getPropertyKeys(), new HashSet<String>());
-        final Collection<String> keys2 = IteratorUtil.addToCollection(pc2.getPropertyKeys(), new HashSet<String>());
+        final Collection<String> keys1 = Iterables.addToCollection(pc1.getPropertyKeys(), new HashSet<String>());
+        final Collection<String> keys2 = Iterables.addToCollection(pc2.getPropertyKeys(), new HashSet<String>());
         keys2.removeAll(ignoreProperties);
         keys1.removeAll(ignoreProperties);
         if (!keys1.equals(keys2)) {
diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index d2d1f43..5fd4bdc 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -2,7 +2,7 @@
 
 import org.neo4j.graphdb.*;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
-import org.neo4j.helpers.collection.IteratorUtil;
+import org.neo4j.helpers.collection.Iterables;
 import org.neo4j.helpers.collection.MapUtil;
 import org.neo4j.helpers.collection.Pair;
 import org.neo4j.io.fs.FileUtils;
@@ -10,6 +10,7 @@
 import org.neo4j.kernel.impl.store.id.IdType;
 import org.neo4j.kernel.internal.GraphDatabaseAPI;
 import org.neo4j.unsafe.batchinsert.*;
+import org.neo4j.unsafe.batchinsert.internal.BatchInserterImpl;
 
 import java.io.*;
 import java.lang.reflect.Field;
@@ -63,8 +64,8 @@ private static void copyStore(String sourceDir, String targetDir, Set<String> ig
         if (!source.exists()) throw new IllegalArgumentException("Source Database does not exist " + source);
 
         Pair<Long, Long> highestIds = getHighestNodeId(source);
-        BatchInserter targetDb = BatchInserters.inserter(target.getAbsolutePath(), config());
-        BatchInserter sourceDb = BatchInserters.inserter(source.getAbsolutePath(), config());
+        BatchInserter targetDb = BatchInserters.inserter(target, config());
+        BatchInserter sourceDb = BatchInserters.inserter(source, config());
         Flusher flusher = getFlusher(sourceDb);
 
         logs = new PrintWriter(new FileWriter(new File(target, "store-copy.log")));
@@ -101,7 +102,7 @@ private static Flusher getFlusher(final BatchInserter db) {
     }
 
     private static Pair<Long, Long> getHighestNodeId(File source) {
-        GraphDatabaseAPI api = (GraphDatabaseAPI) new GraphDatabaseFactory().newEmbeddedDatabase(source.getAbsolutePath());
+        GraphDatabaseAPI api = (GraphDatabaseAPI) new GraphDatabaseFactory().newEmbeddedDatabase(source);
         IdGeneratorFactory idGenerators = api.getDependencyResolver().resolveDependency(IdGeneratorFactory.class);
         long highestNodeId = idGenerators.get(IdType.NODE).getHighestPossibleIdInUse();
         long highestRelId = idGenerators.get(IdType.RELATIONSHIP).getHighestPossibleIdInUse();
@@ -222,7 +223,7 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
     }
 
     private static Label[] labelsArray(BatchInserter db, long node, Set<String> ignoreLabels) {
-        Collection<Label> labels = IteratorUtil.asCollection(db.getNodeLabels(node));
+        Collection<Label> labels = Iterables.asCollection(db.getNodeLabels(node));
         if (labels.isEmpty()) return NO_LABELS;
         if (!ignoreLabels.isEmpty()) {
             for (Iterator<Label> it = labels.iterator(); it.hasNext(); ) {
diff --git a/src/main/java/org/neo4j/tool/StoreUpdate.java b/src/main/java/org/neo4j/tool/StoreUpdate.java
index 93cbee5..72260b8 100644
--- a/src/main/java/org/neo4j/tool/StoreUpdate.java
+++ b/src/main/java/org/neo4j/tool/StoreUpdate.java
@@ -4,6 +4,7 @@
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
 import org.neo4j.helpers.collection.MapUtil;
 
+import java.io.File;
 import java.util.Map;
 
 /**
@@ -29,7 +30,7 @@ public static Map<String, String> config() {
     public static void main(String[] args) {
         GraphDatabaseService db = null;
         try {
-            db = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder("target/data").setConfig(config()).newGraphDatabase();
+            db = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(new File("target/data")).setConfig(config()).newGraphDatabase();
         } finally {
             if (db != null) db.shutdown();
         }

From 9e9194a9cd838ee6d7d3c2d8f066a70d7ce42f1f Mon Sep 17 00:00:00 2001
From: Konstantin Lutovich <konstantin.lutovich@neotechnology.com>
Date: Tue, 25 Oct 2016 18:57:29 +0200
Subject: [PATCH 26/33] Ignore (only log) exceptions when shutting down source
 db in StoreCopy (#15)

This is needed because during shutdown batch inserter tries to flush it's
caches and some entities in those caches might be corrupt for the source
database.

Problem initially seen for unused dynamic label records.
---
 src/main/java/org/neo4j/tool/StoreCopy.java | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index 5fd4bdc..9c55a13 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -2,6 +2,7 @@
 
 import org.neo4j.graphdb.*;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
+import org.neo4j.helpers.Exceptions;
 import org.neo4j.helpers.collection.Iterables;
 import org.neo4j.helpers.collection.MapUtil;
 import org.neo4j.helpers.collection.Pair;
@@ -73,7 +74,11 @@ private static void copyStore(String sourceDir, String targetDir, Set<String> ig
         copyNodes(sourceDb, targetDb, ignoreProperties, ignoreLabels, highestIds.first(),flusher);
         copyRelationships(sourceDb, targetDb, ignoreRelTypes, ignoreProperties, highestIds.other(), flusher);
         targetDb.shutdown();
-        sourceDb.shutdown();
+        try {
+            sourceDb.shutdown();
+        } catch (Exception e) {
+            logs.append(String.format("Noncritical error closing the source database:%n%s", Exceptions.stringify(e)));
+        }
         logs.close();
         copyIndex(source, target);
     }

From ccc4f9ebd6d5f13fe355a8ea3dc525c5366abdb1 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Fri, 11 Nov 2016 18:58:32 +0100
Subject: [PATCH 27/33] added warning about required memory

---
 copy-store.sh | 1 +
 readme.md     | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/copy-store.sh b/copy-store.sh
index a3eabfd..66b33eb 100755
--- a/copy-store.sh
+++ b/copy-store.sh
@@ -10,6 +10,7 @@ CACHE=2G
 echo "Usage: copy-store.sh [community|enterprise] source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]"
 echo "Database config is read from neo4j.properties file in current directory if it exists"
 echo "Using: Heap $HEAP Pagecache $CACHE Edition $EDITION from $SRC to $DST skipping labels: $SKIP_LABELS rels: $SKIP_RELS props $SKIP_PROPS"
+echo "Please note that you will need twice the memory (2x $CACHE + 1x $HEAP) as it opens 2 databases one for reading and one for writing."
 # heap config
 export MAVEN_OPTS="-Xmx$HEAP -Xms$HEAP -Xmn1G -XX:+UseG1GC"
 
diff --git a/readme.md b/readme.md
index 43a823c..5fd29c1 100644
--- a/readme.md
+++ b/readme.md
@@ -26,6 +26,8 @@ Heap config is in the shell-script, default is:
 
 **Please adapt the settings as needed for your store.**
 
+**Please note that you will need twice the memory (2x page-cache + 1x heap) as it opens 2 databases one for reading and one for writing.**
+
 Change the Neo4j version in pom.xml before running as needed. (Currently 3.0.0-M05)
 
 Optionally changeable from the outside with `-Dneo4j.version=3.0.0` on the `mvn` invocation.

From b8a84f1ece8fed3d9ce83a929888187cd83ae581 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Wed, 21 Dec 2016 16:21:14 +0100
Subject: [PATCH 28/33] Better parameter handling in copy-store.sh script,
 update Neo4j to 3.0.8

---
 copy-store.sh | 24 +++++++++++++++++++++++-
 pom.xml       |  5 ++++-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/copy-store.sh b/copy-store.sh
index 66b33eb..bf51392 100755
--- a/copy-store.sh
+++ b/copy-store.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
 EDITION=${1-community}
 shift
 SRC=$1
@@ -8,8 +10,28 @@ SKIP_LABELS=$5
 HEAP=4G
 CACHE=2G
 echo "Usage: copy-store.sh [community|enterprise] source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]"
+
+if [[ "$EDITION" != "enterprise" && "$EDITION" != "community" ]]
+then
+    echo "ATTENTION: The parameter '$EDITION' you passed in for the edition is neither 'community' nor 'enterprise'. Aborting."
+    exit
+fi
+if [[ "$SRC" = "" || "$DST" = "" ]]
+then
+    echo "ATTENTION: Source '$SRC' or target '$DST' directory not provided. Aborting."
+    exit
+fi
+
+if [[ ! -d $SRC ]]
+then
+    echo "ATTENTION: Source '$SRC' is not a directory. Aborting."
+    exit
+fi
+
 echo "Database config is read from neo4j.properties file in current directory if it exists"
-echo "Using: Heap $HEAP Pagecache $CACHE Edition $EDITION from $SRC to $DST skipping labels: $SKIP_LABELS rels: $SKIP_RELS props $SKIP_PROPS"
+echo
+echo "Using: Heap $HEAP Pagecache $CACHE Edition '$EDITION' from '$SRC' to '$DST' skipping labels: '$SKIP_LABELS' rels: '$SKIP_RELS' props '$SKIP_PROPS'"
+echo
 echo "Please note that you will need twice the memory (2x $CACHE + 1x $HEAP) as it opens 2 databases one for reading and one for writing."
 # heap config
 export MAVEN_OPTS="-Xmx$HEAP -Xms$HEAP -Xmn1G -XX:+UseG1GC"
diff --git a/pom.xml b/pom.xml
index 781de76..4204027 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>3.0.3</version>
+    <version>3.0.8</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>
@@ -25,6 +25,9 @@
                 </dependency>
             </dependencies>
         </profile>
+        <profile>
+            <id>community</id>
+        </profile>
     </profiles>
     <dependencies>
         <dependency>

From f304b207fdcc001e77e1f7a18a443c79ecfb5ff6 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Wed, 21 Dec 2016 16:30:48 +0100
Subject: [PATCH 29/33] New branch for Neo4j 3.1.0

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 4204027..11232fd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>3.0.8</version>
+    <version>3.1.0</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>

From 58b9fd709e2a92a7135d743141286cba8c776d35 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Wed, 21 Dec 2016 16:31:39 +0100
Subject: [PATCH 30/33] readme update

---
 readme.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/readme.md b/readme.md
index 5fd29c1..a6edba0 100644
--- a/readme.md
+++ b/readme.md
@@ -9,7 +9,7 @@ Also useful to skip no longer wanted properties or relationships with a certain
 Good for store compaction and reorganization of relationships and properties as
 it rewrites the store file reclaiming space that is sitting empty.
 
-NOTE: With Neo4j 3.0 there are two different store formats, so you have to provide "enterprise" or "community" as first argument of the call!
+NOTE: With Neo4j 3.x there are two different store formats, so you have to provide "enterprise" or "community" as first argument of the call!
 
 ### Store Copy
 
@@ -28,9 +28,9 @@ Heap config is in the shell-script, default is:
 
 **Please note that you will need twice the memory (2x page-cache + 1x heap) as it opens 2 databases one for reading and one for writing.**
 
-Change the Neo4j version in pom.xml before running as needed. (Currently 3.0.0-M05)
+Change the Neo4j version in pom.xml before running as needed. (Currently 3.1.0)
 
-Optionally changeable from the outside with `-Dneo4j.version=3.0.0` on the `mvn` invocation.
+Optionally changeable from the outside with `-Dneo4j.version=3.1.0` on the `mvn` invocation.
 
 ### Internally
 

From 285def03b0bf60337e87d516301d987d91dd31ee Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Mon, 22 May 2017 23:03:49 +0200
Subject: [PATCH 31/33] separate size four source page cache

---
 copy-store.sh                               |  6 ++++--
 src/main/java/org/neo4j/tool/StoreCopy.java | 13 +++----------
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/copy-store.sh b/copy-store.sh
index bf51392..be7c9c3 100755
--- a/copy-store.sh
+++ b/copy-store.sh
@@ -9,6 +9,8 @@ SKIP_PROPS=$4
 SKIP_LABELS=$5
 HEAP=4G
 CACHE=2G
+CACHE_SRC=1G
+#$CACHE
 echo "Usage: copy-store.sh [community|enterprise] source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]"
 
 if [[ "$EDITION" != "enterprise" && "$EDITION" != "community" ]]
@@ -32,11 +34,11 @@ echo "Database config is read from neo4j.properties file in current directory if
 echo
 echo "Using: Heap $HEAP Pagecache $CACHE Edition '$EDITION' from '$SRC' to '$DST' skipping labels: '$SKIP_LABELS' rels: '$SKIP_RELS' props '$SKIP_PROPS'"
 echo
-echo "Please note that you will need twice the memory (2x $CACHE + 1x $HEAP) as it opens 2 databases one for reading and one for writing."
+echo "Please note that you will need this memory ($CACHE + $CACHE_SRC + $HEAP) as it opens 2 databases one for reading and one for writing."
 # heap config
 export MAVEN_OPTS="-Xmx$HEAP -Xms$HEAP -Xmn1G -XX:+UseG1GC"
 
-mvn clean compile exec:java -P${EDITION} -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=$CACHE \
+mvn clean compile exec:java -P${EDITION} -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=$CACHE -Ddbms.pagecache.memory.source=$CACHE_SRC \
       -Dexec.args="$SRC $DST $SKIP_RELS $SKIP_PROPS $SKIP_LABELS"
 
 #-Dneo4j.version=2.3.0
diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index 9c55a13..c2a9235 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -25,14 +25,6 @@ public class StoreCopy {
     private static final Label[] NO_LABELS = new Label[0];
     private static PrintWriter logs;
 
-    @SuppressWarnings("unchecked")
-    public static Map<String, String> config() {
-        return (Map) MapUtil.map(
-                "dbms.pagecache.memory", System.getProperty("dbms.pagecache.memory","2G"),
-                "cache_type", "none"
-        );
-    }
-
     public static void main(String[] args) throws Exception {
         if (args.length < 2) {
             System.err.println("Usage: StoryCopy source target [rel,types,to,ignore] [properties,to,ignore]");
@@ -65,8 +57,9 @@ private static void copyStore(String sourceDir, String targetDir, Set<String> ig
         if (!source.exists()) throw new IllegalArgumentException("Source Database does not exist " + source);
 
         Pair<Long, Long> highestIds = getHighestNodeId(source);
-        BatchInserter targetDb = BatchInserters.inserter(target, config());
-        BatchInserter sourceDb = BatchInserters.inserter(source, config());
+        String pageCacheSize = System.getProperty("dbms.pagecache.memory","2G");
+        BatchInserter targetDb = BatchInserters.inserter(target, MapUtil.stringMap("dbms.pagecache.memory", pageCacheSize));
+        BatchInserter sourceDb = BatchInserters.inserter(source, MapUtil.stringMap("dbms.pagecache.memory", System.getProperty("dbms.pagecache.memory.source",pageCacheSize)));
         Flusher flusher = getFlusher(sourceDb);
 
         logs = new PrintWriter(new FileWriter(new File(target, "store-copy.log")));

From 7774b2bf8b53c92d247f66f6837551b70b91a9a0 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Wed, 24 May 2017 07:42:19 +0200
Subject: [PATCH 32/33] fix node id int to long

---
 src/main/java/org/neo4j/tool/StoreCopy.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index c2a9235..23461f4 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -158,7 +158,7 @@ private static int percent(Number part, Number total) {
     }
 
     private static long firstNode(BatchInserter sourceDb, long highestNodeId) {
-        int node = -1;
+        long node = -1;
         while (++node <= highestNodeId) {
             if (sourceDb.nodeExists(node) && !sourceDb.getNodeProperties(node).isEmpty()) return node;
         }
@@ -191,7 +191,7 @@ private static void createRelationship(BatchInserter targetDb, BatchInserter sou
 
     private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreProperties, Set<String> ignoreLabels, long highestNodeId, Flusher flusher) {
         long time = System.currentTimeMillis();
-        int node = 0;
+        long node = 0;
         long notFound = 0;
         while (node <= highestNodeId) {
             try {

From d6417bc467300ed4538484dd1fe9cc30d0a224c3 Mon Sep 17 00:00:00 2001
From: Michael Hunger <github@jexp.de>
Date: Sat, 27 May 2017 22:51:08 +0200
Subject: [PATCH 33/33] Updated Capabilities of Store Copy

- allow for labels to be specified that cause their nodes to be deleted
- allow for relationships to be skipped if their nodes have not been copied
- allow for specification if node-ids should be compacted
---
 copy-store.sh                               |  12 +--
 neo4j.properties                            |  14 +++
 readme.md                                   |  40 ++++++--
 src/main/java/org/neo4j/tool/StoreCopy.java | 104 ++++++++++++++------
 4 files changed, 127 insertions(+), 43 deletions(-)

diff --git a/copy-store.sh b/copy-store.sh
index be7c9c3..e0723ee 100755
--- a/copy-store.sh
+++ b/copy-store.sh
@@ -7,11 +7,13 @@ DST=$2
 SKIP_RELS=$3
 SKIP_PROPS=$4
 SKIP_LABELS=$5
+DELETE_NODES=$6
+KEEP_NODE_IDS=$7
 HEAP=4G
 CACHE=2G
 CACHE_SRC=1G
 #$CACHE
-echo "Usage: copy-store.sh [community|enterprise] source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]"
+echo "Usage: copy-store.sh [community|enterprise] source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip] [Labels,To,Delete,Nodes]"
 
 if [[ "$EDITION" != "enterprise" && "$EDITION" != "community" ]]
 then
@@ -30,15 +32,13 @@ then
     exit
 fi
 
-echo "Database config is read from neo4j.properties file in current directory if it exists"
-echo
-echo "Using: Heap $HEAP Pagecache $CACHE Edition '$EDITION' from '$SRC' to '$DST' skipping labels: '$SKIP_LABELS' rels: '$SKIP_RELS' props '$SKIP_PROPS'"
+echo "Using: Heap $HEAP Pagecache $CACHE Edition '$EDITION' from '$SRC' to '$DST' skipping labels: '$SKIP_LABELS', removing nodes with labels: '$DELETE_NODES' rels: '$SKIP_RELS' props '$SKIP_PROPS' Keeping Node Ids: $KEEP_NODE_IDS"
 echo
 echo "Please note that you will need this memory ($CACHE + $CACHE_SRC + $HEAP) as it opens 2 databases one for reading and one for writing."
 # heap config
-export MAVEN_OPTS="-Xmx$HEAP -Xms$HEAP -Xmn1G -XX:+UseG1GC"
+export MAVEN_OPTS="-Xmx$HEAP -Xms$HEAP -XX:+UseG1GC"
 
 mvn clean compile exec:java -P${EDITION} -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=$CACHE -Ddbms.pagecache.memory.source=$CACHE_SRC \
-      -Dexec.args="$SRC $DST $SKIP_RELS $SKIP_PROPS $SKIP_LABELS"
+      -Dexec.args="$SRC $DST $SKIP_RELS $SKIP_PROPS $SKIP_LABELS $DELETE_NODES $KEEP_NODE_IDS"
 
 #-Dneo4j.version=2.3.0
diff --git a/neo4j.properties b/neo4j.properties
index fdd3598..f3e1314 100644
--- a/neo4j.properties
+++ b/neo4j.properties
@@ -1,4 +1,18 @@
 dbms.pagecache.memory=2G
+dbms.pagecache.memory.source=2G
+
 cache_type=none
 allow_store_upgrade=true
 
+source_db_dir=
+target_db_dir=
+
+keep_node_ids=true
+
+properties_to_ignore=
+labels_to_ignore=
+labels_to_delete=
+rel_types_to_ignore=
+
+store_copy_log_dir=
+bad_entries_log_dir=
diff --git a/readme.md b/readme.md
index a6edba0..c2df9fc 100644
--- a/readme.md
+++ b/readme.md
@@ -1,23 +1,49 @@
-## Tools to copy Neo4j Stores
+## Tool to copy Neo4j Stores
 
 Uses the BatchInserterImpl to read a store and write the target store keeping the node-ids.
 Copies the manual (legacy) index-files as is, please note it performs no index upgrade!
 
+You will have to recreate any schema indexes too.
+
 Ignores broken nodes and relationships and records them in `target/store-copy.log`
 
-Also useful to skip no longer wanted properties or relationships with a certain type.
+Also useful to skip no longer wanted properties, relationships with a certain type.
+Or of certain labels and even nodes with certain labels.
+
 Good for store compaction and reorganization of relationships and properties as
 it rewrites the store file reclaiming space that is sitting empty.
 
 NOTE: With Neo4j 3.x there are two different store formats, so you have to provide "enterprise" or "community" as first argument of the call!
 
+You can now also decide if you want to compact the node-store, then you have to pass "false" as the parameter for keep-node-ids.
+
+Config is read from `neo4j.properties` file in current directory if it exists, but command line options override.
+
+neo4j.properties
+
+```
+source_db_dir=
+target_db_dir=
+
+keep_node_ids=true
+
+properties_to_ignore=
+labels_to_ignore=
+labels_to_delete=
+rel_types_to_ignore=
+
+store_copy_log_dir=
+bad_entries_log_dir=
+```
+
 ### Store Copy
 
-    copy-store.sh [enterprise|community] source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip]
+    copy-store.sh [enterprise|community] source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip] [Labels,To,Delete,Nodes] [keep-node-ids:true/false]
+
 
-Database config is read from `neo4j.properties` file in current directory if it exists.
-The provided one contains:
+The provided script contains these settings for page-cache (note you can configure a different, smaller setting for the source store than the target store).
 
+    dbms.pagecache.memory.source=2G
     dbms.pagecache.memory=2G
 
 Heap config is in the shell-script, default is:
@@ -26,7 +52,7 @@ Heap config is in the shell-script, default is:
 
 **Please adapt the settings as needed for your store.**
 
-**Please note that you will need twice the memory (2x page-cache + 1x heap) as it opens 2 databases one for reading and one for writing.**
+**Please note that you will need the memory for (source-page-cache + target-page-cache + 1x heap) as it opens 2 databases one for reading and one for writing.**
 
 Change the Neo4j version in pom.xml before running as needed. (Currently 3.1.0)
 
@@ -37,5 +63,5 @@ Optionally changeable from the outside with `-Dneo4j.version=3.1.0` on the `mvn`
 Note: It calls under the hood:
 
     mvn compile exec:java -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Penterprise \
-      -Dexec.args="source-dir target-dir [REL,TYPES,TO,IGNORE] [properties,to,ignore] [Labels,To,Ignore]"
+      -Dexec.args="source-dir target-dir [REL,TYPES,TO,IGNORE] [properties,to,ignore] [Labels,To,Ignore] [Labels,To,Delete,Nodes] [keep-node-ids:true/false]"
 
diff --git a/src/main/java/org/neo4j/tool/StoreCopy.java b/src/main/java/org/neo4j/tool/StoreCopy.java
index 23461f4..3960358 100644
--- a/src/main/java/org/neo4j/tool/StoreCopy.java
+++ b/src/main/java/org/neo4j/tool/StoreCopy.java
@@ -1,5 +1,7 @@
 package org.neo4j.tool;
 
+import org.neo4j.collection.primitive.Primitive;
+import org.neo4j.collection.primitive.PrimitiveLongLongMap;
 import org.neo4j.graphdb.*;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
 import org.neo4j.helpers.Exceptions;
@@ -27,27 +29,38 @@ public class StoreCopy {
 
     public static void main(String[] args) throws Exception {
         if (args.length < 2) {
-            System.err.println("Usage: StoryCopy source target [rel,types,to,ignore] [properties,to,ignore]");
+            System.err.println("Usage: StoryCopy source target [rel,types,to,ignore] [properties,to,ignore] [labels,to,ignore] [labels,to,delete]");
             return;
         }
-        String sourceDir = args[0];
-        String targetDir = args[1];
-        Set<String> ignoreRelTypes = splitOptionIfExists(args, 2);
-        Set<String> ignoreProperties = splitOptionIfExists(args, 3);
-        Set<String> ignoreLabels = splitOptionIfExists(args, 4);
-        System.out.printf("Copying from %s to %s ingoring rel-types %s ignoring properties %s ignoring labels %s %n", sourceDir, targetDir, ignoreRelTypes, ignoreProperties,ignoreLabels);
-        copyStore(sourceDir, targetDir, ignoreRelTypes, ignoreProperties,ignoreLabels);
+        Properties properties = new Properties();
+        properties.load(new FileReader("neo4j.properties"));
+        String sourceDir = getArgument(args,0,properties,"source_db_dir");
+        String targetDir = getArgument(args,1,properties,"target_db_dir");
+
+        Set<String> ignoreRelTypes = splitToSet(getArgument(args,2,properties,"rel_types_to_ignore"));
+        Set<String> ignoreProperties = splitToSet(getArgument(args,3,properties,"properties_to_ignore"));
+        Set<String> ignoreLabels = splitToSet(getArgument(args,4,properties,"labels_to_ignore"));
+        Set<String> deleteNodesWithLabels = splitToSet(getArgument(args,5,properties,"labels_to_delete"));
+        String keepNodeIdsParam = getArgument(args, 6, properties, "keep_node_ids");
+        boolean keepNodeIds = !("false".equalsIgnoreCase(keepNodeIdsParam));
+        System.out.printf("Copying from %s to %s ingoring rel-types %s ignoring properties %s ignoring labels %s removing nodes with labels %s keep node ids %s %n", sourceDir, targetDir, ignoreRelTypes, ignoreProperties,ignoreLabels, deleteNodesWithLabels,keepNodeIds);
+        copyStore(sourceDir, targetDir, ignoreRelTypes, ignoreProperties,ignoreLabels,deleteNodesWithLabels, keepNodeIds);
+    }
+
+    private static String getArgument(String[] args, int index, Properties properties, String key) {
+        if (args.length > index) return args[index];
+        return properties.getProperty(key);
     }
 
-    private static Set<String> splitOptionIfExists(String[] args, final int index) {
-        if (args.length <= index) return emptySet();
-        return new HashSet<String>(asList(args[index].toLowerCase().split(",")));
+    private static Set<String> splitToSet(String value) {
+        if (value == null || value.trim().isEmpty()) return emptySet();
+        return new HashSet<>(asList(value.trim().split(", *")));
     }
 
     interface Flusher {
         void flush();
     }
-    private static void copyStore(String sourceDir, String targetDir, Set<String> ignoreRelTypes, Set<String> ignoreProperties, Set<String> ignoreLabels) throws Exception {
+    private static void copyStore(String sourceDir, String targetDir, Set<String> ignoreRelTypes, Set<String> ignoreProperties, Set<String> ignoreLabels, Set<String> deleteNodesWithLabels, boolean stableNodeIds) throws Exception {
         final File target = new File(targetDir);
         final File source = new File(sourceDir);
         if (target.exists()) {
@@ -64,8 +77,8 @@ private static void copyStore(String sourceDir, String targetDir, Set<String> ig
 
         logs = new PrintWriter(new FileWriter(new File(target, "store-copy.log")));
 
-        copyNodes(sourceDb, targetDb, ignoreProperties, ignoreLabels, highestIds.first(),flusher);
-        copyRelationships(sourceDb, targetDb, ignoreRelTypes, ignoreProperties, highestIds.other(), flusher);
+        PrimitiveLongLongMap copiedNodeIds = copyNodes(sourceDb, targetDb, ignoreProperties, ignoreLabels, deleteNodesWithLabels, highestIds.first(),flusher, stableNodeIds);
+        copyRelationships(sourceDb, targetDb, ignoreRelTypes, ignoreProperties, copiedNodeIds, highestIds.other(), flusher);
         targetDb.shutdown();
         try {
             sourceDb.shutdown();
@@ -119,18 +132,23 @@ private static void copyIndex(File source, File target) throws IOException {
         }
     }
 
-    private static void copyRelationships(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreRelTypes, Set<String> ignoreProperties, long highestRelId, Flusher flusher) {
+    private static void copyRelationships(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreRelTypes, Set<String> ignoreProperties, PrimitiveLongLongMap copiedNodeIds, long highestRelId, Flusher flusher) {
         long time = System.currentTimeMillis();
         long relId = 0;
         long notFound = 0;
+        long removed = 0;
         while (relId <= highestRelId) {
             BatchRelationship rel = null;
             String type = null;
             try {
                 rel = sourceDb.getRelationshipById(relId++);
                 type = rel.getType().name();
-                if (!ignoreRelTypes.contains(type.toLowerCase())) {
-                    createRelationship(targetDb, sourceDb, rel, ignoreProperties);
+                if (!ignoreRelTypes.contains(type)) {
+                    if (!createRelationship(targetDb, sourceDb, rel, ignoreProperties, copiedNodeIds)) {
+                        removed++;
+                    }
+                } else {
+                    removed++;
                 }
             } catch (Exception e) {
                 if (e instanceof org.neo4j.kernel.impl.store.InvalidRecordException && e.getMessage().endsWith("not in use")) {
@@ -145,12 +163,12 @@ private static void copyRelationships(BatchInserter sourceDb, BatchInserter targ
             }
             if (relId % 500000 == 0) {
                 flusher.flush();
-                System.out.printf(" %d / %d (%d%%) unused %d%n", relId, highestRelId, percent(relId,highestRelId), notFound);
+                System.out.printf(" %d / %d (%d%%) unused %d removed %d%n", relId, highestRelId, percent(relId,highestRelId), notFound,removed);
             }
         }
         time = Math.max(1,(System.currentTimeMillis() - time)/1000);
-        System.out.printf("%n copying of %d relationship records took %d seconds (%d rec/s). Unused Records %d (%d%%)%n",
-                relId, time, relId/time, notFound, percent(notFound,relId));
+        System.out.printf("%n copying of %d relationship records took %d seconds (%d rec/s). Unused Records %d (%d%%) Removed Records %d (%d%%)%n",
+                relId, time, relId/time, notFound, percent(notFound,relId),removed, percent(removed,relId));
     }
 
     private static int percent(Number part, Number total) {
@@ -176,27 +194,42 @@ private static void flushCache(BatchInserter sourceDb, long node) {
         }
     }
 
-    private static void createRelationship(BatchInserter targetDb, BatchInserter sourceDb, BatchRelationship rel, Set<String> ignoreProperties) {
-        long startNodeId = rel.getStartNode();
-        long endNodeId = rel.getEndNode();
+    private static boolean createRelationship(BatchInserter targetDb, BatchInserter sourceDb, BatchRelationship rel, Set<String> ignoreProperties, PrimitiveLongLongMap copiedNodeIds) {
+        long startNodeId = copiedNodeIds.get(rel.getStartNode());
+        long endNodeId = copiedNodeIds.get(rel.getEndNode());
+        if (startNodeId == -1L || endNodeId == -1L) return false;
         final RelationshipType type = rel.getType();
         try {
             Map<String, Object> props = getProperties(sourceDb.getRelationshipProperties(rel.getId()), ignoreProperties);
 //            if (props.isEmpty()) props = Collections.<String,Object>singletonMap("old_id",rel.getId()); else props.put("old_id",rel.getId());
             targetDb.createRelationship(startNodeId, endNodeId, type, props);
+            return true;
         } catch (Exception e) {
             addLog(rel, "create Relationship: " + startNodeId + "-[:" + type + "]" + "->" + endNodeId, e.getMessage());
+            return false;
         }
     }
 
-    private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreProperties, Set<String> ignoreLabels, long highestNodeId, Flusher flusher) {
+    private static PrimitiveLongLongMap copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Set<String> ignoreProperties, Set<String> ignoreLabels, Set<String> deleteNodesWithLabels, long highestNodeId, Flusher flusher, boolean stableNodeIds) {
+        PrimitiveLongLongMap copiedNodes = Primitive.offHeapLongLongMap();
         long time = System.currentTimeMillis();
         long node = 0;
         long notFound = 0;
+        long removed = 0;
         while (node <= highestNodeId) {
             try {
                 if (sourceDb.nodeExists(node)) {
-                    targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node, ignoreLabels));
+                    if (labelInSet(sourceDb.getNodeLabels(node),deleteNodesWithLabels)) {
+                        removed ++;
+                    } else {
+                        long newNodeId=node;
+                        if (stableNodeIds) {
+                            targetDb.createNode(node, getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node, ignoreLabels));
+                        } else {
+                            newNodeId = targetDb.createNode(getProperties(sourceDb.getNodeProperties(node), ignoreProperties), labelsArray(sourceDb, node, ignoreLabels));
+                        }
+                        copiedNodes.put(node,newNodeId);
+                    }
                 } else {
                     notFound++;
                 }
@@ -212,12 +245,21 @@ private static void copyNodes(BatchInserter sourceDb, BatchInserter targetDb, Se
             if (node % 500000 == 0) {
                 flusher.flush();
                 logs.flush();
-                System.out.printf(" %d / %d (%d%%)%n unused %d", node, highestNodeId, percent(node,highestNodeId), notFound);
+                System.out.printf(" %d / %d (%d%%) unused %d removed %d%n", node, highestNodeId, percent(node,highestNodeId), notFound, removed);
             }
         }
         time = Math.max(1,(System.currentTimeMillis() - time)/1000);
-        System.out.printf("%n copying of %d node records took %d seconds (%d rec/s). Unused Records %d (%d%%)%n",
-                node, time, node/time, notFound, percent(notFound,node));
+        System.out.printf("%n copying of %d node records took %d seconds (%d rec/s). Unused Records %d (%d%%). Removed Records %d (%d%%).%n",
+                node, time, node/time, notFound, percent(notFound,node),removed, percent(removed,node));
+        return copiedNodes;
+    }
+
+    private static boolean labelInSet(Iterable<Label> nodeLabels, Set<String> labelSet) {
+        if (labelSet == null || labelSet.isEmpty()) return false;
+        for (Label nodeLabel : nodeLabels) {
+            if (labelSet.contains(nodeLabel.name())) return true;
+        }
+        return false;
     }
 
     private static Label[] labelsArray(BatchInserter db, long node, Set<String> ignoreLabels) {
@@ -226,7 +268,7 @@ private static Label[] labelsArray(BatchInserter db, long node, Set<String> igno
         if (!ignoreLabels.isEmpty()) {
             for (Iterator<Label> it = labels.iterator(); it.hasNext(); ) {
                 Label label = it.next();
-                if (ignoreLabels.contains(label.name().toLowerCase())) {
+                if (ignoreLabels.contains(label.name())) {
                     it.remove();
                 }
             }
@@ -235,7 +277,9 @@ private static Label[] labelsArray(BatchInserter db, long node, Set<String> igno
     }
 
     private static Map<String, Object> getProperties(Map<String, Object> pc, Set<String> ignoreProperties) {
-        if (!ignoreProperties.isEmpty()) pc.keySet().removeAll(ignoreProperties);
+        if (pc.isEmpty()) return Collections.emptyMap();
+        if (ignoreProperties.isEmpty()) return pc;
+        pc.keySet().removeAll(ignoreProperties);
         return pc;
     }