From a350c1e5ec49ad1728060d3107f7a7f1eb738da7 Mon Sep 17 00:00:00 2001 From: sharkovadarya Date: Fri, 13 Apr 2018 17:51:54 +0300 Subject: [PATCH 1/5] cw1 initial commit (tests are currently not working) --- cw1/pom.xml | 62 ++++++++++++ .../java/ru/spbau/group202/cw1/MD5Hasher.java | 22 +++++ .../ru/spbau/group202/cw1/MD5MultiThread.java | 84 ++++++++++++++++ .../spbau/group202/cw1/MD5SingleThread.java | 42 ++++++++ .../main/java/ru/spbau/group202/cw1/Main.java | 36 +++++++ .../ru/spbau/group202/cw1/MD5HasherTest.java | 98 +++++++++++++++++++ cw1/src/test/resources/testdir/file1.txt | 1 + cw1/src/test/resources/testdir/file2.txt | 0 8 files changed, 345 insertions(+) create mode 100644 cw1/pom.xml create mode 100644 cw1/src/main/java/ru/spbau/group202/cw1/MD5Hasher.java create mode 100644 cw1/src/main/java/ru/spbau/group202/cw1/MD5MultiThread.java create mode 100644 cw1/src/main/java/ru/spbau/group202/cw1/MD5SingleThread.java create mode 100644 cw1/src/main/java/ru/spbau/group202/cw1/Main.java create mode 100644 cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java create mode 100644 cw1/src/test/resources/testdir/file1.txt create mode 100644 cw1/src/test/resources/testdir/file2.txt diff --git a/cw1/pom.xml b/cw1/pom.xml new file mode 100644 index 0000000..b0bf10b --- /dev/null +++ b/cw1/pom.xml @@ -0,0 +1,62 @@ + + + 4.0.0 + + ru.spbau.group202.sharkova.cw1 + cw1 + 1.0-SNAPSHOT + + + + + junit + junit + 4.9 + test + + + com.github.stefanbirkner + system-rules + 1.16.0 + test + + + + com.intellij + annotations + 12.0 + + + + + + org.mockito + mockito-all + 1.10.19 + test + + + + 1.8 + 1.8 + + + + + + com.zenjava + javafx-maven-plugin + 8.8.3 + + your.package.with.Launcher + + + + + + + \ No newline at end of file diff --git a/cw1/src/main/java/ru/spbau/group202/cw1/MD5Hasher.java b/cw1/src/main/java/ru/spbau/group202/cw1/MD5Hasher.java new file mode 100644 index 0000000..a25abbe --- /dev/null +++ b/cw1/src/main/java/ru/spbau/group202/cw1/MD5Hasher.java @@ -0,0 +1,22 @@ +package ru.spbau.group202.cw1; + +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.nio.file.Path; +import java.security.NoSuchAlgorithmException; + +/** + * Instances of this abstract class calculate MD5 check-sum of a directory. + */ +abstract public class MD5Hasher { + protected final static int BUFFER_SIZE = 4096; + + /** + * This method calculates MD5 check-sum of a file or a directory. + * @param path path to file/directory + * @return byte[] array storing MD5 check-sum + */ + @NotNull + public abstract byte[] getHashFromPath(@NotNull Path path) throws IOException, NoSuchAlgorithmException; +} diff --git a/cw1/src/main/java/ru/spbau/group202/cw1/MD5MultiThread.java b/cw1/src/main/java/ru/spbau/group202/cw1/MD5MultiThread.java new file mode 100644 index 0000000..5fa00b5 --- /dev/null +++ b/cw1/src/main/java/ru/spbau/group202/cw1/MD5MultiThread.java @@ -0,0 +1,84 @@ +package ru.spbau.group202.cw1; + +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.RecursiveTask; +import java.util.stream.Collectors; + +/** + * This implementation of MD5Hasher abstract class uses a ForkJoinPool + * to compute MD5 check-sum. + */ +public class MD5MultiThread extends MD5Hasher { + + private ForkJoinPool pool; + + public MD5MultiThread(int numberOfThreads) { + pool = new ForkJoinPool(numberOfThreads); + } + + /** + * {@inheritDoc} + */ + @NotNull + public byte[] getHashFromPath(@NotNull Path path) throws IOException, NoSuchAlgorithmException { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + if (Files.isDirectory(path)) { + processDirectory(path, md5); + } else { + processFile(path, md5); + } + + return md5.digest(); + } + + private void processDirectory(Path path, MessageDigest md) throws IOException { + md.update(path.getFileName().toString().getBytes()); + ArrayList> tasks = new ArrayList<>(); + for (Path p : Files.walk(path).filter(Files::isRegularFile).collect(Collectors.toList())) { + RecursiveTask task = new RecursiveTask() { + @Override + protected byte[] compute() { + try { + return getHashFromPath(p); + } catch (Exception e) { + } + return null; + } + }; + tasks.add(task); + pool.execute(task); + } + + for (RecursiveTask task : tasks) { + try { + byte[] hash = task.get(); + if (hash == null) { + throw new IOException("Unable to compute"); + } + md.update(hash); + } catch (Exception e) { + throw new IOException("ForkJoinPool exception; unable to compute."); + } + } + } + + private void processFile(Path path, MessageDigest md) throws IOException { + try (DigestInputStream stream = new DigestInputStream(Files.newInputStream(path), md)) { + byte[] buf = new byte[BUFFER_SIZE]; + while (stream.read(buf) != -1) { + ; + } + } + } + + +} diff --git a/cw1/src/main/java/ru/spbau/group202/cw1/MD5SingleThread.java b/cw1/src/main/java/ru/spbau/group202/cw1/MD5SingleThread.java new file mode 100644 index 0000000..8241680 --- /dev/null +++ b/cw1/src/main/java/ru/spbau/group202/cw1/MD5SingleThread.java @@ -0,0 +1,42 @@ +package ru.spbau.group202.cw1; + +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.stream.Collectors; + +/** + * This implementation of MD5Hasher abstract class calculates MD5 check-sum + * in a regular way, using a single thread. + */ +public class MD5SingleThread extends MD5Hasher { + + /** + * {@inheritDoc} + */ + @NotNull + public byte[] getHashFromPath(@NotNull Path path) throws IOException, NoSuchAlgorithmException { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + if (Files.isDirectory(path)) { + md5.update(path.getFileName().toString().getBytes()); + for (Path p : Files.walk(path).filter(Files::isRegularFile).collect(Collectors.toList())) { + md5.update(getHashFromPath(p)); + } + + return md5.digest(); + } + + try (DigestInputStream stream = new DigestInputStream(Files.newInputStream(path), md5)) { + byte[] buf = new byte[BUFFER_SIZE]; + while (stream.read(buf) != -1) { + ; + } + return stream.getMessageDigest().digest(); + } + } +} diff --git a/cw1/src/main/java/ru/spbau/group202/cw1/Main.java b/cw1/src/main/java/ru/spbau/group202/cw1/Main.java new file mode 100644 index 0000000..560bcd7 --- /dev/null +++ b/cw1/src/main/java/ru/spbau/group202/cw1/Main.java @@ -0,0 +1,36 @@ +package ru.spbau.group202.cw1; + +import javax.xml.bind.DatatypeConverter; +import java.nio.file.Paths; + +public class Main { + public static void main(String[] args) { + if (args.length < 1) { + System.out.println("Incorrect usage; specify filepaths."); + return; + } + + MD5MultiThread concurrentHasher = new MD5MultiThread(4); + MD5SingleThread regularHasher = new MD5SingleThread(); + for (String path : args) { + System.out.println(path + ":"); + try { + long start = System.currentTimeMillis(); + byte[] res = regularHasher.getHashFromPath(Paths.get(path)); + System.out.println(DatatypeConverter.printHexBinary(res)); + System.out.println("Single-thread computation: " + (System.currentTimeMillis() - start) + " ms."); + + start = System.currentTimeMillis(); + res = concurrentHasher.getHashFromPath(Paths.get(path)); + System.out.println(DatatypeConverter.printHexBinary(res)); + System.out.println("Multi-thread computation: " + (System.currentTimeMillis() - start) + " ms."); + } catch (Exception e) { + if (e.getMessage() != null) { + System.out.println(e.getMessage()); + } else { + System.out.println("Unable to compute md5 of this file"); + } + } + } + } +} diff --git a/cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java b/cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java new file mode 100644 index 0000000..f86fbd4 --- /dev/null +++ b/cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java @@ -0,0 +1,98 @@ +package ru.spbau.group202.cw1; + +import org.junit.Test; + +import javax.xml.bind.DatatypeConverter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.NoSuchAlgorithmException; + +import static org.junit.Assert.*; + +// IMPORTANT: neither of those tests pass because of incorrect paths +// but all of them have passed manual testing (Main with args) + +/** + * This class tests correctness of MD5Hasher implementations methods + */ +public class MD5HasherTest { + + private final static int numberOfThreads = 4; + + private void testEmptyDir(MD5Hasher hasher) throws NoSuchAlgorithmException, IOException { + // TODO this is currently not working + // but this is how I'd implement the tests if I had time to figure out + // how to get this test to believe the provided path exists + Path path = Paths.get("/src/test/resources/testdir/dir"); + byte[] hash = hasher.getHashFromPath(path); + String str = DatatypeConverter.printHexBinary(hash); + // this md5 checksum was calculated here: http://progs.be/md5.html + assertEquals("736007832d2167baaae763fd3a3f3cf1", str.toLowerCase()); + } + + private void testEmptyFile(MD5Hasher hasher) throws NoSuchAlgorithmException, IOException { + Path path = Paths.get(this.getClass().getResource("/testdir/file2.txt").toString()); + byte[] hash = hasher.getHashFromPath(path); + String str = DatatypeConverter.printHexBinary(hash); + // this md5 checksum was calculated here: http://onlinemd5.com/ + assertEquals("D41D8CD98F00B204E9800998ECF8427E", str); + } + + private void testNonEmptyFile(MD5Hasher hasher) throws NoSuchAlgorithmException, IOException { + Path path = Paths.get(this.getClass().getResource("/testdir/file1.txt").toString()); + byte[] hash = hasher.getHashFromPath(path); + String str = DatatypeConverter.printHexBinary(hash); + // this md5 checksum was calculated here: http://onlinemd5.com/ + assertEquals("40A5D58FFA6E88AA578D6683AC413105", str); + } + + @Test + public void testEmptyDirRegularHasher() throws NoSuchAlgorithmException, IOException { + MD5SingleThread hasher = new MD5SingleThread(); + testEmptyDir(hasher); + } + + @Test + public void testEmptyDirConcurrentHasher() throws NoSuchAlgorithmException, IOException { + MD5MultiThread hasher = new MD5MultiThread(numberOfThreads); + testEmptyDir(hasher); + } + + @Test + public void testEmptyFileRegularHasher() throws NoSuchAlgorithmException, IOException { + MD5SingleThread hasher = new MD5SingleThread(); + testEmptyFile(hasher); + } + + @Test + public void testEmptyFileConcurrentHasher() throws NoSuchAlgorithmException, IOException { + MD5MultiThread hasher = new MD5MultiThread(numberOfThreads); + testEmptyFile(hasher); + } + + @Test + public void testNonEmptyFileRegularHasher() throws NoSuchAlgorithmException, IOException { + MD5SingleThread hasher = new MD5SingleThread(); + testNonEmptyFile(hasher); + } + + @Test + public void testNonEmptyFileConcurrentHasher() throws NoSuchAlgorithmException, IOException { + MD5MultiThread hasher = new MD5MultiThread(numberOfThreads); + testNonEmptyFile(hasher); + } + + @Test + public void testEqualResultsSingleAndMultiThread() throws NoSuchAlgorithmException, IOException { + MD5SingleThread regularHasher = new MD5SingleThread(); + MD5MultiThread concurrentHasher = new MD5MultiThread(numberOfThreads); + + Path path = Paths.get("/src/test/resources/testdir"); + byte[] hash = regularHasher.getHashFromPath(path); + String str1 = DatatypeConverter.printHexBinary(hash); + hash = concurrentHasher.getHashFromPath(path); + String str2 = DatatypeConverter.printHexBinary(hash); + assertEquals(str1, str2); + } +} \ No newline at end of file diff --git a/cw1/src/test/resources/testdir/file1.txt b/cw1/src/test/resources/testdir/file1.txt new file mode 100644 index 0000000..3881968 --- /dev/null +++ b/cw1/src/test/resources/testdir/file1.txt @@ -0,0 +1 @@ +filefile \ No newline at end of file diff --git a/cw1/src/test/resources/testdir/file2.txt b/cw1/src/test/resources/testdir/file2.txt new file mode 100644 index 0000000..e69de29 From cb55814e1cbad7833302c26e88e1136d8dc5e57f Mon Sep 17 00:00:00 2001 From: sharkovadarya Date: Fri, 13 Apr 2018 17:55:30 +0300 Subject: [PATCH 2/5] added important comments to tests --- cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java b/cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java index f86fbd4..50267b9 100644 --- a/cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java +++ b/cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java @@ -13,6 +13,9 @@ // IMPORTANT: neither of those tests pass because of incorrect paths // but all of them have passed manual testing (Main with args) +// IMPORTANT: testEmptyDir will only work if you create 'dir' directory manually +// git doesn't allow empty directories to be commited + /** * This class tests correctness of MD5Hasher implementations methods */ @@ -20,6 +23,7 @@ public class MD5HasherTest { private final static int numberOfThreads = 4; + // create 'dir' directory manually! private void testEmptyDir(MD5Hasher hasher) throws NoSuchAlgorithmException, IOException { // TODO this is currently not working // but this is how I'd implement the tests if I had time to figure out From c5964f80964982450d2aeb303f66eefabc39bdbd Mon Sep 17 00:00:00 2001 From: sharkovadarya Date: Fri, 13 Apr 2018 18:01:36 +0300 Subject: [PATCH 3/5] fixed tests --- .../ru/spbau/group202/cw1/MD5HasherTest.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java b/cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java index 50267b9..b84f6d9 100644 --- a/cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java +++ b/cw1/src/test/java/ru/spbau/group202/cw1/MD5HasherTest.java @@ -3,18 +3,15 @@ import org.junit.Test; import javax.xml.bind.DatatypeConverter; +import java.io.File; import java.io.IOException; import java.nio.file.Path; -import java.nio.file.Paths; import java.security.NoSuchAlgorithmException; import static org.junit.Assert.*; -// IMPORTANT: neither of those tests pass because of incorrect paths -// but all of them have passed manual testing (Main with args) - // IMPORTANT: testEmptyDir will only work if you create 'dir' directory manually -// git doesn't allow empty directories to be commited +// git doesn't allow empty directories to be committed /** * This class tests correctness of MD5Hasher implementations methods @@ -25,10 +22,8 @@ public class MD5HasherTest { // create 'dir' directory manually! private void testEmptyDir(MD5Hasher hasher) throws NoSuchAlgorithmException, IOException { - // TODO this is currently not working - // but this is how I'd implement the tests if I had time to figure out - // how to get this test to believe the provided path exists - Path path = Paths.get("/src/test/resources/testdir/dir"); + File file = new File("src/test/resources/testdir/dir"); + Path path = file.toPath(); byte[] hash = hasher.getHashFromPath(path); String str = DatatypeConverter.printHexBinary(hash); // this md5 checksum was calculated here: http://progs.be/md5.html @@ -36,7 +31,8 @@ private void testEmptyDir(MD5Hasher hasher) throws NoSuchAlgorithmException, IOE } private void testEmptyFile(MD5Hasher hasher) throws NoSuchAlgorithmException, IOException { - Path path = Paths.get(this.getClass().getResource("/testdir/file2.txt").toString()); + File file = new File("src/test/resources/testdir/file2.txt"); + Path path = file.toPath(); byte[] hash = hasher.getHashFromPath(path); String str = DatatypeConverter.printHexBinary(hash); // this md5 checksum was calculated here: http://onlinemd5.com/ @@ -44,7 +40,8 @@ private void testEmptyFile(MD5Hasher hasher) throws NoSuchAlgorithmException, IO } private void testNonEmptyFile(MD5Hasher hasher) throws NoSuchAlgorithmException, IOException { - Path path = Paths.get(this.getClass().getResource("/testdir/file1.txt").toString()); + File file = new File("src/test/resources/testdir/file1.txt"); + Path path = file.toPath(); byte[] hash = hasher.getHashFromPath(path); String str = DatatypeConverter.printHexBinary(hash); // this md5 checksum was calculated here: http://onlinemd5.com/ @@ -92,7 +89,8 @@ public void testEqualResultsSingleAndMultiThread() throws NoSuchAlgorithmExcepti MD5SingleThread regularHasher = new MD5SingleThread(); MD5MultiThread concurrentHasher = new MD5MultiThread(numberOfThreads); - Path path = Paths.get("/src/test/resources/testdir"); + File file = new File("src/test/resources/testdir"); + Path path = file.toPath(); byte[] hash = regularHasher.getHashFromPath(path); String str1 = DatatypeConverter.printHexBinary(hash); hash = concurrentHasher.getHashFromPath(path); From 1593d44afcddfd8a4f91ec1e0c93e1b0322e6088 Mon Sep 17 00:00:00 2001 From: sharkovadarya Date: Fri, 13 Apr 2018 18:10:29 +0300 Subject: [PATCH 4/5] Added annotations and minor fixes --- cw1/src/main/java/ru/spbau/group202/cw1/MD5MultiThread.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cw1/src/main/java/ru/spbau/group202/cw1/MD5MultiThread.java b/cw1/src/main/java/ru/spbau/group202/cw1/MD5MultiThread.java index 5fa00b5..44f34dc 100644 --- a/cw1/src/main/java/ru/spbau/group202/cw1/MD5MultiThread.java +++ b/cw1/src/main/java/ru/spbau/group202/cw1/MD5MultiThread.java @@ -40,7 +40,7 @@ public byte[] getHashFromPath(@NotNull Path path) throws IOException, NoSuchAlgo return md5.digest(); } - private void processDirectory(Path path, MessageDigest md) throws IOException { + private void processDirectory(@NotNull Path path, @NotNull MessageDigest md) throws IOException { md.update(path.getFileName().toString().getBytes()); ArrayList> tasks = new ArrayList<>(); for (Path p : Files.walk(path).filter(Files::isRegularFile).collect(Collectors.toList())) { @@ -50,6 +50,7 @@ protected byte[] compute() { try { return getHashFromPath(p); } catch (Exception e) { + // TODO handle exceptions } return null; } @@ -71,7 +72,7 @@ protected byte[] compute() { } } - private void processFile(Path path, MessageDigest md) throws IOException { + private void processFile(@NotNull Path path, @NotNull MessageDigest md) throws IOException { try (DigestInputStream stream = new DigestInputStream(Files.newInputStream(path), md)) { byte[] buf = new byte[BUFFER_SIZE]; while (stream.read(buf) != -1) { From c16c975b5f75bb016f85dc2814cb99059703bece Mon Sep 17 00:00:00 2001 From: sharkovadarya Date: Fri, 13 Apr 2018 18:15:10 +0300 Subject: [PATCH 5/5] Updated Main --- cw1/src/main/java/ru/spbau/group202/cw1/Main.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cw1/src/main/java/ru/spbau/group202/cw1/Main.java b/cw1/src/main/java/ru/spbau/group202/cw1/Main.java index 560bcd7..460eaf3 100644 --- a/cw1/src/main/java/ru/spbau/group202/cw1/Main.java +++ b/cw1/src/main/java/ru/spbau/group202/cw1/Main.java @@ -18,12 +18,16 @@ public static void main(String[] args) { long start = System.currentTimeMillis(); byte[] res = regularHasher.getHashFromPath(Paths.get(path)); System.out.println(DatatypeConverter.printHexBinary(res)); - System.out.println("Single-thread computation: " + (System.currentTimeMillis() - start) + " ms."); + long diff1 = System.currentTimeMillis() - start; + System.out.println("Single-thread computation: " + diff1 + " ms."); start = System.currentTimeMillis(); res = concurrentHasher.getHashFromPath(Paths.get(path)); System.out.println(DatatypeConverter.printHexBinary(res)); - System.out.println("Multi-thread computation: " + (System.currentTimeMillis() - start) + " ms."); + long diff2 = System.currentTimeMillis() - start; + System.out.println("Multi-thread computation: " + diff2 + " ms."); + + System.out.println("\n The difference is " + (diff1 - diff2) + " ms.\n"); } catch (Exception e) { if (e.getMessage() != null) { System.out.println(e.getMessage());