From 2670f8eaf585f33cb7bf85fc0245c09ba859f31d Mon Sep 17 00:00:00 2001 From: John Martel Date: Tue, 21 Feb 2017 10:03:47 -0500 Subject: [PATCH 01/59] Ignore IntelliJ files in git --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 97d0478..e804f0f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ target .project log .DS_Store -coverage-test.xml \ No newline at end of file +coverage-test.xml +.idea/ +ais-coverage.iml From eb520b7ead50277c203f4305a690db9066c46b72 Mon Sep 17 00:00:00 2001 From: John Martel Date: Wed, 22 Feb 2017 19:56:31 -0500 Subject: [PATCH 02/59] Fix NPE on getting status when no message received Any `GET` request to `/coverage/rest/status` would fail with an NPE until a first message was received, preventing usage such as dragging the map around. --- pom.xml | 6 ++ .../coverage/rest/CoverageRestService.java | 88 ++++++++++--------- .../dma/ais/coverage/AisCoverageBuilder.java | 43 +++++++++ .../rest/CoverageRestServiceTest.java | 77 ++++++++++++++++ 4 files changed, 172 insertions(+), 42 deletions(-) create mode 100644 src/test/java/dk/dma/ais/coverage/AisCoverageBuilder.java create mode 100644 src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java diff --git a/pom.xml b/pom.xml index cf9bba6..f71db34 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,12 @@ jersey-json 1.17 + + org.mockito + mockito-core + 2.7.11 + test + diff --git a/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java b/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java index ab9ae0f..9ea1d81 100644 --- a/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java +++ b/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java @@ -14,51 +14,31 @@ */ package dk.dma.ais.coverage.rest; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import dk.dma.ais.coverage.AisCoverage; +import dk.dma.ais.coverage.CoverageHandler; +import dk.dma.ais.coverage.Helper; +import dk.dma.ais.coverage.calculator.TerrestrialCalculator; +import dk.dma.ais.coverage.data.*; +import dk.dma.ais.coverage.export.data.*; +import dk.dma.ais.coverage.export.generators.ChartGenerator; +import dk.dma.ais.coverage.export.generators.KMLGenerator; +import dk.dma.ais.coverage.export.generators.XMLGenerator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.UriInfo; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.*; //import dk.dma.ais.coverage.export.CSVGenerator; -import dk.dma.ais.coverage.AisCoverage; -import dk.dma.ais.coverage.CoverageHandler; -import dk.dma.ais.coverage.Helper; -import dk.dma.ais.coverage.calculator.TerrestrialCalculator; -import dk.dma.ais.coverage.data.Cell; -import dk.dma.ais.coverage.data.ICoverageData; -import dk.dma.ais.coverage.data.OnlyMemoryData; -import dk.dma.ais.coverage.data.Source; -import dk.dma.ais.coverage.data.TimeSpan; -import dk.dma.ais.coverage.export.data.ExportShipTimeSpan; -import dk.dma.ais.coverage.export.data.JSonCoverageMap; -import dk.dma.ais.coverage.export.data.JsonConverter; -import dk.dma.ais.coverage.export.data.JsonSource; -import dk.dma.ais.coverage.export.data.Status; -import dk.dma.ais.coverage.export.generators.ChartGenerator; -import dk.dma.ais.coverage.export.generators.KMLGenerator; -import dk.dma.ais.coverage.export.generators.XMLGenerator; /** * JAX-RS rest services @@ -425,15 +405,39 @@ public Object satExport(@QueryParam("test") String test, @Context HttpServletRes @Produces(MediaType.APPLICATION_JSON) public Object status() throws IOException { LOG.info("getting status"); - Date first = Helper.firstMessage; - Date last = Helper.latestMessage; - Status s = new Status(); - s.firstMessage = first.getTime(); - s.lastMessage = last.getTime(); - s.analysisStatus = "Running"; - return s; + Status status = new Status(); + Date now = Helper.getFloorDate(new Date()); + + setFirstMessageTimestamp(status, now); + setLastMessageTimestamp(status, now); + + status.analysisStatus = "Running"; + + return status; + } + + private void setFirstMessageTimestamp(Status status, Date now) { + if (messageReceived()) { + status.firstMessage = Helper.firstMessage.getTime(); + } else { + status.firstMessage = now.getTime(); + } + } + private boolean messageReceived() { + return Helper.firstMessage != null; } + private void setLastMessageTimestamp(Status status, Date now) { + if (terrestrialMessageReceived()) { + status.lastMessage = Helper.latestMessage.getTime(); + } else { + status.lastMessage = now.getTime(); + } + } + + private boolean terrestrialMessageReceived() { + return Helper.latestMessage != null; + } } diff --git a/src/test/java/dk/dma/ais/coverage/AisCoverageBuilder.java b/src/test/java/dk/dma/ais/coverage/AisCoverageBuilder.java new file mode 100644 index 0000000..4dd5b82 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/AisCoverageBuilder.java @@ -0,0 +1,43 @@ +package dk.dma.ais.coverage; + +import dk.dma.ais.bus.AisBus; +import dk.dma.ais.configuration.bus.AisBusConfiguration; +import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; + +import java.util.Objects; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Builder class that builds test instances of {@link AisCoverage}. + */ +public class AisCoverageBuilder { + private AisCoverageConfiguration aisCoverageConfiguration; + + public AisCoverageBuilder withMockAisCoverageConfiguration() { + aisCoverageConfiguration = mock(AisCoverageConfiguration.class); + + return this; + } + + public AisCoverageBuilder withMockAisBusConfiguration() { + Objects.requireNonNull(aisCoverageConfiguration, "An AisCoverageConfiguration instance is required. Consider invoking #withMockAisCoverageConfiguration"); + + AisBus aisBus = mock(AisBus.class); + AisBusConfiguration aisBusConfiguration = mock(AisBusConfiguration.class); + when(aisBusConfiguration.getInstance()).thenReturn(aisBus); + when(aisCoverageConfiguration.getAisbusConfiguration()).thenReturn(aisBusConfiguration); + + return this; + } + + public AisCoverage build() { + if (aisCoverageConfiguration == null) { + withMockAisCoverageConfiguration(); + withMockAisBusConfiguration(); + } + + return AisCoverage.create(aisCoverageConfiguration); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java b/src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java new file mode 100644 index 0000000..d367c89 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java @@ -0,0 +1,77 @@ +package dk.dma.ais.coverage.rest; + +import dk.dma.ais.coverage.AisCoverageBuilder; +import dk.dma.ais.coverage.Helper; +import dk.dma.ais.coverage.export.data.Status; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Date; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertThat; + +/** + * Tests in this class are synchronized because they are all using {@code Helper.firstMessage} and + * {@code Helper.latestMessage}, which are static members. + */ +public class CoverageRestServiceTest { + private AisCoverageBuilder aisCoverageBuilder; + private CoverageRestService restService; + + @Before + public void setUp() throws Exception { + aisCoverageBuilder = new AisCoverageBuilder(); + aisCoverageBuilder.build(); + + restService = new CoverageRestService(); + } + + @After + public void tearDown() throws Exception { + Helper.firstMessage = null; + Helper.latestMessage = null; + } + + @Test + public synchronized void givenNoMessageReceivedYet_whenStatus_thenStatusRunningIsReturned() throws IOException { + Date now = Helper.getFloorDate(new Date()); + + Status status = (Status) restService.status(); + + assertThatStatusIsRunning(status); + assertThat(status.firstMessage, is(equalTo(now.getTime()))); + assertThat(status.lastMessage, is(equalTo(now.getTime()))); + } + + private void assertThatStatusIsRunning(Status status) { + assertThat(status, is(not(nullValue()))); + assertThat(status.analysisStatus, is(equalTo("Running"))); + } + + @Test + public synchronized void givenAFewMessagesReceived_whenStatus_thenStatusRunningIsReturned_andFirstMessageTimestampIsSet_andLatestMessageTimestampIsSet() throws IOException { + Helper.firstMessage = Helper.getFloorDate(new Date()); + Helper.latestMessage = Helper.getCeilDate(new Date()); + + Status status = (Status) restService.status(); + + assertThatStatusIsRunning(status); + assertThat(status.firstMessage, is(equalTo(Helper.firstMessage.getTime()))); + assertThat(status.lastMessage, is(equalTo(Helper.latestMessage.getTime()))); + } + + @Test + public synchronized void givenNoTerrestrialMessageReceivedYet_whenStatus_thenStatusRunningIsReturned_andFirstMessageTimestampIsSet() throws IOException { + Helper.firstMessage = Helper.getFloorDate(new Date()); + Date now = Helper.getFloorDate(new Date()); + + Status status = (Status) restService.status(); + + assertThatStatusIsRunning(status); + assertThat(status.firstMessage, is(equalTo(Helper.firstMessage.getTime()))); + assertThat(status.lastMessage, is(equalTo(now.getTime()))); + } +} From 7c54946a7c7c375b2c4c181b56d6dfaaa857e405 Mon Sep 17 00:00:00 2001 From: John Martel Date: Wed, 22 Feb 2017 19:59:12 -0500 Subject: [PATCH 03/59] Increment patch number This assumes [semantic versioning](http://semver.org/) is used. The pre-release identifier `CCG` means `Canadian Coast Guard`. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f71db34..c2a65d2 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ dk.dma.ais ais-coverage - 0.2-SNAPSHOT + 0.2.1-CCG AIS coverage analyzer AIS coverage analyzer From fbb0d7c1ea071c938e8dcfc734c736702f2673ec Mon Sep 17 00:00:00 2001 From: John Martel Date: Wed, 22 Feb 2017 20:08:10 -0500 Subject: [PATCH 04/59] Add Travis-CI configuration file See also: https://travis-ci.org/ --- .travis.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..dff5f3a --- /dev/null +++ b/.travis.yml @@ -0,0 +1 @@ +language: java From 2eae5b9aa01d67def2d44781c830d0a1f239530e Mon Sep 17 00:00:00 2001 From: John Martel Date: Wed, 22 Feb 2017 20:10:42 -0500 Subject: [PATCH 05/59] Instruct travis to build using java 8 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index dff5f3a..9bcf999 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1 +1,3 @@ language: java +jdk: + - oraclejdk8 From 2bbea2249089334a4048f219276a97c0ffeebcd6 Mon Sep 17 00:00:00 2001 From: John Martel Date: Thu, 23 Feb 2017 12:12:07 -0500 Subject: [PATCH 06/59] Store coverage data periodically to MongoDB See issue: [#140237535](https://www.pivotaltracker.com/story/show/140237535) --- pom.xml | 8 +- .../java/dk/dma/ais/coverage/AisCoverage.java | 49 +++-- .../dk/dma/ais/coverage/CoverageHandler.java | 64 +++---- .../configuration/DatabaseConfiguration.java | 9 + .../dma/ais/coverage/data/OnlyMemoryData.java | 5 +- .../DatabaseConnectionException.java | 11 ++ .../persistence/DatabaseInstance.java | 43 +++++ .../persistence/DatabaseInstanceFactory.java | 9 + .../MemoryOnlyDatabaseInstance.java | 32 ++++ .../persistence/MongoDatabaseInstance.java | 140 ++++++++++++++ .../persistence/PersistenceResult.java | 40 ++++ .../persistence/PersisterService.java | 77 ++++++++ .../TypeBasedDatabaseInstanceFactory.java | 19 ++ .../UnknownDatabaseTypeException.java | 11 ++ .../dma/ais/coverage/AisCoverageBuilder.java | 68 ++++++- .../dk/dma/ais/coverage/AisCoverageTest.java | 88 +++++++++ .../DatabaseConfigurationTest.java | 17 ++ .../MongoDatabaseInstanceTest.java | 173 ++++++++++++++++++ .../persistence/PersistenceResultTest.java | 26 +++ .../persistence/PersisterServiceTest.java | 41 +++++ .../TypeBasedDatabaseInstanceFactoryTest.java | 66 +++++++ .../rest/CoverageRestServiceTest.java | 23 ++- 22 files changed, 951 insertions(+), 68 deletions(-) create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/DatabaseConnectionException.java create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstanceFactory.java create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/PersistenceResult.java create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactory.java create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/UnknownDatabaseTypeException.java create mode 100644 src/test/java/dk/dma/ais/coverage/AisCoverageTest.java create mode 100644 src/test/java/dk/dma/ais/coverage/configuration/DatabaseConfigurationTest.java create mode 100644 src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java create mode 100644 src/test/java/dk/dma/ais/coverage/persistence/PersistenceResultTest.java create mode 100644 src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java create mode 100644 src/test/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactoryTest.java diff --git a/pom.xml b/pom.xml index c2a65d2..09e5182 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,7 @@ org.mongodb mongo-java-driver - 2.11.0 + 3.4.2 org.eclipse.jetty @@ -109,6 +109,12 @@ 2.7.11 test + + de.bwaldvogel + mongo-java-server + 1.6.0 + test + diff --git a/src/main/java/dk/dma/ais/coverage/AisCoverage.java b/src/main/java/dk/dma/ais/coverage/AisCoverage.java index f795a78..a523ade 100644 --- a/src/main/java/dk/dma/ais/coverage/AisCoverage.java +++ b/src/main/java/dk/dma/ais/coverage/AisCoverage.java @@ -16,18 +16,21 @@ import java.util.function.Consumer; -import net.jcip.annotations.GuardedBy; -import net.jcip.annotations.ThreadSafe; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import dk.dma.ais.bus.AisBus; import dk.dma.ais.bus.consumer.DistributerConsumer; import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; +import dk.dma.ais.coverage.persistence.DatabaseInstance; +import dk.dma.ais.coverage.persistence.DatabaseInstanceFactory; +import dk.dma.ais.coverage.persistence.PersisterService; +import dk.dma.ais.coverage.persistence.TypeBasedDatabaseInstanceFactory; import dk.dma.ais.coverage.web.WebServer; import dk.dma.ais.packet.AisPacket; import dk.dma.ais.reader.AisReader; +import net.jcip.annotations.GuardedBy; +import net.jcip.annotations.ThreadSafe; /** @@ -46,16 +49,20 @@ public final class AisCoverage { private final AisBus aisBus; private final WebServer webServer; private AisReader aisReader; + private PersisterService persisterService; + private final DatabaseInstance databaseInstance; - private AisCoverage(AisCoverageConfiguration conf) { + private AisCoverage(AisCoverageConfiguration conf, DatabaseInstanceFactory databaseInstanceFactory) { this.conf = conf; - // Create handler - handler = new CoverageHandler(conf); + databaseInstance = databaseInstanceFactory.createDatabaseInstance(conf.getDatabaseConfiguration().getType()); + databaseInstance.open(conf.getDatabaseConfiguration()); + databaseInstance.createDatabase(); - // Create AisBus + handler = new CoverageHandler(conf); aisBus = conf.getAisbusConfiguration().getInstance(); - + persisterService = new PersisterService(databaseInstance, handler.getDataHandler()); + persisterService.intervalInSeconds(conf.getDatabaseConfiguration().getPersistenceIntervalInSeconds()); // Create web server if (conf.getServerConfiguration() != null) { @@ -82,23 +89,23 @@ public void start() { aisBus.start(); aisBus.startConsumers(); aisBus.startProviders(); - LOG.info("aisbus startet"); + LOG.info("aisbus started"); } // Start web server if (webServer != null) { try { webServer.start(); - LOG.info("webserver startet"); + LOG.info("webserver started"); } catch (Exception e) { LOG.error("Failed to start web server: " + e.getMessage()); e.printStackTrace(); } } + + persisterService.start(); } public void stop() { - - // Start web server if (webServer != null) { try { webServer.stop(); @@ -109,9 +116,15 @@ public void stop() { } } - // Stop AisBus aisBus.cancel(); LOG.info("aisbus stopped"); + + persisterService.stop(); + try { + databaseInstance.close(); + } catch (Exception e) { + LOG.warn("Could not close DatabaseInstance cleanly", e); + } } public AisCoverageConfiguration getConf() { @@ -123,7 +136,12 @@ public CoverageHandler getHandler() { } public static synchronized AisCoverage create(AisCoverageConfiguration conf) { - instance = new AisCoverage(conf); + instance = new AisCoverage(conf, new TypeBasedDatabaseInstanceFactory()); + return instance; + } + + public static synchronized AisCoverage create(AisCoverageConfiguration conf, DatabaseInstanceFactory databaseInstanceFactory) { + instance = new AisCoverage(conf, databaseInstanceFactory); return instance; } @@ -131,4 +149,7 @@ public static synchronized AisCoverage get() { return instance; } + void setPersisterService(PersisterService persisterService) { + this.persisterService = persisterService; + } } diff --git a/src/main/java/dk/dma/ais/coverage/CoverageHandler.java b/src/main/java/dk/dma/ais/coverage/CoverageHandler.java index e1fa5cc..1ec5803 100644 --- a/src/main/java/dk/dma/ais/coverage/CoverageHandler.java +++ b/src/main/java/dk/dma/ais/coverage/CoverageHandler.java @@ -17,7 +17,9 @@ import java.util.ArrayList; import java.util.Date; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,28 +37,11 @@ * Handler for received AisPackets */ public class CoverageHandler { - private static final Logger LOG = LoggerFactory.getLogger(CoverageHandler.class); - //list of calculators - private ArrayList calculators = new ArrayList(); - public ArrayList getCalculators() { - return calculators; - } - - public void setCalculators(ArrayList calculators) { - this.calculators = calculators; - } - + private List calculators = new ArrayList(); private ICoverageData dataHandler; - - public ICoverageData getDataHandler() { - return dataHandler; - } - - public void setDataHandler(ICoverageData dataHandler) { - this.dataHandler = dataHandler; - } + private AisCoverageConfiguration conf; //A doublet filtered message buffer, where a custom message will include a list of all sources private LinkedHashMap doubletBuffer = new LinkedHashMap() { @@ -68,7 +53,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { return this.size() > 10000; } }; - + //Fields used for debugging purposes private int unfiltCount; private long biggestDelay; @@ -76,39 +61,52 @@ protected boolean removeEldestEntry(Map.Entry eldest) { private int delayedMoreThanTen; private int delayedLessThanTen; - private AisCoverageConfiguration conf; - public CoverageHandler(AisCoverageConfiguration conf) { this.conf=conf; Helper.conf=conf; - + //Creating up data handler dataHandler = new OnlyMemoryData(); LOG.info("coverage calculators set up with memory only data handling"); - + //creating calculators calculators.add(new TerrestrialCalculator(false)); calculators.add(new SatCalculator()); - + for (AbstractCalculator calc : calculators) { calc.setDataHandler(dataHandler); } // Logging grid granularity - LOG.info("grid granularity initiated with lat: " + conf.getLatSize() + " and lon: " + conf.getLonSize()); - + LOG.info("grid granularity initiated with lat: " + conf.getLatSize() + " and lon: " + conf.getLonSize()); + // One could set grid granularity based on meter scale and a latitude position like this - // Helper.setLatLonSize(meters, latitude); - + // Helper.setLatLonSize(meters, latitude); + if (conf.getVerbosityLevel() > 0) { verboseDebug(); } - + // window size LOG.info("Max window size is " + conf.getWindowSize()+" hours"); Purger purger = new Purger(conf.getWindowSize(), dataHandler, 5); purger.start(); + } + + public List getCalculators() { + return calculators; + } + + public void setCalculators(List calculators) { + this.calculators = calculators; + } + + public ICoverageData getDataHandler() { + return dataHandler; + } + public void setDataHandler(ICoverageData dataHandler) { + this.dataHandler = dataHandler; } public void receiveUnfiltered(AisPacket packet) { @@ -132,13 +130,12 @@ public void receiveUnfiltered(AisPacket packet) { } - private void process(CustomMessage m){ + void process(CustomMessage m){ for (AbstractCalculator calc : calculators) { calc.calculate(m); } } - - + public void verboseDebug(){ final Date then = new Date(); Thread t = new Thread(new Runnable() { @@ -189,5 +186,4 @@ public void run() { public SatCalculator getSatCalc() { return (SatCalculator) calculators.get(1); } - } diff --git a/src/main/java/dk/dma/ais/coverage/configuration/DatabaseConfiguration.java b/src/main/java/dk/dma/ais/coverage/configuration/DatabaseConfiguration.java index 5679544..1d9aaee 100644 --- a/src/main/java/dk/dma/ais/coverage/configuration/DatabaseConfiguration.java +++ b/src/main/java/dk/dma/ais/coverage/configuration/DatabaseConfiguration.java @@ -19,6 +19,7 @@ public class DatabaseConfiguration { private String dbName = "nordicCoverage"; private String addr = "localhost"; private int port = 5000; + private int persistenceIntervalInSeconds = 60; public String getType() { return type; @@ -51,4 +52,12 @@ public int getPort() { public void setPort(int port) { this.port = port; } + + public int getPersistenceIntervalInSeconds() { + return persistenceIntervalInSeconds; + } + + public void setPersistenceIntervalInSeconds(int persistenceIntervalInSeconds) { + this.persistenceIntervalInSeconds = persistenceIntervalInSeconds; + } } diff --git a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java index ec65633..0414dd2 100644 --- a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java +++ b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java @@ -29,7 +29,6 @@ import dk.dma.ais.coverage.AisCoverage; import dk.dma.ais.coverage.Helper; -import dk.dma.ais.coverage.Purger; import dk.dma.ais.coverage.calculator.AbstractCalculator; import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; import dk.dma.ais.coverage.data.Ship.ShipClass; @@ -198,7 +197,7 @@ public void incrementReceivedSignals(String sourceMmsi, double lat, cell.getFixedWidthSpans().put(id.getTime(), ts); } ts.setMessageCounterTerrestrial(ts.getMessageCounterTerrestrial() + 1); - + cell.incrementNOofReceivedSignals(); } @Override @@ -217,7 +216,7 @@ public void incrementMissingSignals(String sourceMmsi, double lat, cell.getFixedWidthSpans().put(id.getTime(), ts); } ts.incrementMissingSignals(); - + cell.incrementNOofMissingSignals(); } public CustomMessage packetToCustomMessage(AisPacket packet) { diff --git a/src/main/java/dk/dma/ais/coverage/persistence/DatabaseConnectionException.java b/src/main/java/dk/dma/ais/coverage/persistence/DatabaseConnectionException.java new file mode 100644 index 0000000..f5ae737 --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/DatabaseConnectionException.java @@ -0,0 +1,11 @@ +package dk.dma.ais.coverage.persistence; + +/** + * Indicates that the system could not connect to a given database. + */ +public class DatabaseConnectionException extends RuntimeException { + + public DatabaseConnectionException(String errorMessage, Throwable cause) { + super(errorMessage, cause); + } +} diff --git a/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java b/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java new file mode 100644 index 0000000..9f7eec7 --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java @@ -0,0 +1,43 @@ +package dk.dma.ais.coverage.persistence; + +import java.util.List; + +import dk.dma.ais.coverage.configuration.DatabaseConfiguration; +import dk.dma.ais.coverage.data.Cell; + +/** + * Data layer abstraction. Implementations care for the details of each operation for a specific database type. + */ +public interface DatabaseInstance extends AutoCloseable { + + /** + * Opens the connection to the database based on the provided configuration. + * Any opened instance should be closed using {@link #close()}. + * + * @param configuration + * the database configuration parameters + * @throws DatabaseConnectionException + * when connection to the database server is impossible for any reason + */ + void open(DatabaseConfiguration configuration); + + /** + * Creates the database where coverage data is collected if required. + * + * @throws DatabaseConnectionException + * when connection to the database server is impossible for any reason + */ + void createDatabase(); + + /** + * Saves the coverage data stored in in-memory cells to the underlying database. + * + * @param coverageData + * the coverage data stored in a {@link List} of {@link Cell}, as provided by {@link dk.dma.ais.coverage.data.ICoverageData} + * @return + * the result of the save operation + * @throws DatabaseConnectionException + * when connection to the database server is impossible for any reason + */ + PersistenceResult save(List coverageData); +} diff --git a/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstanceFactory.java b/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstanceFactory.java new file mode 100644 index 0000000..c51f79f --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstanceFactory.java @@ -0,0 +1,9 @@ +package dk.dma.ais.coverage.persistence; + +/** + * Implementations of this factory type create new {@link DatabaseInstance} implementations instances. + */ +public interface DatabaseInstanceFactory { + + DatabaseInstance createDatabaseInstance(String databaseType); +} diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java b/src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java new file mode 100644 index 0000000..8d09608 --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java @@ -0,0 +1,32 @@ +package dk.dma.ais.coverage.persistence; + +import java.util.List; + +import dk.dma.ais.coverage.configuration.DatabaseConfiguration; +import dk.dma.ais.coverage.data.Cell; + +/** + * {@link DatabaseInstance} implementation that keeps data in memory. Data is never persisted anywhere. + */ +class MemoryOnlyDatabaseInstance implements DatabaseInstance { + + @Override + public void open(DatabaseConfiguration configuration) { + + } + + @Override + public void createDatabase() { + + } + + @Override + public PersistenceResult save(List coverageData) { + return null; + } + + @Override + public void close() throws Exception { + + } +} diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java b/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java new file mode 100644 index 0000000..dc0153c --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java @@ -0,0 +1,140 @@ +package dk.dma.ais.coverage.persistence; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.bson.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import com.mongodb.MongoException; +import com.mongodb.ServerAddress; +import com.mongodb.client.MongoDatabase; +import dk.dma.ais.coverage.configuration.DatabaseConfiguration; +import dk.dma.ais.coverage.data.Cell; + +/** + * {@link DatabaseInstance} implementation that keeps data in a MongoDB database. + */ +class MongoDatabaseInstance implements DatabaseInstance { + private static final Logger LOG = LoggerFactory.getLogger(MongoDatabaseInstance.class); + private static final String COVERAGE_DATA = "coverageData"; + + private String mongoServerHost; + private int mongoServerPort; + private String databaseName; + private MongoClientOptions mongoClientOptions = MongoClientOptions.builder().build(); + private MongoClient client; + + @Override + public void open(DatabaseConfiguration configuration) { + mongoServerHost = configuration.getAddr(); + mongoServerPort = configuration.getPort(); + databaseName = configuration.getDbName(); + + LOG.info("Establishing connection with MongoDB database"); + + try { + client = connectToMongoServer(); + LOG.info("Connection established with MongoDB database"); + } catch (MongoException e) { + logAndTransformException(e); + } + } + + private MongoClient connectToMongoServer() { + ServerAddress serverAddress = new ServerAddress(mongoServerHost, mongoServerPort); + return new MongoClient(serverAddress, mongoClientOptions); + } + + private void logAndTransformException(Exception cause) throws DatabaseConnectionException { + String errorMessage = String.format("Could not connect to MongoDB database [%s] at [%s:%d]", databaseName, mongoServerHost, mongoServerPort); + LOG.error(errorMessage, cause); + throw new DatabaseConnectionException(errorMessage, cause); + } + + @Override + public void close() throws Exception { + if (client != null) { + LOG.info("Closing connection with MongoDB database"); + client.close(); + LOG.info("Connection with MongoDB database closed"); + } + } + + @Override + public void createDatabase() { + requireOpenConnection(); + + try { + createCoverageDataCollection(); + } catch (MongoException e) { + logAndTransformException(e); + } + } + + private void requireOpenConnection() { + if (client == null) { + throw new IllegalStateException("DatabaseInstance must be opened before executing operations: #open() should be invoked first."); + } + } + + private void createCoverageDataCollection() { + MongoDatabase database = client.getDatabase(databaseName); + if (!collectionExists(database)) { + database.createCollection(COVERAGE_DATA); + } + } + + private boolean collectionExists(MongoDatabase database) { + for (String collectionName : database.listCollectionNames()) { + if (COVERAGE_DATA.equalsIgnoreCase(collectionName)) { + return true; + } + } + + return false; + } + + @Override + public PersistenceResult save(List coverageData) { + requireOpenConnection(); + + Document coverageDataDocument = new Document(); + long numberOfSavedCells = 0; + coverageDataDocument.put("dataTimestamp", ZonedDateTime.now(ZoneId.of("UTC")).format(DateTimeFormatter.ISO_DATE_TIME)); + List> grid = new ArrayList<>(); + for (Cell cell : coverageData) { + Map savedCell = new LinkedHashMap<>(); + savedCell.put("cellId", cell.getId()); + savedCell.put("latitude", cell.getLatitude()); + savedCell.put("longitude", cell.getLongitude()); + savedCell.put("numberOfReceivedSignals", cell.getNOofReceivedSignals()); + savedCell.put("numberOfMissedSignals", cell.getNOofMissingSignals()); + grid.add(savedCell); + + numberOfSavedCells++; + } + coverageDataDocument.put("cells", grid); + + try { + client.getDatabase(databaseName).getCollection(COVERAGE_DATA).insertOne(coverageDataDocument); + return PersistenceResult.success(numberOfSavedCells); + } catch (MongoException e) { + logAndTransformException(e); + } + + return PersistenceResult.failure(); + } + + void setMongoClientOptions(MongoClientOptions mongoClientOptions) { + this.mongoClientOptions = mongoClientOptions; + } +} diff --git a/src/main/java/dk/dma/ais/coverage/persistence/PersistenceResult.java b/src/main/java/dk/dma/ais/coverage/persistence/PersistenceResult.java new file mode 100644 index 0000000..1f5d82b --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/PersistenceResult.java @@ -0,0 +1,40 @@ +package dk.dma.ais.coverage.persistence; + +/** + * Represents the result of saving a list of cells to a database. + */ +public class PersistenceResult { + + public enum Status { + SUCCESS, FAILURE + } + + public static PersistenceResult success(long writtenCells) { + PersistenceResult result = new PersistenceResult(); + result.status = Status.SUCCESS; + result.writtenCells = writtenCells; + + return result; + } + + public static PersistenceResult failure() { + PersistenceResult result = new PersistenceResult(); + result.status = Status.FAILURE; + + return result; + } + + private Status status; + private long writtenCells; + + private PersistenceResult() { + } + + public Status getStatus() { + return status; + } + + public long getWrittenCells() { + return writtenCells; + } +} diff --git a/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java new file mode 100644 index 0000000..14ad8a1 --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java @@ -0,0 +1,77 @@ +package dk.dma.ais.coverage.persistence; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import dk.dma.ais.coverage.data.ICoverageData; + +/** + * Performs asynchronous save operation to configured database. + */ +public class PersisterService { + private static final Logger LOG = LoggerFactory.getLogger(PersisterService.class); + + private final DatabaseInstance databaseInstance; + private final ICoverageData coverageData; + private long persistenceIntervalInSeconds = 60; + private ScheduledExecutorService executor; + + public PersisterService(DatabaseInstance databaseInstance, ICoverageData coverageData) { + this.databaseInstance = databaseInstance; + this.coverageData = coverageData; + } + + public void start() { + LOG.info("Starting PersisterService, persisting every [{}] seconds", persistenceIntervalInSeconds); + + executor = Executors.newScheduledThreadPool(1); + executor.scheduleAtFixedRate(new SaveOperation(), persistenceIntervalInSeconds, persistenceIntervalInSeconds, TimeUnit.SECONDS); + + LOG.info("PersisterService started"); + } + + public void stop() { + LOG.info("Stopping PersisterService"); + + executor.shutdown(); + try { + if (!executor.awaitTermination(persistenceIntervalInSeconds, TimeUnit.SECONDS)) { + executor.shutdownNow(); + if (!executor.awaitTermination(persistenceIntervalInSeconds, TimeUnit.SECONDS)) { + LOG.warn("PersisterService thread pool did not terminate cleanly"); + } + } + } catch (InterruptedException e) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + + LOG.info("PersisterService stopped"); + } + + public void intervalInSeconds(long persistenceIntervalInSeconds) { + this.persistenceIntervalInSeconds = persistenceIntervalInSeconds; + } + + long getIntervalInSeconds() { + return persistenceIntervalInSeconds; + } + + private class SaveOperation implements Runnable { + + @Override + public void run() { + PersistenceResult persistenceResult = databaseInstance.save(coverageData.getCells(null)); + + if (PersistenceResult.Status.SUCCESS.equals(persistenceResult.getStatus())) { + LOG.info("Saved [{}] cells to MongoDB database", persistenceResult.getWrittenCells()); + } else { + LOG.info("Failed saving cells to MongoDB database"); + } + } + } +} diff --git a/src/main/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactory.java b/src/main/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactory.java new file mode 100644 index 0000000..4699ec0 --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactory.java @@ -0,0 +1,19 @@ +package dk.dma.ais.coverage.persistence; + +/** + * This {@link DatabaseInstanceFactory} implementation supports creating {@link DatabaseInstance} instances for MongoDB + * or MemoryOnly. + */ +public class TypeBasedDatabaseInstanceFactory implements DatabaseInstanceFactory { + + @Override + public DatabaseInstance createDatabaseInstance(String databaseType) { + if ("MemoryOnly".equalsIgnoreCase(databaseType)) { + return new MemoryOnlyDatabaseInstance(); + } else if ("MongoDB".equalsIgnoreCase(databaseType)) { + return new MongoDatabaseInstance(); + } else { + throw new UnknownDatabaseTypeException(String.format("Unsupported database type: [%s]", databaseType)); + } + } +} diff --git a/src/main/java/dk/dma/ais/coverage/persistence/UnknownDatabaseTypeException.java b/src/main/java/dk/dma/ais/coverage/persistence/UnknownDatabaseTypeException.java new file mode 100644 index 0000000..e610c03 --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/UnknownDatabaseTypeException.java @@ -0,0 +1,11 @@ +package dk.dma.ais.coverage.persistence; + +/** + * Indicates that an unknown or unsupported database type has been specified. + */ +public class UnknownDatabaseTypeException extends RuntimeException { + + public UnknownDatabaseTypeException(String message) { + super(message); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/AisCoverageBuilder.java b/src/test/java/dk/dma/ais/coverage/AisCoverageBuilder.java index 4dd5b82..6ca1b51 100644 --- a/src/test/java/dk/dma/ais/coverage/AisCoverageBuilder.java +++ b/src/test/java/dk/dma/ais/coverage/AisCoverageBuilder.java @@ -1,19 +1,29 @@ package dk.dma.ais.coverage; -import dk.dma.ais.bus.AisBus; -import dk.dma.ais.configuration.bus.AisBusConfiguration; -import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.util.Objects; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import dk.dma.ais.bus.AisBus; +import dk.dma.ais.configuration.bus.AisBusConfiguration; +import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; +import dk.dma.ais.coverage.configuration.DatabaseConfiguration; +import dk.dma.ais.coverage.persistence.DatabaseInstance; +import dk.dma.ais.coverage.persistence.DatabaseInstanceFactory; +import dk.dma.ais.coverage.persistence.PersisterService; /** * Builder class that builds test instances of {@link AisCoverage}. */ public class AisCoverageBuilder { + private static final String MISSING_COVERAGE_CONFIGURATION_ERROR_MESSAGE = "An AisCoverageConfiguration instance is required. Consider invoking #withMockAisCoverageConfiguration"; + private AisCoverageConfiguration aisCoverageConfiguration; + private DatabaseInstanceFactory mockingDatabaseInstanceFactory; + private PersisterService persisterService; public AisCoverageBuilder withMockAisCoverageConfiguration() { aisCoverageConfiguration = mock(AisCoverageConfiguration.class); @@ -22,7 +32,7 @@ public AisCoverageBuilder withMockAisCoverageConfiguration() { } public AisCoverageBuilder withMockAisBusConfiguration() { - Objects.requireNonNull(aisCoverageConfiguration, "An AisCoverageConfiguration instance is required. Consider invoking #withMockAisCoverageConfiguration"); + Objects.requireNonNull(aisCoverageConfiguration, MISSING_COVERAGE_CONFIGURATION_ERROR_MESSAGE); AisBus aisBus = mock(AisBus.class); AisBusConfiguration aisBusConfiguration = mock(AisBusConfiguration.class); @@ -32,12 +42,56 @@ public AisCoverageBuilder withMockAisBusConfiguration() { return this; } + public AisCoverageBuilder withDatabaseConfiguration(DatabaseConfiguration databaseConfiguration) { + Objects.requireNonNull(aisCoverageConfiguration, MISSING_COVERAGE_CONFIGURATION_ERROR_MESSAGE); + + when(aisCoverageConfiguration.getDatabaseConfiguration()).thenReturn(databaseConfiguration); + + return this; + } + + public AisCoverageBuilder withMockDatabaseConfiguration() { + Objects.requireNonNull(aisCoverageConfiguration, MISSING_COVERAGE_CONFIGURATION_ERROR_MESSAGE); + + DatabaseConfiguration mockDatabaseConfiguration = new DatabaseConfiguration(); + when(aisCoverageConfiguration.getDatabaseConfiguration()).thenReturn(mockDatabaseConfiguration); + + return this; + } + + public AisCoverageBuilder withDatabaseInstanceForType(DatabaseInstance databaseInstance, String databaseType) { + mockingDatabaseInstanceFactory = mock(DatabaseInstanceFactory.class); + when(mockingDatabaseInstanceFactory.createDatabaseInstance(eq(databaseType))).thenReturn(databaseInstance); + + return this; + } + + public AisCoverageBuilder withMockDatabaseInstanceForAnyType() { + mockingDatabaseInstanceFactory = mock(DatabaseInstanceFactory.class); + DatabaseInstance mockDatabaseInstance = mock(DatabaseInstance.class); + when(mockingDatabaseInstanceFactory.createDatabaseInstance(anyString())).thenReturn(mockDatabaseInstance); + + return this; + } + + public AisCoverageBuilder withPersisterService(PersisterService persisterService) { + this.persisterService = persisterService; + + return this; + } + public AisCoverage build() { if (aisCoverageConfiguration == null) { withMockAisCoverageConfiguration(); withMockAisBusConfiguration(); } - return AisCoverage.create(aisCoverageConfiguration); + AisCoverage aisCoverage = AisCoverage.create(aisCoverageConfiguration, mockingDatabaseInstanceFactory); + + if (persisterService != null) { + aisCoverage.setPersisterService(persisterService); + } + + return aisCoverage; } } diff --git a/src/test/java/dk/dma/ais/coverage/AisCoverageTest.java b/src/test/java/dk/dma/ais/coverage/AisCoverageTest.java new file mode 100644 index 0000000..9402588 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/AisCoverageTest.java @@ -0,0 +1,88 @@ +package dk.dma.ais.coverage; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import org.junit.Test; + +import dk.dma.ais.coverage.configuration.DatabaseConfiguration; +import dk.dma.ais.coverage.persistence.DatabaseInstance; +import dk.dma.ais.coverage.persistence.PersisterService; + +public class AisCoverageTest { + + @Test + public void whenNewInstance_thenDatabaseIsCreated() { + DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(); + databaseConfiguration.setType("MongoDB"); + + DatabaseInstance mockDatabaseInstance = mock(DatabaseInstance.class); + + aisCoverageWithMockDatabaseInstance(databaseConfiguration, mockDatabaseInstance); + + verify(mockDatabaseInstance).open(databaseConfiguration); + verify(mockDatabaseInstance).createDatabase(); + } + + private AisCoverage aisCoverageWithMockDatabaseInstance(DatabaseConfiguration databaseConfiguration, DatabaseInstance mockDatabaseInstance) { + AisCoverageBuilder builder = new AisCoverageBuilder(); + builder.withMockAisCoverageConfiguration() + .withDatabaseConfiguration(databaseConfiguration) + .withMockAisBusConfiguration() + .withDatabaseInstanceForType(mockDatabaseInstance, "MongoDB"); + + return builder.build(); + } + + @Test + public void whenStart_thenPersisterServiceIsStarted() { + PersisterService persisterService = mock(PersisterService.class); + AisCoverage aisCoverage = aisCoverageWithMockPersisterService(persisterService); + + aisCoverage.start(); + + verify(persisterService).start(); + } + + private AisCoverage aisCoverageWithMockPersisterService(PersisterService persisterService) { + AisCoverageBuilder builder = new AisCoverageBuilder(); + return builder + .withMockAisCoverageConfiguration() + .withMockAisBusConfiguration() + .withMockDatabaseConfiguration() + .withMockDatabaseInstanceForAnyType() + .withPersisterService(persisterService).build(); + } + + @Test + public void whenStop_thenPersisterServiceIsStopped() { + PersisterService persisterService = mock(PersisterService.class); + AisCoverage aisCoverage = aisCoverageWithMockPersisterService(persisterService); + + aisCoverage.stop(); + + verify(persisterService).stop(); + } + + @Test + public void whenStop_thenDatabaseInstanceIsClosed() throws Exception { + DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(); + databaseConfiguration.setType("MongoDB"); + + DatabaseInstance mockDatabaseInstance = mock(DatabaseInstance.class); + + PersisterService persisterService = mock(PersisterService.class); + + AisCoverageBuilder builder = new AisCoverageBuilder(); + AisCoverage aisCoverage = builder + .withMockAisCoverageConfiguration() + .withMockAisBusConfiguration() + .withDatabaseConfiguration(databaseConfiguration) + .withDatabaseInstanceForType(mockDatabaseInstance, "MongoDB") + .withPersisterService(persisterService).build(); + + aisCoverage.stop(); + + verify(mockDatabaseInstance).close(); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/configuration/DatabaseConfigurationTest.java b/src/test/java/dk/dma/ais/coverage/configuration/DatabaseConfigurationTest.java new file mode 100644 index 0000000..0c850c3 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/configuration/DatabaseConfigurationTest.java @@ -0,0 +1,17 @@ +package dk.dma.ais.coverage.configuration; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +import org.junit.Test; + +public class DatabaseConfigurationTest { + + @Test + public void whenNewInstance_thenPersistenceIntervalInSecondsDefaultsTo60() { + DatabaseConfiguration configuration = new DatabaseConfiguration(); + + assertThat(configuration.getPersistenceIntervalInSeconds(), is(equalTo(60))); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java b/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java new file mode 100644 index 0000000..71e8511 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java @@ -0,0 +1,173 @@ +package dk.dma.ais.coverage.persistence; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertThat; + +import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.Collections; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import com.mongodb.ServerAddress; +import com.mongodb.client.MongoIterable; +import de.bwaldvogel.mongo.MongoServer; +import de.bwaldvogel.mongo.backend.memory.MemoryBackend; +import dk.dma.ais.coverage.configuration.DatabaseConfiguration; +import dk.dma.ais.coverage.data.Cell; + +public class MongoDatabaseInstanceTest { + private static final String NORDIC_COVERAGE_DATABASE = "nordicCoverage"; + private static final String COVERAGE_DATA_COLLECTION = "coverageData"; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private MongoServer mongoServer; + private DatabaseConfiguration configuration; + private MongoDatabaseInstance databaseInstance; + private InetSocketAddress serverAddress; + private MongoClient mongoClient; + + @Before + public void setUp() throws Exception { + databaseInstance = new MongoDatabaseInstance(); + } + + @After + public void tearDown() throws Exception { + databaseInstance.close(); + closeMongoClient(); + shutdownMongoServer(); + } + + private void closeMongoClient() { + if (mongoClient != null) { + mongoClient.close(); + } + } + + private void shutdownMongoServer() { + if (mongoServer != null) { + mongoServer.shutdown(); + } + } + + @Test + public void givenNoDatabase_whenCreateDatabase_thenDatabaseIsCreated() { + startMongoServer(); + useStartedMongoServer(); + + databaseInstance.createDatabase(); + + initializeMongoClient(); + MongoIterable existingDatabases = mongoClient.listDatabaseNames(); + assertThat(existingDatabases, hasItem(NORDIC_COVERAGE_DATABASE)); + assertThat(mongoClient.getDatabase(NORDIC_COVERAGE_DATABASE), is(notNullValue())); + } + + + private void startMongoServer() { + mongoServer = new MongoServer(new MemoryBackend()); + serverAddress = mongoServer.bind(); + } + + private void useStartedMongoServer() { + configuration = new DatabaseConfiguration(); + configuration.setType("MongoDB"); + configuration.setDbName(NORDIC_COVERAGE_DATABASE); + configuration.setAddr(serverAddress.getHostName()); + configuration.setPort(serverAddress.getPort()); + + databaseInstance.open(configuration); + } + + private void initializeMongoClient() { + MongoClientOptions options = MongoClientOptions.builder().serverSelectionTimeout(10).build(); + mongoClient = new MongoClient(new ServerAddress(serverAddress), options); + } + + @Test + public void givenAnExistingDatabase_whenCreateDatabase_thenNoExceptionIsThrown() { + startMongoServer(); + useStartedMongoServer(); + preCreateCollection(); + + databaseInstance.createDatabase(); + } + + private void preCreateCollection() { + initializeMongoClient(); + mongoClient.getDatabase(NORDIC_COVERAGE_DATABASE).createCollection(COVERAGE_DATA_COLLECTION); + } + + @Test + public void givenATimeOut_whenCreateDatabase_thenDatabaseConnectionExceptionIsThrown() { + testThatInvalidConfigurationThrowsDatabaseConnectionException("10.1.1.1", 16256); + } + + private void testThatInvalidConfigurationThrowsDatabaseConnectionException(String host, int port) { + thrown.expect(DatabaseConnectionException.class); + thrown.expectCause(is(notNullValue(Throwable.class))); + thrown.expectMessage(containsString(NORDIC_COVERAGE_DATABASE)); + thrown.expectMessage(containsString(String.format("%s:%d", host, port))); + + DatabaseConfiguration invalidConfiguration = new DatabaseConfiguration(); + invalidConfiguration.setType("MongoDB"); + invalidConfiguration.setDbName(NORDIC_COVERAGE_DATABASE); + invalidConfiguration.setAddr(host); + invalidConfiguration.setPort(16256); + + databaseInstance.setMongoClientOptions(MongoClientOptions.builder().serverSelectionTimeout(10).build()); + + databaseInstance.open(invalidConfiguration); + databaseInstance.createDatabase(); + } + + @Test + public void givenAnUnknownHost_whenCreateDatabase_thenDatabaseConnectionExceptionIsThrown() { + testThatInvalidConfigurationThrowsDatabaseConnectionException("unknownHost", 16256); + } + + @Test + public void givenACell_whenSave_thenCellIsPersistedInCoverageDataCollection() { + startMongoServer(); + useStartedMongoServer(); + databaseInstance.createDatabase(); + + Cell aCell = new Cell(49.9324, -64.6364, null); + aCell.addReceivedSignals(3); + aCell.addNOofMissingSignals(1); + + PersistenceResult result = databaseInstance.save(Arrays.asList(aCell)); + + initializeMongoClient(); + assertThat(result.getStatus(), is(equalTo(PersistenceResult.Status.SUCCESS))); + assertThat(result.getWrittenCells(), is(1L)); + assertThat(mongoClient.getDatabase(NORDIC_COVERAGE_DATABASE).getCollection(COVERAGE_DATA_COLLECTION).count(), is(1L)); + } + + @Test + public void givenDatabaseInstanceIsNotOpened_whenCreateDatabase_thenIllegalStateExceptionIsThrown() { + thrown.expect(IllegalStateException.class); + + databaseInstance.createDatabase(); + } + + @Test + public void givenDatabaseInstanceIsNotOpened_whenSave_thenIllegalStateExceptionIsThrown() { + thrown.expect(IllegalStateException.class); + + databaseInstance.save(Collections.emptyList()); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/persistence/PersistenceResultTest.java b/src/test/java/dk/dma/ais/coverage/persistence/PersistenceResultTest.java new file mode 100644 index 0000000..bec7e8c --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/persistence/PersistenceResultTest.java @@ -0,0 +1,26 @@ +package dk.dma.ais.coverage.persistence; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +import org.junit.Test; + +public class PersistenceResultTest { + + @Test + public void whenSuccess_thenResultStatusIsSuccessAndWrittenNumberOfCellsIsAsProvided() { + PersistenceResult result = PersistenceResult.success(3L); + + assertThat(result.getStatus(), is(equalTo(PersistenceResult.Status.SUCCESS))); + assertThat(result.getWrittenCells(), is(3L)); + } + + @Test + public void whenFailure_thenResultStatusIsFailureAndWrittenNumberOfCellsIsZero() { + PersistenceResult result = PersistenceResult.failure(); + + assertThat(result.getStatus(), is(equalTo(PersistenceResult.Status.FAILURE))); + assertThat(result.getWrittenCells(), is(0L)); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java b/src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java new file mode 100644 index 0000000..ab1ed02 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java @@ -0,0 +1,41 @@ +package dk.dma.ais.coverage.persistence; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.Test; + +import dk.dma.ais.coverage.data.ICoverageData; + +public class PersisterServiceTest { + + @Test + public void whenStart_thenSaveIsInvokedOnDatabaseInstanceAtInterval() throws InterruptedException { + DatabaseInstance databaseInstance = mock(DatabaseInstance.class); + when(databaseInstance.save(anyList())).thenReturn(PersistenceResult.success(1)); + ICoverageData coverageData = mock(ICoverageData.class); + PersisterService persisterService = new PersisterService(databaseInstance, coverageData); + persisterService.intervalInSeconds(1L); + + persisterService.start(); + Thread.sleep(5 * 1000); + persisterService.stop(); + + verify(databaseInstance, atLeast(4)).save(anyList()); + } + + @Test + public void whenNewInstance_thenIntervalInSecondsDefaultsTo60() { + DatabaseInstance databaseInstance = mock(DatabaseInstance.class); + ICoverageData coverageData = mock(ICoverageData.class); + PersisterService persisterService = new PersisterService(databaseInstance, coverageData); + + assertThat(persisterService.getIntervalInSeconds(), is(equalTo(60L))); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactoryTest.java b/src/test/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactoryTest.java new file mode 100644 index 0000000..4a63022 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactoryTest.java @@ -0,0 +1,66 @@ +package dk.dma.ais.coverage.persistence; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class TypeBasedDatabaseInstanceFactoryTest { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private DatabaseInstanceFactory factory; + + @Before + public void setUp() throws Exception { + factory = new TypeBasedDatabaseInstanceFactory(); + } + + @Test + public void givenMemoryOnlyDatabaseType_whenCreateDatabaseInstance_thenInstanceReturnedIsOfTypeMemoryOnly() { + DatabaseInstance databaseInstance = factory.createDatabaseInstance("MemoryOnly"); + + assertThat(databaseInstance, is(instanceOf(MemoryOnlyDatabaseInstance.class))); + } + + @Test + public void givenMemoryOnlyDatabaseType_whenCreateDatabaseInstance_thenTypeMatchingIsCaseInsensitive() { + testThatTypeMatchingIsCaseInsensitive("MemoryOnly", MemoryOnlyDatabaseInstance.class); + } + + private void testThatTypeMatchingIsCaseInsensitive(String databaseType, Class expectedInstanceType) { + DatabaseInstance camelCaseMemoryOnlyDatabaseInstance = factory.createDatabaseInstance(databaseType); + DatabaseInstance lowerCaseMemoryOnlyDatabaseInstance = factory.createDatabaseInstance(databaseType.toLowerCase()); + DatabaseInstance upperCaseMemoryOnlyDatabaseInstance = factory.createDatabaseInstance(databaseType.toUpperCase()); + + assertThat(camelCaseMemoryOnlyDatabaseInstance, is(instanceOf(expectedInstanceType))); + assertThat(lowerCaseMemoryOnlyDatabaseInstance, is(instanceOf(expectedInstanceType))); + assertThat(upperCaseMemoryOnlyDatabaseInstance, is(instanceOf(expectedInstanceType))); + } + + @Test + public void givenMongoDbDatabaseType_whenCreateDatabaseInstance_thenInstanceReturnedIsOfTypeMongoDb() { + DatabaseInstance databaseInstance = factory.createDatabaseInstance("MongoDB"); + + assertThat(databaseInstance, is(instanceOf(MongoDatabaseInstance.class))); + } + + @Test + public void givenMongoDbDatabaseType_whenCreateDatabaseInstance_thenTypeMatchingIsCaseInsensitive() { + testThatTypeMatchingIsCaseInsensitive("MongoDB", MongoDatabaseInstance.class); + } + + @Test + public void givenUnknownDatabaseType_whenCreateDatabaseInstance_thenUnknownDatabaseTypeExceptionIsThrown() { + thrown.expect(UnknownDatabaseTypeException.class); + thrown.expectMessage(containsString("MySQL")); + + factory.createDatabaseInstance("MySQL"); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java b/src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java index d367c89..c86d036 100644 --- a/src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java +++ b/src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java @@ -1,17 +1,18 @@ package dk.dma.ais.coverage.rest; -import dk.dma.ais.coverage.AisCoverageBuilder; -import dk.dma.ais.coverage.Helper; -import dk.dma.ais.coverage.export.data.Status; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertThat; import java.io.IOException; import java.util.Date; -import static org.hamcrest.CoreMatchers.*; -import static org.junit.Assert.assertThat; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import dk.dma.ais.coverage.AisCoverageBuilder; +import dk.dma.ais.coverage.Helper; +import dk.dma.ais.coverage.export.data.Status; /** * Tests in this class are synchronized because they are all using {@code Helper.firstMessage} and @@ -24,7 +25,11 @@ public class CoverageRestServiceTest { @Before public void setUp() throws Exception { aisCoverageBuilder = new AisCoverageBuilder(); - aisCoverageBuilder.build(); + aisCoverageBuilder.withMockAisCoverageConfiguration() + .withMockAisBusConfiguration() + .withMockDatabaseConfiguration() + .withMockDatabaseInstanceForAnyType() + .build(); restService = new CoverageRestService(); } From ed483e9235a6073179fcba2aef1c87578c8a4ccb Mon Sep 17 00:00:00 2001 From: John Martel Date: Thu, 2 Mar 2017 14:39:59 -0500 Subject: [PATCH 07/59] Load latest saved coverage data from MongoDB --- pom.xml | 7 + .../java/dk/dma/ais/coverage/AisCoverage.java | 21 ++- .../dma/ais/coverage/data/OnlyMemoryData.java | 12 +- .../dk/dma/ais/coverage/data/TimeSpan.java | 4 + .../persistence/CoverageDataMarshaller.java | 16 ++ .../persistence/DatabaseInstance.java | 2 + .../MemoryOnlyDatabaseInstance.java | 9 +- .../MongoCoverageDataMarshaller.java | 119 ++++++++++++++ .../persistence/MongoDatabaseInstance.java | 58 ++++--- .../TypeBasedDatabaseInstanceFactory.java | 3 +- .../dk/dma/ais/coverage/AisCoverageTest.java | 3 +- .../dma/ais/coverage/fixture/CellFixture.java | 58 +++++++ .../MemoryOnlyDatabaseInstanceTest.java | 33 ++++ .../MongoCoverageDataMarshallerTest.java | 152 ++++++++++++++++++ .../MongoDatabaseInstanceTest.java | 61 ++++++- .../rest/CoverageRestServiceTest.java | 8 +- 16 files changed, 525 insertions(+), 41 deletions(-) create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/CoverageDataMarshaller.java create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java create mode 100644 src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java create mode 100644 src/test/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstanceTest.java create mode 100644 src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java diff --git a/pom.xml b/pom.xml index 09e5182..27470f3 100644 --- a/pom.xml +++ b/pom.xml @@ -115,6 +115,13 @@ 1.6.0 test + + + org.apache.commons + commons-lang3 + 3.5 + test + diff --git a/src/main/java/dk/dma/ais/coverage/AisCoverage.java b/src/main/java/dk/dma/ais/coverage/AisCoverage.java index a523ade..6429d44 100644 --- a/src/main/java/dk/dma/ais/coverage/AisCoverage.java +++ b/src/main/java/dk/dma/ais/coverage/AisCoverage.java @@ -14,6 +14,7 @@ */ package dk.dma.ais.coverage; +import java.util.List; import java.util.function.Consumer; import org.slf4j.Logger; @@ -22,6 +23,7 @@ import dk.dma.ais.bus.AisBus; import dk.dma.ais.bus.consumer.DistributerConsumer; import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; +import dk.dma.ais.coverage.data.Cell; import dk.dma.ais.coverage.persistence.DatabaseInstance; import dk.dma.ais.coverage.persistence.DatabaseInstanceFactory; import dk.dma.ais.coverage.persistence.PersisterService; @@ -54,15 +56,15 @@ public final class AisCoverage { private AisCoverage(AisCoverageConfiguration conf, DatabaseInstanceFactory databaseInstanceFactory) { this.conf = conf; + handler = new CoverageHandler(conf); databaseInstance = databaseInstanceFactory.createDatabaseInstance(conf.getDatabaseConfiguration().getType()); databaseInstance.open(conf.getDatabaseConfiguration()); databaseInstance.createDatabase(); + loadCoverageDataFromDatabase(); - handler = new CoverageHandler(conf); aisBus = conf.getAisbusConfiguration().getInstance(); - persisterService = new PersisterService(databaseInstance, handler.getDataHandler()); - persisterService.intervalInSeconds(conf.getDatabaseConfiguration().getPersistenceIntervalInSeconds()); + createPersisterService(); // Create web server if (conf.getServerConfiguration() != null) { @@ -83,6 +85,19 @@ public void accept(AisPacket packet) { aisBus.registerConsumer(unfilteredConsumer); } + private void loadCoverageDataFromDatabase() { + List loadedCoverageData = databaseInstance.loadLatestSavedCoverageData(); + + for (Cell cell : loadedCoverageData) { + handler.getDataHandler().updateCell(cell); + } + } + + private void createPersisterService() { + persisterService = new PersisterService(databaseInstance, handler.getDataHandler()); + persisterService.intervalInSeconds(conf.getDatabaseConfiguration().getPersistenceIntervalInSeconds()); + } + public void start() { // Start aisBus if (aisReader == null) { diff --git a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java index 0414dd2..8bc3dbb 100644 --- a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java +++ b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java @@ -69,9 +69,13 @@ public Cell getCell(String sourceMmsi, double lat, double lon) { } @Override - public void updateCell(Cell c) { - // TODO Auto-generated method stub - + public void updateCell(Cell newCell) { + Source superSource = getSource(AbstractCalculator.SUPERSOURCE_MMSI); + Cell oldCell = superSource.getCell(newCell.getLatitude(), newCell.getLongitude()); + if (oldCell == null) { + oldCell = superSource.createCell(newCell.getLatitude(), newCell.getLongitude()); + } + oldCell.setFixedWidthSpans(newCell.getFixedWidthSpans()); } @Override @@ -197,7 +201,6 @@ public void incrementReceivedSignals(String sourceMmsi, double lat, cell.getFixedWidthSpans().put(id.getTime(), ts); } ts.setMessageCounterTerrestrial(ts.getMessageCounterTerrestrial() + 1); - cell.incrementNOofReceivedSignals(); } @Override @@ -216,7 +219,6 @@ public void incrementMissingSignals(String sourceMmsi, double lat, cell.getFixedWidthSpans().put(id.getTime(), ts); } ts.incrementMissingSignals(); - cell.incrementNOofMissingSignals(); } public CustomMessage packetToCustomMessage(AisPacket packet) { diff --git a/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java b/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java index 2288134..e10c86f 100644 --- a/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java +++ b/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java @@ -51,6 +51,10 @@ public int getMissingSignals() { return missingSignals; } + public void setMissingSignals(int missingSignals) { + this.missingSignals = missingSignals; + } + public Map getDistinctShipsTerrestrial() { return distinctShipsTerrestrial; } diff --git a/src/main/java/dk/dma/ais/coverage/persistence/CoverageDataMarshaller.java b/src/main/java/dk/dma/ais/coverage/persistence/CoverageDataMarshaller.java new file mode 100644 index 0000000..dc58489 --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/CoverageDataMarshaller.java @@ -0,0 +1,16 @@ +package dk.dma.ais.coverage.persistence; + +import java.time.ZonedDateTime; +import java.util.List; + +import dk.dma.ais.coverage.data.Cell; + +/** + * Implementations are responsible for marshalling and unmarshalling coverage data to and from a database typing system. + */ +interface CoverageDataMarshaller { + + T marshall(List coverageData, ZonedDateTime dataTimestamp); + + List unmarshall(T coverageData); +} diff --git a/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java b/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java index 9f7eec7..8ca3236 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java @@ -40,4 +40,6 @@ public interface DatabaseInstance extends AutoCloseable { * when connection to the database server is impossible for any reason */ PersistenceResult save(List coverageData); + + List loadLatestSavedCoverageData(); } diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java b/src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java index 8d09608..bc0647e 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java @@ -1,5 +1,6 @@ package dk.dma.ais.coverage.persistence; +import java.util.Collections; import java.util.List; import dk.dma.ais.coverage.configuration.DatabaseConfiguration; @@ -12,12 +13,10 @@ class MemoryOnlyDatabaseInstance implements DatabaseInstance { @Override public void open(DatabaseConfiguration configuration) { - } @Override public void createDatabase() { - } @Override @@ -26,7 +25,11 @@ public PersistenceResult save(List coverageData) { } @Override - public void close() throws Exception { + public List loadLatestSavedCoverageData() { + return Collections.emptyList(); + } + @Override + public void close() throws Exception { } } diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java b/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java new file mode 100644 index 0000000..49430f0 --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java @@ -0,0 +1,119 @@ +package dk.dma.ais.coverage.persistence; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.bson.Document; + +import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.data.TimeSpan; + +/** + * A {@link CoverageDataMarshaller} that marshalls coverage data in a format usable to store in a MongoDB database. + */ +class MongoCoverageDataMarshaller implements CoverageDataMarshaller { + + @Override + public Document marshall(List coverageData, ZonedDateTime dataTimestamp) { + Document coverageDataDocument = new Document(); + + coverageDataDocument.put("dataTimestamp", dataTimestamp.withZoneSameInstant(ZoneId.of("UTC")).format(DateTimeFormatter.ISO_DATE_TIME)); + + List> grid = new ArrayList<>(); + if (coverageData != null) { + marshallCells(coverageData, grid); + } + coverageDataDocument.put("cells", grid); + + return coverageDataDocument; + } + + private void marshallCells(List coverageData, List> grid) { + for (Cell cell : coverageData) { + Map savedCell = marshallCell(cell); + grid.add(savedCell); + } + } + + private Map marshallCell(Cell cell) { + Map savedCell = new LinkedHashMap<>(); + savedCell.put("cellId", cell.getId()); + savedCell.put("latitude", cell.getLatitude()); + savedCell.put("longitude", cell.getLongitude()); + + Map> fixedWidthTimeSpans = marshallCellTimeSpans(cell); + savedCell.put("timespans", fixedWidthTimeSpans); + + return savedCell; + } + + private Map> marshallCellTimeSpans(Cell cell) { + Map> fixedWidthTimeSpans = new LinkedHashMap<>(); + for (Map.Entry fixedWidthTimeSpan : cell.getFixedWidthSpans().entrySet()) { + Map messages = new LinkedHashMap<>(); + messages.put("firstMessage", fixedWidthTimeSpan.getValue().getFirstMessage().getTime()); + messages.put("lastMessage", fixedWidthTimeSpan.getValue().getLastMessage().getTime()); + messages.put("messageCounterSat", fixedWidthTimeSpan.getValue().getMessageCounterSat()); + messages.put("messageCounterTerrestrial", fixedWidthTimeSpan.getValue().getMessageCounterTerrestrial()); + messages.put("messageCounterTerrestrialUnfiltered", fixedWidthTimeSpan.getValue().getMessageCounterTerrestrialUnfiltered()); + messages.put("missingSignals", fixedWidthTimeSpan.getValue().getMissingSignals()); + + fixedWidthTimeSpans.put(fixedWidthTimeSpan.getKey().toString(), messages); + } + + return fixedWidthTimeSpans; + } + + @Override + public List unmarshall(Document coverageData) { + List unmarshalledCoverageData = new ArrayList<>(); + + if (coverageData != null) { + unmarshallCells(coverageData, unmarshalledCoverageData); + } + + return unmarshalledCoverageData; + } + + private void unmarshallCells(Document coverageData, List unmarshalledCoverageData) { + Object cells = coverageData.get("cells"); + + if (cells != null && (cells instanceof List)) { + List> grid = (List>) cells; + for (Map cell : grid) { + Cell unmarshalledCell = unmarshallCell(cell); + unmarshalledCoverageData.add(unmarshalledCell); + } + } + } + + private Cell unmarshallCell(Map cell) { + Cell unmarshalledCell = new Cell((double) cell.get("latitude"), (double) cell.get("longitude"), (String) cell.get("cellId")); + + Map> fixedWidthTimeSpans = (Map>) cell.get("timespans"); + Map unmarshalledTimeSpans = unmarshallTimeSpans(fixedWidthTimeSpans); + + unmarshalledCell.setFixedWidthSpans(unmarshalledTimeSpans); + return unmarshalledCell; + } + + private Map unmarshallTimeSpans(Map> fixedWidthTimeSpans) { + Map unmarshalledTimeSpans = new LinkedHashMap<>(); + for (Map.Entry> timespan : fixedWidthTimeSpans.entrySet()) { + TimeSpan unmarshalledTimeSpan = new TimeSpan(new Date(timespan.getValue().get("firstMessage").longValue())); + unmarshalledTimeSpan.setLastMessage(new Date(timespan.getValue().get("lastMessage").longValue())); + unmarshalledTimeSpan.setMessageCounterSat(timespan.getValue().get("messageCounterSat").intValue()); + unmarshalledTimeSpan.setMessageCounterTerrestrial(timespan.getValue().get("messageCounterTerrestrial").intValue()); + unmarshalledTimeSpan.setMessageCounterTerrestrialUnfiltered(timespan.getValue().get("messageCounterTerrestrialUnfiltered").intValue()); + unmarshalledTimeSpan.setMissingSignals(timespan.getValue().get("missingSignals").intValue()); + unmarshalledTimeSpans.put(Long.valueOf(timespan.getKey()), unmarshalledTimeSpan); + } + return unmarshalledTimeSpans; + } +} diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java b/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java index dc0153c..d561b1f 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java @@ -2,9 +2,8 @@ import java.time.ZoneId; import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.LinkedHashMap; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -16,7 +15,9 @@ import com.mongodb.MongoClientOptions; import com.mongodb.MongoException; import com.mongodb.ServerAddress; +import com.mongodb.client.FindIterable; import com.mongodb.client.MongoDatabase; +import dk.dma.ais.coverage.Helper; import dk.dma.ais.coverage.configuration.DatabaseConfiguration; import dk.dma.ais.coverage.data.Cell; @@ -27,12 +28,17 @@ class MongoDatabaseInstance implements DatabaseInstance { private static final Logger LOG = LoggerFactory.getLogger(MongoDatabaseInstance.class); private static final String COVERAGE_DATA = "coverageData"; + private CoverageDataMarshaller marshaller; private String mongoServerHost; private int mongoServerPort; private String databaseName; private MongoClientOptions mongoClientOptions = MongoClientOptions.builder().build(); private MongoClient client; + public MongoDatabaseInstance(CoverageDataMarshaller marshaller) { + this.marshaller = marshaller; + } + @Override public void open(DatabaseConfiguration configuration) { mongoServerHost = configuration.getAddr(); @@ -107,22 +113,8 @@ private boolean collectionExists(MongoDatabase database) { public PersistenceResult save(List coverageData) { requireOpenConnection(); - Document coverageDataDocument = new Document(); - long numberOfSavedCells = 0; - coverageDataDocument.put("dataTimestamp", ZonedDateTime.now(ZoneId.of("UTC")).format(DateTimeFormatter.ISO_DATE_TIME)); - List> grid = new ArrayList<>(); - for (Cell cell : coverageData) { - Map savedCell = new LinkedHashMap<>(); - savedCell.put("cellId", cell.getId()); - savedCell.put("latitude", cell.getLatitude()); - savedCell.put("longitude", cell.getLongitude()); - savedCell.put("numberOfReceivedSignals", cell.getNOofReceivedSignals()); - savedCell.put("numberOfMissedSignals", cell.getNOofMissingSignals()); - grid.add(savedCell); - - numberOfSavedCells++; - } - coverageDataDocument.put("cells", grid); + Document coverageDataDocument = marshaller.marshall(coverageData, ZonedDateTime.now(ZoneId.of("UTC"))); + long numberOfSavedCells = ((List>) coverageDataDocument.get("cells")).size(); try { client.getDatabase(databaseName).getCollection(COVERAGE_DATA).insertOne(coverageDataDocument); @@ -134,7 +126,37 @@ public PersistenceResult save(List coverageData) { return PersistenceResult.failure(); } + @Override + public List loadLatestSavedCoverageData() { + requireOpenConnection(); + + List latestCoverageData = new ArrayList<>(); + + try { + Document orderByDataTimestamp = new Document(); + orderByDataTimestamp.put("dataTimestamp", -1); + + FindIterable foundDocuments = client.getDatabase(databaseName).getCollection(COVERAGE_DATA).find().sort(orderByDataTimestamp).limit(1); + if (foundDocuments.iterator().hasNext()) { + Document document = foundDocuments.iterator().next(); + latestCoverageData.addAll(marshaller.unmarshall(document)); + } + } catch (MongoException e) { + logAndTransformException(e); + } + + if (!latestCoverageData.isEmpty()) { + Helper.firstMessage = latestCoverageData.get(0).getFixedWidthSpans().values().iterator().next().getFirstMessage(); + } + + return Collections.unmodifiableList(latestCoverageData); + } + void setMongoClientOptions(MongoClientOptions mongoClientOptions) { this.mongoClientOptions = mongoClientOptions; } + + void setMarshaller(MongoCoverageDataMarshaller marshaller) { + this.marshaller = marshaller; + } } diff --git a/src/main/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactory.java b/src/main/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactory.java index 4699ec0..86ed739 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactory.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/TypeBasedDatabaseInstanceFactory.java @@ -11,7 +11,8 @@ public DatabaseInstance createDatabaseInstance(String databaseType) { if ("MemoryOnly".equalsIgnoreCase(databaseType)) { return new MemoryOnlyDatabaseInstance(); } else if ("MongoDB".equalsIgnoreCase(databaseType)) { - return new MongoDatabaseInstance(); + MongoCoverageDataMarshaller marshaller = new MongoCoverageDataMarshaller(); + return new MongoDatabaseInstance(marshaller); } else { throw new UnknownDatabaseTypeException(String.format("Unsupported database type: [%s]", databaseType)); } diff --git a/src/test/java/dk/dma/ais/coverage/AisCoverageTest.java b/src/test/java/dk/dma/ais/coverage/AisCoverageTest.java index 9402588..792c1ef 100644 --- a/src/test/java/dk/dma/ais/coverage/AisCoverageTest.java +++ b/src/test/java/dk/dma/ais/coverage/AisCoverageTest.java @@ -12,7 +12,7 @@ public class AisCoverageTest { @Test - public void whenNewInstance_thenDatabaseIsCreated() { + public void whenNewInstance_thenDatabaseIsCreatedAndExistingDataIsLoaded() { DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration(); databaseConfiguration.setType("MongoDB"); @@ -22,6 +22,7 @@ public void whenNewInstance_thenDatabaseIsCreated() { verify(mockDatabaseInstance).open(databaseConfiguration); verify(mockDatabaseInstance).createDatabase(); + verify(mockDatabaseInstance).loadLatestSavedCoverageData(); } private AisCoverage aisCoverageWithMockDatabaseInstance(DatabaseConfiguration databaseConfiguration, DatabaseInstance mockDatabaseInstance) { diff --git a/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java b/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java new file mode 100644 index 0000000..e63c6fe --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java @@ -0,0 +1,58 @@ +package dk.dma.ais.coverage.fixture; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.commons.lang3.RandomUtils; + +import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.data.TimeSpan; + +public class CellFixture { + + public static Cell createCellWithTimeSpans() { + Cell cell = createCell(); + + Map fixedWidthTimeSpans = createTimeSpans(); + cell.setFixedWidthSpans(fixedWidthTimeSpans); + + return cell; + } + + private static Cell createCell() { + double latitude = RandomUtils.nextDouble(); + double longitude = RandomUtils.nextDouble(); + String cellId = latitude + "_" + longitude; + + return new Cell(latitude, longitude, cellId); + } + + private static Map createTimeSpans() { + Map fixedWidthTimeSpans = new LinkedHashMap<>(); + + for (int i = 0; i <= 1; i++) { + long timespanKey = ZonedDateTime.now(ZoneId.of("UTC")).toInstant().toEpochMilli() + i; + TimeSpan timespan = new TimeSpan(new Date(timespanKey)); + timespan.setMessageCounterSat(RandomUtils.nextInt()); + timespan.setMessageCounterTerrestrial(RandomUtils.nextInt()); + timespan.setMessageCounterTerrestrialUnfiltered(RandomUtils.nextInt()); + timespan.setMissingSignals(RandomUtils.nextInt()); + + fixedWidthTimeSpans.put(timespanKey, timespan); + } + + return fixedWidthTimeSpans; + } + + public static Cell createCellWithNoTimeSpan() { + Cell cell = createCell(); + + Map fixedWidthTimeSpans = new LinkedHashMap<>(); + cell.setFixedWidthSpans(fixedWidthTimeSpans); + + return cell; + } +} diff --git a/src/test/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstanceTest.java b/src/test/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstanceTest.java new file mode 100644 index 0000000..8f0d905 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstanceTest.java @@ -0,0 +1,33 @@ +package dk.dma.ais.coverage.persistence; + +import static java.util.Collections.emptyList; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.junit.Assert.assertThat; + +import java.util.List; + +import org.junit.Test; + +import dk.dma.ais.coverage.data.Cell; + +public class MemoryOnlyDatabaseInstanceTest { + + @Test + public void whenSave_thenNullPersistenceResultIsReturned() { + DatabaseInstance databaseInstance = new MemoryOnlyDatabaseInstance(); + + PersistenceResult persistenceResult = databaseInstance.save(emptyList()); + + assertThat(persistenceResult, is(nullValue())); + } + + @Test + public void whenLoadLatestSavedCoverageData_thenEmptyListIsReturned() { + DatabaseInstance databaseInstance = new MemoryOnlyDatabaseInstance(); + + List coverageData = databaseInstance.loadLatestSavedCoverageData(); + + assertThat(coverageData.isEmpty(), is(true)); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java b/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java new file mode 100644 index 0000000..c862cdd --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java @@ -0,0 +1,152 @@ +package dk.dma.ais.coverage.persistence; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.bson.Document; +import org.junit.Before; +import org.junit.Test; + +import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.data.TimeSpan; +import dk.dma.ais.coverage.fixture.CellFixture; + +public class MongoCoverageDataMarshallerTest { + private MongoCoverageDataMarshaller marshaller; + + @Before + public void setUp() throws Exception { + marshaller = new MongoCoverageDataMarshaller(); + } + + @Test + public void givenNoCoverageData_whenMarshall_thenDocumentWithEmptyCellsArrayAndTimestampIsReturned() { + ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC")); + + Document marshalledCoverageData = marshaller.marshall(null, now); + + assertThatDocumentHasEmptyCellsAndDataTimestamp(marshalledCoverageData, now); + + marshalledCoverageData = marshaller.marshall(Collections.emptyList(), now); + + assertThatDocumentHasEmptyCellsAndDataTimestamp(marshalledCoverageData, now); + } + + private void assertThatDocumentHasEmptyCellsAndDataTimestamp(Document marshalledCoverageData, ZonedDateTime now) { + ZonedDateTime dataTimestamp = ZonedDateTime.parse((String) marshalledCoverageData.get("dataTimestamp"), DateTimeFormatter.ISO_DATE_TIME); + assertThat(dataTimestamp, is(equalTo(now))); + assertThat(((List>) marshalledCoverageData.get("cells")).isEmpty(), is(true)); + } + + @Test + public void givenManyCells_whenMarshall_thenDocumentWithTheseCellsAndTimestampIsReturned() { + ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC")); + + List cells = new ArrayList<>(); + Cell firstCell = CellFixture.createCellWithTimeSpans(); + Cell secondCell = CellFixture.createCellWithNoTimeSpan(); + cells.add(firstCell); + cells.add(secondCell); + + Document marshalledCoverageData = marshaller.marshall(cells, now); + + ZonedDateTime dataTimestamp = ZonedDateTime.parse((String) marshalledCoverageData.get("dataTimestamp"), DateTimeFormatter.ISO_DATE_TIME); + assertThat(dataTimestamp, is(equalTo(now))); + + List> grid = (List>) marshalledCoverageData.get("cells"); + assertThat(grid.size(), is(equalTo(2))); + + Map firstMarshalledCell = grid.get(0); + assertThat(firstMarshalledCell.get("cellId"), is(equalTo(firstCell.getId()))); + assertThat(firstMarshalledCell.get("latitude"), is(equalTo(firstCell.getLatitude()))); + assertThat(firstMarshalledCell.get("longitude"), is(equalTo(firstCell.getLongitude()))); + + Map> firstMarshalledCellTimespans = (Map>) firstMarshalledCell.get("timespans"); + assertThat(firstMarshalledCellTimespans.size(), is(equalTo(2))); + + for (Long timespanKey : firstCell.getFixedWidthSpans().keySet()) { + TimeSpan expectedTimeSpan = firstCell.getFixedWidthSpans().get(timespanKey); + Map marshalledTimeSpan = firstMarshalledCellTimespans.get(timespanKey.toString()); + + assertThat(marshalledTimeSpan.get("firstMessage"), is(equalTo(expectedTimeSpan.getFirstMessage().getTime()))); + assertThat(marshalledTimeSpan.get("lastMessage"), is(equalTo(expectedTimeSpan.getLastMessage().getTime()))); + assertThat(marshalledTimeSpan.get("messageCounterSat"), is(equalTo(expectedTimeSpan.getMessageCounterSat()))); + assertThat(marshalledTimeSpan.get("messageCounterTerrestrial"), is(equalTo(expectedTimeSpan.getMessageCounterTerrestrial()))); + assertThat(marshalledTimeSpan.get("messageCounterTerrestrialUnfiltered"), is(equalTo(expectedTimeSpan.getMessageCounterTerrestrialUnfiltered()))); + assertThat(marshalledTimeSpan.get("missingSignals"), is(equalTo(expectedTimeSpan.getMissingSignals()))); + } + + Map secondMarshalledCell = grid.get(1); + assertThat(secondMarshalledCell.get("cellId"), is(equalTo(secondCell.getId()))); + assertThat(secondMarshalledCell.get("latitude"), is(equalTo(secondCell.getLatitude()))); + assertThat(secondMarshalledCell.get("longitude"), is(equalTo(secondCell.getLongitude()))); + + Map> secondMarshalledCellTimespans = (Map>) secondMarshalledCell.get("timespans"); + assertThat(secondMarshalledCellTimespans.isEmpty(), is(true)); + } + + @Test + public void givenNoCoverageData_whenUnmarshall_thenEmptyListIsReturned() { + List coverageData = marshaller.unmarshall(null); + assertThat(coverageData.isEmpty(), is(true)); + + coverageData = marshaller.unmarshall(new Document()); + assertThat(coverageData.isEmpty(), is(true)); + + Document documentWithEmptyCells = new Document(); + documentWithEmptyCells.put("cells", Collections.emptyMap()); + coverageData = marshaller.unmarshall(documentWithEmptyCells); + assertThat(coverageData.isEmpty(), is(true)); + } + + @Test + public void givenManyCells_whenUnmarshall_thenTheseCellsAreReturned() { + ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC")); + List cells = new ArrayList<>(); + Cell firstCell = CellFixture.createCellWithTimeSpans(); + Cell secondCell = CellFixture.createCellWithNoTimeSpan(); + cells.add(firstCell); + cells.add(secondCell); + Document marshalledCoverageData = marshaller.marshall(cells, now); + + List unmarshalledCoverageData = marshaller.unmarshall(marshalledCoverageData); + + assertThat(unmarshalledCoverageData.size(), is(equalTo(2))); + + Cell firstUnmarshalledCell = unmarshalledCoverageData.get(0); + assertThat(firstUnmarshalledCell.getId(), is(equalTo(firstCell.getId()))); + assertThat(firstUnmarshalledCell.getLatitude(), is(equalTo(firstCell.getLatitude()))); + assertThat(firstUnmarshalledCell.getLongitude(), is(equalTo(firstCell.getLongitude()))); + + Map firstUnmarshalledCellFixedWidthSpans = firstUnmarshalledCell.getFixedWidthSpans(); + assertThat(firstUnmarshalledCellFixedWidthSpans.size(), is(equalTo(2))); + + for (Long timespanKey : firstUnmarshalledCellFixedWidthSpans.keySet()) { + TimeSpan expectedTimeSpan = firstCell.getFixedWidthSpans().get(timespanKey); + TimeSpan unmarshalledTimeSpan = firstUnmarshalledCellFixedWidthSpans.get(timespanKey); + + assertThat(unmarshalledTimeSpan.getFirstMessage(), is(equalTo(expectedTimeSpan.getFirstMessage()))); + assertThat(unmarshalledTimeSpan.getLastMessage(), is(equalTo(expectedTimeSpan.getLastMessage()))); + assertThat(unmarshalledTimeSpan.getMessageCounterSat(), is(equalTo(expectedTimeSpan.getMessageCounterSat()))); + assertThat(unmarshalledTimeSpan.getMessageCounterTerrestrial(), is(equalTo(expectedTimeSpan.getMessageCounterTerrestrial()))); + assertThat(unmarshalledTimeSpan.getMessageCounterTerrestrialUnfiltered(), is(equalTo(expectedTimeSpan.getMessageCounterTerrestrialUnfiltered()))); + assertThat(unmarshalledTimeSpan.getMissingSignals(), is(equalTo(expectedTimeSpan.getMissingSignals()))); + } + + Cell secondUnmarshalledCell = unmarshalledCoverageData.get(1); + assertThat(secondUnmarshalledCell.getId(), is(equalTo(secondCell.getId()))); + assertThat(secondUnmarshalledCell.getLatitude(), is(equalTo(secondCell.getLatitude()))); + assertThat(secondUnmarshalledCell.getLongitude(), is(equalTo(secondCell.getLongitude()))); + + assertThat(secondUnmarshalledCell.getFixedWidthSpans().isEmpty(), is(true)); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java b/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java index 71e8511..3f0fbc6 100644 --- a/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java +++ b/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java @@ -6,11 +6,16 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; import java.net.InetSocketAddress; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.Arrays; import java.util.Collections; +import java.util.List; +import org.bson.Document; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -25,6 +30,7 @@ import de.bwaldvogel.mongo.backend.memory.MemoryBackend; import dk.dma.ais.coverage.configuration.DatabaseConfiguration; import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.fixture.CellFixture; public class MongoDatabaseInstanceTest { private static final String NORDIC_COVERAGE_DATABASE = "nordicCoverage"; @@ -33,15 +39,17 @@ public class MongoDatabaseInstanceTest { @Rule public ExpectedException thrown = ExpectedException.none(); + private MongoDatabaseInstance databaseInstance; + private CoverageDataMarshaller marshaller; private MongoServer mongoServer; private DatabaseConfiguration configuration; - private MongoDatabaseInstance databaseInstance; private InetSocketAddress serverAddress; private MongoClient mongoClient; @Before public void setUp() throws Exception { - databaseInstance = new MongoDatabaseInstance(); + marshaller = mock(CoverageDataMarshaller.class); + databaseInstance = new MongoDatabaseInstance(marshaller); } @After @@ -93,7 +101,7 @@ private void useStartedMongoServer() { } private void initializeMongoClient() { - MongoClientOptions options = MongoClientOptions.builder().serverSelectionTimeout(10).build(); + MongoClientOptions options = MongoClientOptions.builder().serverSelectionTimeout(100).build(); mongoClient = new MongoClient(new ServerAddress(serverAddress), options); } @@ -143,15 +151,13 @@ public void givenAnUnknownHost_whenCreateDatabase_thenDatabaseConnectionExceptio public void givenACell_whenSave_thenCellIsPersistedInCoverageDataCollection() { startMongoServer(); useStartedMongoServer(); - databaseInstance.createDatabase(); + preCreateCollection(); + databaseInstance.setMarshaller(new MongoCoverageDataMarshaller()); - Cell aCell = new Cell(49.9324, -64.6364, null); - aCell.addReceivedSignals(3); - aCell.addNOofMissingSignals(1); + Cell aCell = CellFixture.createCellWithTimeSpans(); PersistenceResult result = databaseInstance.save(Arrays.asList(aCell)); - initializeMongoClient(); assertThat(result.getStatus(), is(equalTo(PersistenceResult.Status.SUCCESS))); assertThat(result.getWrittenCells(), is(1L)); assertThat(mongoClient.getDatabase(NORDIC_COVERAGE_DATABASE).getCollection(COVERAGE_DATA_COLLECTION).count(), is(1L)); @@ -170,4 +176,43 @@ public void givenDatabaseInstanceIsNotOpened_whenSave_thenIllegalStateExceptionI databaseInstance.save(Collections.emptyList()); } + + @Test + public void givenDatabaseInstanceIsNotOpened_whenLoadLatestSavedCoverageData_thenIllegalStateExceptionIsThrown() { + thrown.expect(IllegalStateException.class); + + databaseInstance.loadLatestSavedCoverageData(); + } + + @Test + public void givenNoSavedCoverageData_whenLoadLatestSavedCoverageData_thenEmptyListIsReturned() { + startMongoServer(); + useStartedMongoServer(); + preCreateCollection(); + + List cells = databaseInstance.loadLatestSavedCoverageData(); + + assertThat(cells.isEmpty(), is(true)); + } + + @Test + public void givenSavedCoverageData_whenLoadLatestSavedCoverageData_thenThisDataIsReturned() { + startMongoServer(); + useStartedMongoServer(); + preCreateCollection(); + + Cell aCell = CellFixture.createCellWithTimeSpans(); + Cell anotherCell = CellFixture.createCellWithNoTimeSpan(); + List cells = Arrays.asList(aCell, anotherCell); + MongoCoverageDataMarshaller marshaller = new MongoCoverageDataMarshaller(); + Document document = marshaller.marshall(cells, ZonedDateTime.now(ZoneId.of("UTC"))); + + mongoClient.getDatabase(NORDIC_COVERAGE_DATABASE).getCollection(COVERAGE_DATA_COLLECTION).insertOne(document); + + databaseInstance.setMarshaller(marshaller); + + List loadedCoverageData = databaseInstance.loadLatestSavedCoverageData(); + + assertThat(loadedCoverageData.size(), is(equalTo(2))); + } } diff --git a/src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java b/src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java index c86d036..5cf0673 100644 --- a/src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java +++ b/src/test/java/dk/dma/ais/coverage/rest/CoverageRestServiceTest.java @@ -1,7 +1,11 @@ package dk.dma.ais.coverage.rest; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.Date; @@ -77,6 +81,6 @@ public synchronized void givenNoTerrestrialMessageReceivedYet_whenStatus_thenSta assertThatStatusIsRunning(status); assertThat(status.firstMessage, is(equalTo(Helper.firstMessage.getTime()))); - assertThat(status.lastMessage, is(equalTo(now.getTime()))); + assertTrue(status.lastMessage >= now.getTime()); } } From 8c6b7334bb09698a8fdf99836a34da0fd6d62f82 Mon Sep 17 00:00:00 2001 From: John Martel Date: Thu, 2 Mar 2017 14:52:19 -0500 Subject: [PATCH 08/59] Use the maven wrapper to build --- .gitattributes | 1 + .mvn/wrapper/maven-wrapper.jar | Bin 0 -> 49519 bytes .mvn/wrapper/maven-wrapper.properties | 1 + mvnw | 236 ++++++++++++++++++++++++++ mvnw.cmd | 146 ++++++++++++++++ 5 files changed, 384 insertions(+) create mode 100644 .gitattributes create mode 100644 .mvn/wrapper/maven-wrapper.jar create mode 100644 .mvn/wrapper/maven-wrapper.properties create mode 100755 mvnw create mode 100644 mvnw.cmd diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c425e6f --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +mvnw.cmd eol=crlf diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..c6feb8bb6f76f2553e266ff8bf8867105154237e GIT binary patch literal 49519 zcmb@tV|1n6wzeBvGe*U>ZQHh;%-Bg)Y}={WHY%yuwkkF%MnzxVwRUS~wY|@J_gP;% z^VfXZ{5793?z><89(^dufT2xlYVOQnYG>@?lA@vQF|UF0&X7tk8BUf?wq2J& zZe&>>paKUg4@;fwk0yeUPvM$yk)=f>TSFFB^a8f|_@mbE#MaBnd5qf6;hXq}c%IeK zn7gB0Kldbedq-vl@2wxJi{$%lufroKUjQLSFmt|<;M8~<5otM5ur#Dgc@ivmwRiYZW(Oco7kb8DWmo|a{coqYMU2raB9r6e9viK6MI3c&%jp05-Tf*O#6@8Ra=egYy01 z-V!G;_omANEvU-8!*>*)lWka9M<+IkNsrsenbXOfLc6qrYe`;lpst;vfs*70$z9UM zq%L>pFCOr$X*|9&3L2h;?VA9-IU*iR6FiGlJ=b~DzE5s^thxXUs4%~*zD#K&k>wZAU8 zpaa!M+Z-zjkfGK15N!&o<3=cgbZV7%ex@j^)Q9V`q^i;Fsbkbe6eHJ;dx{QbdCCs1 zdxq^WxoPsr`eiK3D0Ep}k$ank-0G&+lY!ZHDZBYEx%% z2FyE?Lb0cflLB)kDIj;G=m`^UO<4h(RWdF-DT>p{1J5J90!K!AgC0)?jxPbm$KUjg zJED+#7xQmAmr`(S%BQTV-c97As~r3zD$E;3S)@}p5udA@m6pLgRL5h-;m>LvCq?&Q zokC7Vnk-zBEaa;=Y;6(LJHS>mOJV&%0YfRdUOqbKZy~b z(905jIW0Pg;y`Yv2t+RnDvL4yGEUX*tK)JT6TWn4ik~L)fX#tAV!d8)+A)qWtSjcr z7s|f%f;*%XW!jiRvv9ayj@f&dc|1tKDc{O3BWcLGsn-OYyXRLXEOEwP4k?c`nIut0 z?4S;eO@EoynmkxHq>QpDL1q^wOQxrl))2qya?dk05^5hK? z{P6;WKHUaHw9B0dd&|xw&CYN2fVrn};Gq<=Z^QZk3e~HzzY~JrnPCs0XwMp#B<9Gm zw0?7h#4EY%O-ub6mi&O2vcpIkuM?st;RtEpKSz^Xr#3WHhpsZd!gh|_jGQ`KA30T- zKlz9vgB;pY^}Uh??nQKSzk>2&J+Qi*r3DeX4^$%2ag9^x_YckA-f9p_;8ulh(8j9~ zes{O#{v!m%n^el(VryTF-C%xfJJ$rZj)|Y|8o&))q9CEwg2;Wz&xzyHD=@T_B%b}C z=8G^*4*J4#jUJn{7-3^U(_uUp6E8+GDt#le)nya-Q4kL5ZGiFxT4bF+mX`whcif*? z>CL&Ryn3HHT^^QmWYr<}Q1_Jj7fOh}cS8r+^R#at-CnNl3!1_$96&7nR}gh}))7a0J&z-_eI))+{RCt)r8|7|sV9o01^9nv?aePxMqwPP!x|sNmnn&6{K$K*mVX9lxSAmcqAV1(hKA-=coeTb*otxTOGYXsh zW$31^q7L@<#y~SUYoNKP1JK?4|FQNQb$i8mCG@WhX9i_^;@M2f#!nq7_K*M!4lGz1 z5tfADkO7BZDLgVQ?k7C)f;$eqjHI&zgxhf}x$8^ZEwFfm-qY=+M+fbS)9r8fFE5H9 zv{WPU35cR8%z;(W%5<>y+E&v84J4^Y##N!$B++RI`CZ1i3IW9Nau=*pSxW&^Ov-F> zex=&9XYLVcm1Y?am>2VC`%gMev9$#~; zYwxYvMfeKFsd!OBB@eOb2QNHFcsfKm;&z{OVEUiYmQ}~L@>$Ms@|Ptf3jQO-=Q;1+ zFCw+p+Z3lK_FmIAYnk2V;o915cDM}%Ht5RH%w}P>Yg9{h1mZ}~R6tUII4X7i4-2i% z2Uiw3_uHR!d~5(s;p6btI@-xhAkRg9K|n#}PNT9Dw9P>z$3>30lP1(=mcQ|tpyv3@ ze1qU!69OAx4s7$8r7Y-#5I`m!BXq`f!6C(BtUlG-oq+liqMCS_D@0nSFc%y+N6_Zh zi%L3LhF3zZP{d1)L&SXxPD(fp@T@J;jZeNaf$zl>vAh7=tI z2;wS^QyRdZm~)Ur&!af;8eB8*7(F96K^=WbC$)#TWvB~Awo5AtPf8Il4snD}Xsqd< z>cH+gcg72nTg5tl>oFbwdT{BDyy1=f=4~h~L$)UX;FXa;NdSlyF{(YLrx&VDp`pQI zh3pQtC=d8i1V6yUmFon*LQsNYWen?eO-gSZ4cvYcdEd0klSxcBYw+|5AyCv6TT96h z{7Yh9`h}biU?3oBFn=d8>Hn`1Q*w6rgeX^QbC-WFwjY}Int0;qUny4WMjIee@#0%l z>YAWLVCNo1lp$>9L$Tx`t!dp?>5Pfbhc*!*wzfWkj_x`Q?`3Jc@9r8uq~dgb+lgeh zlA`eUal3e2ZnWQSSYB>qy#85^>j7!=uO-hG5*erp22NaC81#Ytioc>r?D9$b_JiC+ zSp)8KR$%}FjFNRkeE#c5vKbXNJDBoO< z)73Jt7Y|3v45efud1xkg2GO3OwYfsuBV`f6S_D>Aoh2%=`1Y$bHP>0kBvTSowX57H z&1nbbx=IT>X^ScKYL&&{LNq~^UNgR|at`D;SxTYpLvnj_F*bGgNV2tEl1k$ccA&NW zmX(LV*>Op)BOgoric(98mIU)$eUa&jM5bKlnOrHm$p^v@u;W0J)!@XWg+#X=9En(-tiw!l?65rD=zzl(+%<)bI{ZN;SRco{jO;>7 zlSY|TIxuN|d#YHx^^~>iYj2V>cC>wQwWzGVI!6#epjJ6tl_`7tDY17WMKMB@s*Jr& zXOs*@>EwQ6s>M13eZEBJ#q0|;8jao{wK4keesH9?$OSk~_3#*x`8fAzQa7fprQ6(Z zi$}B%m81y*S)RxaX;wW!5{{EDw8)IE3XDRO1Y^%TMr}c|Y>WBAKT=b*K&uMT(?JSl zO>gVtl_bKQ$??TeWr7wYO+Vbl?CTQj?JrW&td`|#@;R2Gca9jq^p`{@)KY97o3}Af zfTh{pUUWD;P7sq=I!lA6;*hq0Nq`F56T)x$K?BMOk}tptYw(%$?*otp2N6IF3#GgqM46Cda!qzvGZcMgcGV`bY5ZIfOB6^;US#WgRai zq#vS8ZqPY953|eFw<-p2Cakx|z#_{4pG}mk{EANI{PnK*CUslvS8whko=OTe13|It z>{O2p=mmanR2-n>LQHaMo}noWCmjFO@7^z~`Y{V>O`@rT{yBS=VXsb}*Pi_zDqM3? zjCZqWR}fEzAkms+Hiq8~qRAFvo}dVW{1gcZ?v&PdX?UG*yS}zT9g7nZ!F1WRH}sHA zJ4~B2Br~8?uhbaX!3g+7=3fVM)q^wEzv**rk5e34==NRCV z3G$G5B!DICFslm)c){oesa_0muLxGoq`xYVNURl*NhE#v2>y9vDz&vJwrB`Q>DhN# zY2GnY!Y^8E%PU0}haXL$8a5QN1-&7NWuC~{62j| z2ozmFyx8GpOzj?&KK1JF28;E8H_p4N^LMm9K0y}!lCxcK79eFGTtGm?7jy?t94Q@X zli|our1#|>f*68fyA0bSn=YisYSl8HB(dFN4Y$qb7p4DR0YQt=^eEMnJkgiM48$>QV6x5*^a|D|t zMPDk}u<^YEYrt|H&hy)DRk%rDIb{LTo;h7=fp^J9Lr&`{9`8_pS*tQ_$KXB$2#5{h z-&yPbN-zInq{7aYZuaItS8-2Mb4OQe2jD*&)0~898E|HlAq`o!M&It@vvnj z_y@))>~_oR%S8OfmFTGYIat^#8_YKMqWLac<^}RZFDcJqvSJa>&6HaLS7p-$)QyL= zHrO|t75`d41Bp37RZtKR%g^%o@9C5Ce=CjuvVQ-KI#Uw2WWa>cho;jztUt~Le*_pT zkfA2iif9QFp;vhd)|A?tdAQ?9o~?EqgL;=)eKFQ{E^u?OIP}fl^5A;$^ZVutCIqj5 z&*i+G?!Px|5~~6zTYf>~uw*kM`5p&Hju&#w!7^An3*mQwTK22wC7p^OsvMjWf`$MY zLX|ZFV#+>Uq2!QyRD9cgbI9nswteMAMWtK(_=d%r?TLrx?_rkjbjI(rbK#T9Gn}J| z5ajow3ZErpw+%}YfVL-q^{r~##xJ^_ux2yO1!LJZXg)>F70STV=&Ruwp&XP^_?$h0 zn>$a?!>N+Kt$UXzg`e+szB}*uw)Z$uL6?>*!0IrE)SgV~#a?Qgg7HuTsu3ncrcs|l z=sQSMtr}S!sQ4SriKg=M`1Y|bC`XJ+J(YT)op!Q);kj0_e)YNVNw8SI|1f%9%X?i5>$lLE(Wfc$wY?(O985d5e*)UPtF!7gG3(Kd z-^=-%-wWCEK`r4oFh^{|;Ci%W^P>K%9dBNDqi%c$Q{iY#(zbwN7~pQI=SHd%WuV7Z zO?0P;Zc6yeN;)IbJIP0=>W)EgE!76jM^?IyQ*D(T})1NGmP z~YAb6T^#R6;)Ls;cV~LWk z33lcLpbSjxStw9Z>Nv&+rPOXxCGB=?ttZs?{OF7;GYlV&w7-82POb$XrogqFpLA2`j&MLZXr=IG>PAFSb2np~x;E_kV{ zsDwbK$?iYRn7$;mHYZhQn6P2#_hXAHd?;q~!Zy}%;@%wT3u|Sa-!WxxOE_fwyFv*Db@>X;Rl+fK1oP?55*dN0#2%SuikZ)y7Kx>`8*9d?}5 zKvXF7J5&Ey6{A8qUFxrFOh<$xdSWV^dw7z|`7RVZJhAwO72V zRrM_3*wI`^ycl7~>6KaCYBr#WGR>}B)Q(V%&$MhVrU>u~ql zjGeZF&>=_ld$oY!V}5}Gb> z*iP38KOav9RHY)0uITwgz99w- zJX-0BGCdY*$c7pi@>@-`2>#>}c(DHaI62ntpKz z`c01Z#u7WuMZ71!jl7hv5|o61+uv5nG?*dffEL~328P5HlKh2&RQ;9X@f>c1x<>v= zZWNSz3Ii~oyAsKCmbd}|$2%ZN&3gc9>(NV=Z4Fnz2F@)PPbx1wwVMsUn=-G=cqE3# zjY{G4OI~2o$|*iuswTg1=hcZK$C=0^rOt-aOwXuxU=*uT?yF00)6sE}ZAZyy*$ZTH zk!P*xILX#5RygHy{k?2((&pRQv9_Ew+wZ>KPho_o1-{~I*s1h8 zBse@ONdkk-8EG?r5qof}lwTxdmmEN|%qw(STW|PFsw1LD!h_Vjo;C4?@h|da4Y;*; zvApQ=T&=jWU39Uz=_yN@Bn0{{)yn8RZ2&X!<*KBv-7tcWdkF1Ij8D0mU zwbcs}0vDaLGd@xx%S_QZ1H)GTt`~>+#z}HXJTl9S!sd9seVJc|_wUMSdD$>k`K_RG zlq(fsnR@KM^;C}}&vG2t+}_nGPuI5ovg$6TYeMPIREGxP@2r~RKd@>gV`mq0XENsh z%IRZ-ZNP+4#J`o-yRpP;w@;CrSr3wiix3e9Qc|s(WapRq950P->g|JYC$A)$YrGeH zz5dKlAHAPJ>%?llqqB&#+#VU3sp=9>Xms1J;tSYN>LMwNtU68yr!})K4X>%^IrIDp z>SHy&6fJHybwS^BW>okFeaQp6wxaVP`hy;ZX#e+=w3c?PGD&_LmeqL8oZ*YaM1+#S z5WNAKo4+99JW(+qcMjh;+c%R#R?t;(aQ`2`C=bo((ERzgAwKKazXy*0wHN;v;P|f> zBW&?`h#_I^?Bc5GX7XP@|MOiw%&-#?EQ|w+FdCl_&qPN&s$|Z17UCF9oXS#N z)px6>zm&}0osTnCGI;AXsj`q=LpIsW4x}q~70uey5N_NpdJ*Gv^@$g@f2{EB>LP7Y zE5P`jZh1vHNgk7LfMT({jLCjRZa4ubW;UA#%<@Zj?efrPdm{W3J5UEFgm`YkVqz;AMFetZuM5uQpvORb1GDX`WZGwTrF z46+&sAri5QXCfGYpdgonWR5`>ZEa;?jrKvfNvXF<&l)1uU-3q#4X16R2~?P0yg3H` zfw82QWZo^cac+%(g^_6`+2>~Fvy{pOCGnj86+=-!N`GPWAjus1ejhn6f4|mDkU6EE z&u~;xfdRMkj=h;4d~~+4(>L8weT3cz9e@E11EH!tX<IC!@kS+dsIQA`HQ2vdoS zzSD0U?mb1M0@qXu{yhZk2Y6}2B-AvvYg|tRr6z*_*2l*VLiR6G;M{O^Znq~LI%=I_ zCEU{htx&Bo+69G`p|A@R>KlY1*;;!{aWq?Pc0Cu!mT-0S`!>3<@s%Ri;utYNQ+CXDj+LC5<*$4*$-mogGg^S~3JRv{ry zPJzKJg!XKb>P}yJVc^1V@T&MV{z;@DLhvV{dG?RogCcPkROivliSr58>5Zw&&A2?n z9`JOLU;eQGaOr6GB(u{t3!+$NaLge$x#M&*sg!J;m~rRc)Ij5|?KX_4WiM-eE%t8e zqUM7eZ~ZonavR;K4g2t$4Fj=UVyEHM7LPb%8#0?Ks{~?!qhx9)2^>rg8{0npLtFKR zJB)19TFiD^T7IUXA8wt!@n5gj&@OK~EO}MR6^qd?^-?%-0~b2K9RWh+_mSEQQWsLCFOt#JlAQMgNxvv-m z;sF*r;WZ*Wi@I|6pMN+|_rLYKlWwvpKZY9rA;fo8l8hFQGI?4#kt1-r4UL;nPF@{~ z2T~a@2>yD|GuU55boxoIIe_BFo2Vq&rs&2itv|B>OC*bIeOqMBRw~y5KRMwiVHc)` zIBdliiY?Ai7*+k#NZf3MW5!hya~RZ6r7k)b?HF0e(n`ZX=iCpT7St`FDwL@SGgKlq zNnnU*3IcnYDzJg{7V$cb`xeb4(s(({&%f69XMTw-JQErS%?X_}?&y&tvHw@>1v{#R z4J@(=el^kRI+jGa;4)l#v%-jM^$~0ulxh6-{w*4Lsa>Tuc z>ElR3uM~GUChI)c{TW${73A3$vs<&iH;e?4HjW2MvSz9tp9@69+`_@x{Qte^eFo5IlAi&zw$=t6u8K%8JtjRI88PFNM7R>DaCO3rgngmk zI-RMOyt@kr-gVra=tl^@J#tI7M$dird(?aU!`&1xcm~2;dHN(RCxh4H((f|orQ!BS zu;(3Vn+^doXaqlhnjBJj-)w?5{;EEZTMx+?G>Rp4U^g<_yw_blAkdbj=5YrNhZB9@ zNmW=-!yFx5?5aF^+6*1XI|s3lIn_eyh`uv%?liNzSC#z&z^R(mqEYL@TdWzgkf>g1 zedzs*={eJavn{8vF%4nf@et<@wkOPR>NiVuYtESbFXQ;sDz_;|ITVeoW|me5>jN5P z5--{13JT{3ktkAf9M;Jty)yectg#{+9sK{C;2CvPU81tB3{8S5>hK{EXdVe?fR?sd8m`V zPM*$)g$HKp0~9Xf6#z!YJ&g!%VkCMxkt>ofE!62?#-&%|95^)JJ9 zk;GlJdoH0HwtDF(_aTv}mt$?EyRyE6@pm5DG~Gj-2%3HcZT13e)$)z99bdK_WCx|Q zQNza(R)Z>ZKTn8oIdcw%c^pFaMpFZ4HOds!BODgSBWJJYW3I_WJvoEm4xsfs%#LZ6 zdPCk{5XJ>2f7Hj-i*9lTW6BKCIuy)3L!b3(uPoSgW1WA+OEYYBRgSsJq7wjHh%c8ymMs3FU%~cprqL*084p*^T3{J%Gwq`jB30n(&y6- zII8-_r-s5&CVtsoNZ9%On?7yn;oZG03-$wx^uRk9>b*ufh15|HHk|%=MA^ioyb9CYU$7y$4R|M5HvpiCTxKSU`LUg$+ zB3IBl&{qO}agqF~BFM6&11wMeR-#Rkuh_(^j+P4{;X_w|siva$5P`dykyhfAUD%e8 z+{G0|7(Q`_U91sMKFO^rHoCWfXi0$^ev)-187G}klYv@+Rf%uZ&T4-Uhh=)pcU6O1 znXc^c5)!$X+39|4`yNHuCj0wkm+K1VN0G3_EL?-ZH$p5Y*v6ec4MV zS~1~}ZUhl&i^4`Fa|zyH4I%rXp;D6{&@*^TPEX2;4aI$}H@*ROEyFfe^RZI%;T>X> z>WVSUmx@2gGBxkV&nfyPK=JI$HxRKUv(-*xA_C;lDxT|PgX*&YYdkrd5-*3E1OSXBs>35DLsHHp%zm+n0N(Yu{lMo>_t&d1Xy zfCxl=(CNNx>ze+7w)60mp>(M``Qn$aUrVb$cJAb6=Do7VgW`Qn2;v5{9tB)jP$_mB zn{Hb_sMs4yxK|!`PI7+zO68}{Iv)dpu!+ZZl)xuoVU(oFsm<3gT{j2c*ORl|Lt+?dR^M?0 znW6rNA)cR*ci;z?BaG(f(XynY_y+kTjj~T$9{N{>ITQ4-DmZ6{cOkoea9*LpYL{Apo0hSpLqJu z9`tjP&ei;%pn9QY>-$9=<73M#X;qGb+%Bt0x>=u`eDtthI+LWB9CdAO=ulZo9&Ohs2X8GW>b7#&U|py28KTvPBl#Nqv^{AgkVXrOyS z@%3)}$I&mJOYWoG$BBb)Kb~0ptDmBxHNH^i6B8FA7NR2HfTnjP?eDnoY4NS_aYg4P zGGPw11sAf^^fTkY#j@T#6Ll*^GVaPo-1;aS6_a}{r{tWZilzse2m zc?LS=B|EWxCD|!O%|%t3C@Rd7=rKJRsteAWRoDu|*Kx-QwYZQeYpGrZ_1J%mFM;*S*u=0 z%1OC9>kmCGqBBu#-1jVPRVW*BTv%3uPI8fO?JOZD#P_W^V+K7&KVB>hzZ@PdY*%Ezo;}|5Mk`Mo2m*_K%no*jDJGp(s9j;&U`Z>z zO#SEe)k!p$VE-j2xDoX$!;Up5%8x$c`GH$l+gTA*YQaE0jwCOA<*__2NkV){z_u2=4NQ zSk$(oj$%ygio?3V8T3IyGMYvPs`t{im2IoHs7or+>>MYvG%Q?PwOLqe%73uGh6Wn; zo>e7qI$9?%cVVkvQLOLKcU5n*`~qn8pzkdu=Z4#2VnhUy>S*;kT=NqA!dQtnE?wVg zOKobxJ|QCjk`!(2*~5NQx{{=Lr=)ndyn{V|&PxUa=xQXVU?#M24F8H%C*uvs(#Va0 zSkp}0EFYq0#9xp&$O?gIInc#^^_6Ol88W%)S5A@HeE0(SR&!Yl>u=*5JEoUViDR@2 zJBjTsp=Y44W`Nb2+*CcZCkwP(QChX1s)b09DEIZCKt1$q2~;&DJ9!{bQ1Y6&T_9u1 zZM8^im8Wf#FUO6tZqc7#`z0cN_JA>#U_b7he%?cCnlV2&47y5Fc)Z7bp5xGe1zNq9 zl1VaV-tsm3fY=oIX^SPl!P;9$o?**0brq#ShM~3CXhh^SK0oOKB9O>;q3G@ z&4&h$mLSgohc^5IC|H>IGfZvVQFUT>T$|U7{znY`56<5d)07oiv*2R0+-BGPPkWJ! zIOzKF+<5o2YLWP|SGCx8w@<>u6K1o`++xJ+6kaJrt<&0Haq zyUccgxI$sR07Vo9-pF);heBva;?&NcAzC*gSSG9B3c?A;IH9J zl$j%F4*8;F0;H2Cjo*kWz4{kSh?nX}23&&KL+U(#nOAuR`wn@uwUNkWEgb*ZShKPy z`aXTJT4f*Um4`iv2KOfzf-~`#pOfH8>is*xnLBDTyx2Xuc8Y2Od6z((P2AZK@b_96 z#0V6jdw>sEDJ#uNGV|EshD1g&bYZCzCZTZ)286HLHc8Eyy_HPi;d#%;Wx}d6tUUxq z_VB$+898z_{9-A<*v6VI7?(dC04o!8$>DQ$OdbrA_@<6auiBNp{Dw$Hs@@gcybIQT zAU7Pc5YEX&&9IZ~iDo&V`&8K$-4o$)g?wF8xdv1I8-n}1bc7tviIBqt z#iIl1Hn;W?>2&#bU#VZ1wxq(7z=Q15#0yoz)#|r`KSPKI-{aN%l61^?B4RMDt?Vk` z)G#K6vUN?C!t{Q<@O4$0(qI>$U@@TI2FVF;AhSSb5}LtXx&=k&8%MWM3wv;Xq0p~W z#ZX;QFv5G9-i6=+d;R7Dwi)ciIZ1_V!aw;K^etau+g0fOA2HXpV#LQZGzf?h#@}(o z|3w!sZ|&mp$;tmDiO=zef5C|Alz+@@4u5#yZ7yNpP=&`432%a{K#{;nsS!jwk-$Qs zZRty}+N`Y~)c8|$&ra{bOQWM2K7qa}4Y{ndK%dKp&{ zFCvX{PAy_C{xzS_-`0>JlPP7&5!5 zBQ$NQz^z#2y-VeIxnfY|RzU`w+1t6vwQ|wM)LlpuaUzYehGII;>2DYyR|~wC@l97s zgX=f*1qtfDyco%BHmN+o<2qoi`D67R+RM$$NN5-moE4kx3MCFfuip*45nComOZKQf z3!(8tkSdhY5+A%@Y=eVEZkXU3S6B2V-R$ZuRIXWhsrJg3g)p4vXY@RV60bKuG zT6T!enE<;(A{*HPQhae*(@_!maV~AWD4EOwq10tkCXq+HPoe_Pu?d4Kg=2ypcs?&f zLa>mEmPF4ucJ%i~fEsNIa{QmQU27%Abh|w(`q)s~He5$5WYQ_wNJX6Qop<=7;I1jd zNZak`}0lVm+^O!i;|Lwo}ofXuJ)*UtH4xaPm*R7?YS*<&D__=@Kki>{f_Z-XqM;Tj195+~@d;rx zh5pj8oMuupWa#E(%85**I~1Zat-Sa^_R11-CiKdd`8m(DGuzOm9lX$Dd!DX!_Al}d zS!-|}dWG80S;`jSKDH%Uv;-OJNeBI0Bp$z->{_>1KU%h&Af7nns(L=xRN1 zLvOP=*UWIr)_5G2+fCsUV7mV|D>-~_VnvZ3_>=9 z_bL6`eK%W*9eJ34&Puz^@^ZIyoF@%DTun#OOEdUEn8>N9q(}?5*?`o?!_<(i%yc`k zf!xXD6SQscHgPgiHt>x6{n{+}%azrfV4VHi#umyi0;11c816`E??2`$;Rc`)qA2H( z5L|{o=ut7Te=^~@cR0_#cah0?w0Me$&>}ga8xxy=?DDl#}S~Y z4o2n`%IyGjQEP%8qS|v(kFK&RCJbF1gsRVJ>ceSjU`LuYJu%C>SRV#l`)ShD&KKzv ztD<9l0lcW0UQ8xjv|1NXRrCZhZh3JFX_BNT@V|u9$o~8M=cjOX|5iBS|9PAGPvQLc z6sA~BTM(~!c&V=5<}ZIx}O7A;|&bd7vR_y)t+ z?Vm7kb^gJ88g;!fRfMTSvKaPozQz4WcYD8l#0WxQ${P%0A$pwhjXzyA0ZzErH{1@M z22-6b1SQ!SMNyqj_7MXE2cwcEm)W)YwB)ji`3Y^5ABx--A11WB3mBQB<7K!~``j&@ z8PKJ^KSa>#M(rar$h}aBFuNI9sB5uAquDlzKW+hYB&WKf9i&+q$j5P;sz2u$f`uHS zaX8$!@N2b81<<0w<{CpXzQGqSZRpfVb3R%bjsw-Kl}2UH>}1M?MLA#ojYaagiYL!P z$_@7yOl~PbidzJ8yx{Jz9&4NS99(R5R&lf~X_{xjXj|tuvPgvzbyC}#ABy^+H+FN0 z8p5U!{kxOvdv3fr35|Kb`J(eXzo*GvF6`_5GI)&6EW}&OGp=!8n`W0mr_o~Xq-t?% z_pDDfIW#L^DmX?q#mA%Jz-f86KG`^7V|1zdA#4#<=}91g$#@J`gOqMu+7H&yMdNIt zp02(*8z*i{Zu;#S#uP#q!6oNjQzC|?>fgzorE(d+S#iv4$if+$-4$8&eo zuSZJ1>R2HJ^3T9dr{tn+#JMGv#x@&C$EZapW9)uhp0`rDsISKrv`~3j)08JZlP&}HwA!z^~-?Ma(x0_AS{@r z8!(Z}5d8+5f7`r3pw_a=Z`!0r6r4%OAGYBoq3T7^xI@9xG3prNo>`}k>@VAQk>(=DIy(szD&6@u?YVdC|pJLT@lx{=IZ; zIkO4)YWp*Dpp$`H$Ok#yf;yBmHvTb@)4j)jVNF-O?$nD25z7)I!cWQ|Yt zeS<_C{i|BS4HICD=}T(|)@vd(v!?P4t4>APo7`K5RJvcTpr_KgWeB~zMLknrKMgpx zyN-EI%es5e)FNho=}qGu$`98v(QDPUMUGrY4tq>?x$md>qgNO0@aAQLMLr8XD8z%; z2Osn1D>N^22w4Xb8{~fi^i~SthAo7%ZjNb)ikgj0_AsXqF_0+W6E_doOUi0uV6Lvg z98Xk#>IK|-YHx!XV64==b(nYKMEyqPF?D)yxE=~;LS?LI_0)|1!T3ZtLa?(qd|YlXdI-e$W z(3J*FbOe3cSXvDaTHU^Hqpf2i8aH+ZzqY$cFFIH;fxMtW^(AmiMkBtb9esujw?rte zoo&0%Afb~VBn6A1@R1!OFJ0)6)Fn72x{}7n z+b#5gMommvlyz7c@XE`{ zXj(%~zhQne`$UZ5#&JH0g={XdiEKUyUZwIMH1rZTl%r@(dsvBg5PwEk^<+f_Yd~a@ z%+u%0@?lPzTD>!bR(}RQoc>?JwI|dTEmoL`T?7B zYl^`d{9)rW)|4&_Uc3J=RW25@?ygT$C4l-nsr+B0>HjK~{|+nFYWkm77qP!iX}31a z^$Mj&DlEuh+s(y*%1DHpDT`(sv4|FUgw5IwR_k{lz0o=zIzuCNz|(LMNJwongUHy#|&`T5_TnHLo4d+5bE zo*yU%b=5~wR@CN3YB0To^mV?3SuD~%_?Q{LQ+U){I8r*?&}iWNtji=w&GuF9t~=Q2 z$1cFAw1BTAh23~s$Ht$w!S2!8I;ONwQnAJ;-P4$qOx-7&)dWgIoy-8{>qC8LE?LhJ zR-L4qCha@z*X+j|V<+C(v)-UZmK0CYB?5`xkI)g2KgKl-q&7(tjcrhp5ZaBma4wAd zn`{j>KNPG>Q$xr7zxX}iRo=M#@?>}?F`Sv+j6>G9tN!g@14LUf(YfA4e=z+4f zNpL4g?eJK`S${tcfA{wbn({8i+$wMaLhSJo`-Yp@G2i0Yq~@wdyFxoVH$w9{5Ql2t zFdKG?0$ zV7nmYC@PSsDhnELrvd8}+T=C6ZcR?`uapdWLc2eaww5vKtjQQgbvEr^)ga?IF;@1(?PAE8Xx5`Ej&qg|)5L}yQA1<^}Y zp7WZpk%}L9gMMyB^(mFrl&2Ng$@#Ox3@Z6r%eJ`sGDQbT0a9ruO`T|71C;oCFwTVT zaTnu)eVKURM`1QuvrBhj;1e>1TEZW54sKUfx0Z=N*;Jpdh~Aj-3WB zR|EYVGDxSvnjeA?xxGF41Wj?~loVahklw|zJ=v3pOEVZFJG^TvR z-tJN5m;wZp!E7=z;5J*Oaq%2bc|Jw!{|O+*sja+B(0D2_X`c2)nVkzP1S~LOj~xs!@>aN z3$K2^pW}@R-70K!X&s4DHHoV&BmGWTG4vi9P1H$JxmD|t_V{GlHZv(`yJ234IVuSr z~!;~#ublS8qdL8SJG@XRCwWhkZyg_EKH(sB2}QQSv4W}|CT0ntD_4Eyp519d1%yKvc33|`yW9QzeJ4*XLP7@l=td+bwxSL~jCf-ny)IDC^~u5s)E-y^FdtU?)hkN{82Y{Lo)bCWcBOx;Jbw;)Pg9bWQQTY-3RWehpok!>D>Sa2EcEOS@ua)#G3I+GxL_ra^92Y!}tMX zwAp*Fv-aAarn`ME7N#Uyim%ynre6u?KS15L#$#rKZSgLnXx;g8TP9suMpO055p278 z%o-6eT(3gdIVFN}Gb3k$zbTyrHYel1x6OxETsk&h0E?&}KUA4>2mi0len7~*;{Io~ znf+tX?|;&u^`Bk-KYtx6Rb6!y7F)kP<5OGX(;)+Re0Y;asCLP;3yO#p>BRy*>lC$}LiEEUGJHB!a=&3CddUu?Qw>{{zm)83wYRy%i}UV2s| z9e>ZXHzuMV#R1yJZato0-F|Jl_w2sUjAw@FzM=DxH}vM>dlB&bQ!>51aGc}&WAH`b z6M6iG$AyJIAJ7-c0+(;pf=2=!B=%yoM1i9r==Q+}CK3uW%##U1rP~mwjUb8PLsi8Q zq!aTLLYK4HQ$vN1sU;d3XW{oFA{u@1$tduWmdOqc(~AqWq+`V)G&?YOOwAK20x>{q zOgII2&A_FXPzVtgrD80Y5J+_SEmyUcdM2N%q);|ZF_m z)6PBcOcAAy3kN*`8ac%zPH3^61_zn6_2FT#NCOWYx>ezqZzCC;tzM%pJC^gFAFcTs ze6C3WE-a*=nt8tErPG9zfPRn$QHqB7aHe8x3w&rWT(0F54<2uBJDYtbB}y|@9V6T( zmM!t}T5SuwxyTCma14&l|yiQRw5Pn|OiDBkx z?4tUGrIVsC9zs=F{W>zl9XeknEc+~Mz7zCnefUPUF8iF?A)QJK8=84#-TLLxq?BTM z=VYjYW%TOhrBp>3D@K{vStlEUt%e{HRc=766AQ+s7V_F|1A!)P3?y*=gUgbZO;O39 zX*BC((-XbnoaRGxxhRQRVKCDG9|qC6?7TwCz{A{OZp$Wu(~0DFo(w^P3f>4gr8@P^ zl8`!vA=_fvwTZc%-Z42}m>Q;KQ~&v;ipZzbA2;}Peg*v}TlKRmU%4WNN<%qb!cLo= zoSx;XBrv4}ErykT!)z)Qar4o?(q6!mpWLNFe~Nz0S@yI{1)Lxt<0K=Q$~>*HH+Wbp zQ~fx0aup_lZb|e6*@IJOJjw~Ypiwdq69&Y2vthfGq6u1!Joy%;v;~4`B@B*S(}}i- zmZc^*aHOK(dd(geOKg)P+J4+*eThk;P@wRjvm}e)h|#EpsV9YoqqRW{)ABhRlvGA* zL$&k5w*_-X1ITCwXiH=)=5lzjxY5tQJTBrv<{dM7$98pdK%i;RGZtiJKaSGCji7w)aNrHu_9_IPGHS-mMN5AheTn_ia^YdunCzcp2ap8eI-RQEm zj(q7_CT)o|w_noPm@MVqIjv%H4Bdo6*9*!Zj)bLx!p9POp(`$dj1QW`V=;=|`Gx8QST=OnK5jlJX3!KBz>v7j$&5b5YrhIArRVL)1C^o{@DJ}*mk*s=< zDK{e2f%fG)mK_Mz*x@#ahOO)cQQ#VH+8Wef>NKWcu4J>PIc3iz8y6PwCmY|UQ(O3!B;HtsE&jvyv^XjL7Env5#i zH4-k5GzPr-%36#%+Hvw1*UiOIk3b7F^|1dPi!-i7C^ZWp~_KI%D!sGYb@@zXa?*{XfjZ~%Y^mT!kaK_>K8 z_jL78^ zS0eRdqZ0v~WWow1CE;vDBh#{w9R4JgB!})W9N{{D=p-RMnehZ#pH*ABzDP46ryZkt z4ek|LHS{CDhTTMQa3a5fO9OLg?y$+#Gi2}Fv>QD-+ZEQKX2Fv{jr~miXz1ZpPcXvJ zNvQT@kQbBz_Y4Kg)*`E2t;tPh5_7tSGvL-|-A`lgHX3uVG4jLev9>YCZUeNNzioL? z;OBD{z+=Gs3+*ph)#bO#7IHl|rOFfvpK%cF>W??Q!Nh&B@hByD&}g|>a?GJ4uhX3g zPJXKKAh&zWv&wITO66G{PuGLsxpWSqaadFsv>_vQt?LVslVob7wylsa+O`IYWySoO z$tw#v7=&7ZGZqS}N!c##5-bC%>ze*s0H9J%d|!JgE#uZ|k1_bAn*x(Y%r{c=(HLwNkPZOUT#@j4{YfG#@=49YJ{?7? zddbK}G-@Dod&^Vf`GOo)G|`n@kq?Z=o84x{889+?F*dQz(kr@9lQ-TXhGN`)^-Li1 zb}xO2W(FvB2)EA;%qAkHbDd&#h`iW06N1LYz%)9;A&A25joc!4x+4%D@w1R+doLs= z#@(A@oWJq?1*oT>$+4=V=UnuMvEk;IcEnp4kcC<_>x=Hw9~h+03Og7#DK(3y3ohIp z-gQ$-RQIJTx%0o@PDST|NW41VgAR?CH`Sj-OTS0)?Y*M_wo|92;Oz)aya`^I0@?S{ z<%^epAw!Tw(bvSmU_k~Im^%#|0`Xkcmxj;31jX2Gg?PbzdXp9Dg~P)PW+Xi%iWiCr zV-Vv9IR5guDS2lGV!lfTWxkD8w%yz=UB`2j2Zb0eg~arRA*Q6>`q=8#4&OC|L6O}8 z)!w(idG0yk-BF#~k@Avk>an9z_ibOP*Rb;db_PsakNWYdNoygT?yRG=+5>ud<6Vxhk?P9rk!+8?xMg!x5kD*f2XOd^`O3U zlO;ImEy0SYI_J05cMW{dk@%d@iZFCNhIVtOm8$viM>=zM+EKJG%c0)dZ0D$4*-psQ zW+Fq|WmbYkBh5|^-l$w-`Uy8#T#<+3=}z!(6RadEpFlr1f6OFuQ5sG735YicWaoYR z`wuEZT2dntHGC7G*Kzk$tsm?Fd25LTHJj?Zo2RH;9rW9WY1`;@t_O3NC};dayX;Ib zgq6afb4!50qL-o5%yzgcR-1Xm-l4SE!rE>o!L=E`Jeug(IoZ36piq6d)aek0AV)EJ zaha2uBM!>RkZHRN0#w07A=yf4(DBmy(IN6NdGe$?(7h?5H)*?(Li#GjB!M{nq@C3# z^y{4CK_XQKuO>(88PRb&&8LbRDW1Ib>gl6qu(7g}zSkf<8=nFPXE1~pvmOT3pn^sa z+6oK0Bn$TBMWYTmhJzk_6)$>>W)nF^N$ld9 z8f^Y^MLVz@5b}F0fZID^9%hRL#()Xw*%yhs&~|PK|MGI8zuO!f!FqbmX9icd zXU(JOCwac|Z|=Yr(>Q3)HsXl!^$8VSzsgI#)D2XkpZ2=WOBcFF!2&d;*nF%h0I!`mRHl$91jYzqtLfNHUoYzrMzjR)u zP_|Hti4^){G?Ge6L_T^zVdS@KHwtq^+*+aBNl=hVc6#KB-It()qb&8LhnVW9Yxn&S z&^s^u1OzB(d_ByXz=xm4cpJzNzV+Txh`~H(176n4RGlY6( zg?ed(a!J?4(oL}@UfBpgPL*)KrGtM_hMIdu!RywK@d!b-{YAY?(?w3yB@Fi3g|G)| zho%)<=%Q$Lo7S-BxEjTL;M74{y+`Q^Xg#j}VvF|Y>X7s+Ps~aqT--tJNd9U6;Ej&o zj@|!`{Xy90t_Zdb>+m8tCFJ@X(Y$mR>%)gv4Vt;oGr`idhQ7H1^L3v4<_2}-UoguorcscRfdgumUVa0mK7-Wm~#vbrnX9ro}@82q=9t;lM9nH<} zLL#=1L7*f+mQWfyFnETMi*fe8AI+gdY6BM7CkRS&i4$ZRv$v*=*`oo>TjZ84sYD&T zI!DgZ4ueeJKvjBAmHNu|A?R2>?p{kQCRy zRnGg@C%oB#-;H-o-n##G`wcPWhTviRCjB{?mR20|wE9Kn3m6(%Sf_oNXWP^b;dz7( zb{blETKwpl`AT#W7E6T|0*bl?%r{}-BYdwrn0zN(DZXM1~53hGjjP9xzr$p z>ZH?35!~7LHiD7yo7-zzH18eTSAZjW>7-q5TYzDvJ$$S$Z@q)h)ZnY(3YBl+_ZK~* zd6T1UEKdrzmv2xc>eFj2^eQPu;gqBdB@TLqWgPk|#WAS0c@!t08Ph)b>F3 zGP}9_Pfp;kelV05nUfnb%*Oa{h;3Yi^B5xyDM~1r@o%v#RYi-%EYfSYY&02eW#bGb zu8(H8i9zhyn%?kx5Txx^6 z2i}CK(HeQ_R2_u?PFp#6CK zjr}k8Cx#C?DFgP`uN<;}x*Gd$-JgG3J_i3s>fk@_Po}b|JNz=Dm+<{^51m=mO;n4B&azYm{>+VhB{iyxuW+j>w@>VHcJyoSBQi=hu0;p zPw3Aj?%Ai^UeD{ySPIqsf|v0L&f_fmE7oh(s|jwbkK5^AQ9F|;a5V}EdSE?fyxdgf zHTq!f0;+-V{0oF+l_~>rMGk?f~m^wDXlxqt1@+)6Zv?BNR$+%$i z*NF93f}~4d9H2C7@?IibyqUtLL!XZW2ap4fkkxMqDZuZ>`+AfWJQ%~O2WR}NoA=OP zieg@q!mP z?=qU=EE6L0_UpzXt0qwX2tF~}c|;`#MUY2TMz6k({hpkiSz>Dxt*4-PtkAdAA*0hn zk~CK6#V=*^m5 zg$tB6rSO-=9l>GAl^DjJBHdk0wD0(L!OrcZ?qmtYbl+}s(@rtE-O=RTx*1cZq~u~5 zQPVt(IB=*?Pm;Le%#i1SFxHY|>=Y$^RF-FGAUSkBpn`|+p!4RHyv-Q(XgZ5Xg5W}J z8RcT?+4FdVQ>z~9kP5By8eM95f_LDnsnA%K;i6`OpcuJS=^n|6nH-B2EhH=dLbO@Z zuw=Ug>7gsu33`Pzy3Lji0x8OCH={?VRqFEi;@oDIS<*?dG@9X1*tlYCm4YUIMhyfo zJ~=K@-X$D z<-4dH<-5o#yMj%f@U{nfWYVdrREJ}_o4&|c*_+M6gk z-Up9-i~jM-bwR;Bf0&C5wteli>r7ZjGi+mHk3aC4mS5 zPC^{w+G%menlWun+&<#i&DJ41thvk;OKZEB`S%sZ6 zzYpO2x_Ce@fa0LuIeC=7gRHN#os!MQ7h}m9k3@u68K2$&;_mSe2`>uvV<`RgC)TKX z`J}&Kb%*f{Oznj$%-QafB}Zb$Pi%@D&^ZTcgJ0+Bk6-iOJ-P|Q10)5ie2u0JzKb2r z2C@{f?ZBcPw5%h&aKG+6%Qvhw(t1Y{hZ82YE4(Tlk`2VCgE&1x;AUt+5U*$%>P|iWLeb_PJL!VX=b4#>#QM;TGjFHBNRy+d{v>2cVXFyqaLd300 zFHWrc8lB1KSOH3dkJClJ%A5oE^31WrQZ3^-3`Zk?1GqoV7Wr62=V9C=(;#R zhzXAT03)d z9OdZ|;CjSnqQeqF-CUNR=x9x76JYnpr|T+6u#$y=7cMVG72k4f*BJIG>l1NNvyv6NQzr4U`r;= z&%W1Ri2sI5p|8%q5~zM-AMptHj_eX7FzJN7t(%+2dA)efyFbePBsClxY_yMqWbEdT z+jm?SZgH3mCzU?e^psnyd8UK zfZ$^_^}C1WYB1-$m4qwT@#=wsAq$9Xj=%IRvc#V?1azEi|RSc;M zQn;3%Gjk3D)R+3`gZplB>Pt;g?#EiwRzxON;% z#P5IK*YAh1Md<$o21R}j^8Y#t#`fP`nErnb@&CkI{`XNXulcVIXwLcS%VE4i4-!8a zpj-q)#TqXkFg&z4G9pG45A-$B_Lfacr)H85ge*yqTLAb(oY1$6Xu7Rc%^aVOmzsKd z=WEXA40~hm@7FKD9t14nSRt)m0XWkP1YbAE009nIupf`md=v&J;C}estaY0%^Z;;lf>5AF-y%Xf1QEK(}4n+ zhKsTx^bQSpwM=UWd3WRcpEQfw>P%zuhLeEdY}s%cGitMZa14Ui*Mzm%=(7<#b2gHmJ?kdeymT7H+Z8k8tgd zp-dhC)R!P!)w(n%RgOi%^)LGZX)yxC%@f@d4x@IRbq{elrCHyIuphEE6qd6l6O`;B zi0WQg;j`hcu51uYTBSSYNvY{Lkn$iu=Ae0g6o1cSTRwXmEvNcNI zv;)Z_?g>?aG`Zp}*gY8%LGI}{>J#`x;v=*ykuY@z2Erz>@b*)tMp2>=C20MI8|{Z2 z9hbyDJ7d#MdWK&fyZB>Jdm!#x_uRw%>`OuM!&QMim}baa76{L|VAuq%1UpXVHsClm zPD4}hjj{lj`)aaD;x|PJ9v@?8gZ!t5hER6!b~HJ_l9P|(h&R6js3mAfrC|c+fcH^1 zPF*w*_~+k%_~6|eE;-x}zc%qi-D-UpTcAg|5@FCEbYw6FhECLo+mVn^>@s-RqkhuDbDmM~lo<4sa`|9|$AltN_;g>$|B}Qs zpWVSnKNq69{}?|I`EOT~owb>vzQg|?@OEL`xKtkxLeMnWZ@ejqjJ%orYIs!jq3 zTfqdNelN8sLy2|MAkv`bxx`RN?4Dq{EIvjMbjI57d*`pO?Ns{7jxNsbUp=rF$GCut z7#7Dm#Gvh}E8~2Tyhj2reA%=ji|G6yr%@QV{(90cE{JYOW$0F|2MO+TM^`cAu$B7s zmBV^{IqUIbw5~muv}st`dDdIxSU@Eb>xf3$qwEcg;H+vp1^ArN@A)RtQ4hrid2B{9 zb~pG8?SC3#xctpJXWRGXt=cx6Cw!IqoJrK)kuLL&`UYYB{R6Dw)k9nKy>R#q_X|V* z%zVsST$=d(HozVBc|=9<175^~M$v$hL9azT^)TL7BIA#qt>N2^iWvMQgt;!YZt~cv zn!x^OB!3mOVj>^^{mloGiJhLI4qy3Vt-148>9j~d8coH)q|Cg5P89Xj>>hjtzq5iT z%go41Nhi}x7ZztTWj|deVpj>Oc#IrI{NxIm;qhnuNlvNZ0}d=DVa}=H0}Vi-I+wKK z*1uD=0_)b-!9S^5#(%_>3jcS-mv^;yFtq$1)!wGk2QP%=EbpoW++nvbFgbun1Eqri z<%yp)iPo|>^$*IHm@*O74Jve%nSmDeNGrZ&)N9 z)1rSz4ib+_{4ss2rSXRiDy zgh(descvk^&W|y)Oj#V@#)C658!**J#=ckpxGniX#zs0tA~NG>E#Hn3Q3wdKBfMG& zK}2y#|FLt}E`UQ6t3jK#G&e22bMBc3=C)LyqU706frdCAqa;~Q0L5)KJ4?@h*FFu4 z!s=hOC;G?Q)BRKJ1q_XJ9W5LLejp1L*187&5Bo4Of)k>T=WpQl3v#4iX$574fW`p+ z3m}r-F8Gjv1m3yTia=+2An1+E&psbXKjH2{<1xMb37`|D<%7c`0`~m0r>AQD^%nUJ`%PxS>)*{i zg?VHw)ju!$@$>xGszUyM_BsCF3*%>rxVZ8vrYB?PvDBBHQWz04T&UpxKU7{ zrb~8R4W>e)){FrKo^O5ts8O^r^t70=!se(2-(8&aTdaFU2;SR=dyECLBp|MVU@JIt z)z$TAHMKRnyX*5;O<*xm+(>Fo41G;Tk0w01ilh#uFJa{teQne`QCOHZp`&du5gkAWr@9Ywz%@P@KB0bD{lXo7PmrPC%J!A z%orlB>F}qRa$`XC2Ai_4L56#h2GWm;>sScPxhMO5a*guk2 z+56H}PZnq-sxASPn!B~W#8B1W=OQPf-lEbhOh%>%{AND;w%w;t<8%a%HNk`LQ0GpT z6au2l)=Brql2Fq{Kw316jHdW-WF<{46(Xad0uxi%3aEARVi*dKaR^jjW)$<$7QEiF z0uK-~dQ@|hxT5M|t$pBl+9IJig2o;?4>qY%<|sZ4Rk0Dc{ud;zd`g$&UcwLjY))aV z4jh&lc(;hjQaWB)K9EB@b^I)LQ~N_;SFEEWA&}`)g!E7-wzF%J8)yZaSOeR=igBiM zaU=T>5*oyz3jYaqv-RSC;r$%d^Z(cbLGwTQiT+3KCMt*OBOD@rPZ}8;)1_*l<5aBp zjl{A?HiE$Y6$NWUgPY(x@k^9)A|CC#nqZ?B&q-ceGE;Y7F{@0{lQuPnsj0~YX(VoZ zdJ})6X8821kH4_0vt$gocDeSve(SuROm_bM98&+q72$1m(x?A;;)@TWyuVXQV!{#( z41CN;(vq_a|56Yny*sb>5`lt+>?dvF0++3L!wQ_eJmXi)z_1UAmNi80_bG^|J$GZs zK^|0X@8jq9pyPt$dpiWWAG)mNg7X_BME=&UYoq>nc0gtk_YoXNb5hYb!hG ztf(P(6Bcy6`wroiv-5NLLjVBx&|;W6WwKMmB+ph%7$AJfV95||OktlFlTMqdKP0i#Y*rj`(XeYUz=adk`3hA(LvO`y z|0%R3GMWC#x}RbCNX_Cf;_wEOS}%lqj#-CXQDIpi8Qis%Radz>q0vjbY&8DdR>jXU zmvR%au!=9lMN?P=hzQpNGOJRw?Cn8@B@kEp4r5$bgdM0?Fdua~*H~mGTf}17rZog% z!Kj#>m=l>Po$A`_fcT-pHy*aya+n%rXmG0CJ6a{nF%>TfyzKC2Dit7a;!8r;X^G$~ zS03MClV}lI)S^Py2I2rLnpjR64L!#Fl!mCP0td}~3GFB3?F31>5JCwIC zC~8VAun2Z}@%MZ{PlIWpU@CJ06F_<61le-_Ws+FSmJ@j>XyyV(BH@K!JRR^~iGjAh zQ+NnRD1C)ttcyijf*{xky2tyhTpJvac8m%=FR-LL@s>rN`?kMDGf2yMliwkYj= zwEEJ0wlFp%TmE6|fiti_^wVrxJ#gh7z@f0+P!kS>c>;BHH)N`PW0JHTqA?B~fz6H+ zdQq>iwU2Kne+4kR2e~l2`>(-^qqujX*@|w7k>s=e)Y-lwoI{$Tx_2}&y$9LZzKG-w z{TH06d?a9;01ze%EvqDCEt;qAaOYdf@X)zT)ScQs**7gQ**A5+o9p#P*X5~lMpNl2 z6p=Ecy7#f++P2sk;I2Nd`w-!5Y^3QHV0RVy2<55pqQ z&Q&b+JIKTf&6N(UjwrECT(BwKhkdpc#(Aq= zyG*N2frC~4B2Ko7O)bOHP8(}XKc;_(GP&+{?#dJ;Y$YXT$y<%YZmc>C?Sik?i?6E1 zk~VKGMLlNws0d#wk-11tBrAf?Tbes4F)oqxr_*7R-?Yn4IlyyP_ce6(J&tXSFI~P^ zYG1K1&Y@OY%nE}Gsa8~iq!!=l4a+yi7?Rxi#owl|2CnVfey<;AkI<2^CN^r`;-)ob zX7Ccao0G6Ic0ENcm7#3(8Y>}hb9aL6Gi?llW(Kss_CW07Z*0rgVhbod7+2-z3EC%( zq7QLJy|>bn^fyDVwISg;I%*4-lpnL5wLoe=B5sV^!Vdseg%7piW`#>KU*HD}MZ&J=jCFG;)9zqX;~A15Xsg;+mAtJruykiiD4Qc5$;lWT@^-j>F$$|0*{U zmrM6Kwy7I0>uJ&DC#8>dW7&)!1!_uGQ@Mvr)n^bH?_w|*J_E0?B{C&x%7+%$9&Umb zMv=?f8jwV=X`(6MfQLkyXGt_A~#T^(h~B7+v?~%F6k&ziM^m_Cqb!a zf0y+(L*8N@-&FfWsxPx%V97(F{QW`L&>2NJyB_}HBTWa|xRs*TT-y}_qovhF=%OCJ zf)sDf8#yYtG3ySQ*(qqz9dXI;CfS6yLi>4H9w9ii-!j5NwHL>oEN83>IsEP+V_1~u z`?}q?(o8RjDY5V?z9HC@t*0V_hFqA|HyZ8k)T!UJQ`KEKMLlNlIq<$2s!x;)o#SW0?w*zVYU?yc(v(2qyZg z0(^T!7Qzhpm)`?PLS7z|(>s+ZUO?_>f0y8LjB9{7he}@4-%l99L!vhyLW=yQr!);4vCSd-wC1QX-%H=?#UM-D_Wg8t3W z0*rY0Q4xwb5i(lBSOs^u(IgRSP$j!PkhbcIr^rh}e})V_kU5jW{q)m0CALP$`wKi& z?444cDxl;D;SqSw0^h%eA6Ro@BhxmD!}qpGb6OxRi6;iFai!)ctW|gmF3jQz2*O}Z z*TPvZAxFr1-Dd!53U_WQMQh$aauyVf;O60e>&G;Mg83(TOZt!6;s2KT{}By>k&-_m zA1YA0q3ID6fx`!qxy=@dYO@Rn%rEb~7P_%;Dxvl(WAfiJUtti0?~ah#_1`K#A}P2n z7^D~GQL#`hC}2w`btD`i%)VBWnn*jWF=d!kI*6T5-wBdsT)$EZD=mrn&EhxJQ^3>1 zbLeDA3&BIDAv=kWsp0t6>a3lITA;khMX^(B8Ecb^U%P-|RNGB@XLq*Q5a zR9aZ8RFNDYvD`dcva-5ti*`CcV%ltLG;emYG)5Hvo^Boe6!Fu0ekZ(k<<5G3_4>Mg z-?ILGT9yB`Gy?Cnu(PO#(bsKyf9>@F_MJQFZFaBE?dA7x40K@HNwA20g&JE&q z6&$MUcmsL)Sq;;@a9!*!?ct(XynVCJutm{pZ5w3Xci1lQ!9oB`xCdL! z6i6sX5X8iljX<8L4KC)P_hyjfBo3W=8BfQ5^inG|_NhXI*k)fvrDRq;Mtl#IdM%t^ zo(9yQnnQj}I{C__YBGYykMvG(5)bL%7>X@vm&+vnDMvZ(QMVC;#;@DZ9#6!r74JA`7phVA#`JE` z>BU^K@B>jj8Maz2m^>t$!%J^m)e|Ylem4L>e=OHtOVBCDy{0or$Np^VjdNl=g3xT8 zqsE*&O{Q9{>LhP;F2vpR<1t@fO4^Fbd{cO753U@l zLFAlS*(cze1w03?ZyLxG9S&n_udo?=8ddzgt#cv5fKd+uyogyl;44IK1&z^wj=!YK zzUD&kgK%`pt9A4nks?WMImECKCAt*xUXcPbo9e1&PmWU$X9~!}HO|j@r(`+=V^^Lc zcLMKF*Yj`EaS|pmb1uaDbkZvx6m%4{=z+MdgTuv?mT=4T&n?h7T_tQNFYhz$`~(DF zx4T%9nS-@(gWPm3?tZwJIpHDGWzAJ__zZKP;Hw>~%&n=s$Pn?6CaJ>bJzY?o)(O#~ z1fxWpkgP7ukZGyitR1C364Jp*?#{WzBom;9o=XrY;V#_Y5@5*}T5v*hcW#I;Sb)H; z6^g4&{fOcGP0zWCURc5J$ExdSY5s?r-^r#;|BS)8NjQH2--6b}!Q-Aa$mx_pNnz4q z(1_zCdqOu|4b4oo+-*jjTTV_j3WmL9=u`0(l@>00B5Vg?4f?fqwWRCX*2JwC(Yd+i z5A-Rm0r4e~4ceSJnEmWF6Nk>Q;(7sYyQ<-CgPa1fO8m6_pu=Maf0e2hd92Q#i7j?U z-VR;%F~r=@Xs>J2`Nx))UK=X`Shhg3AWzbwE<#%hM+KSQ)y~F!~7j*2}qu zgT9Z6kE4Z|n9Leb=N0%JnFI$AeNrV+!>E(WT7dyOjN~44BhNVL4(%Eo(1JGjS^)Oc zjSPsu`3wT8k`$>Na;G3pMU(9;+ov}PpiRt6*)WNMy(rEUak-14^(K`73yJ1#LZna? zS)ypsH=xt_ z1V%Pk;E@JqJeE1&xI}|JylZJSsu+mw#r=)G*5DBGv*`Q|1AC+!MW979QEZ{H5*8ZW z_U8EI1(M1LDjG^#yy~(OGH)?SdmR~=ma_^2Q#k>)`v#$t=~Ih|79!ZutXQTK^S&w` z1)ONotPDL(cz!_@bFBBOo6W@;7Zz--d9JaOs{)ss4P|Mr%>FaiMR=(fn-Y3SA->6~ zp`5h}dOcY_YfweZB*^el7qqa$&_r-Lg-I+9~U z`JxVCD<$VmoiR$g^3dU%7Sij)XYi*?$#ihSxCBHGOaRRr|Lo9+E}O~M>I}tnokI`}F32Aty#b8rpABEKl|B;*o8ge^^)Kyk z0!(>gFV=c)Q2Y%>gz+sa3xYTUy_X`rK5ca{{erC9WJ3EPKG{|Nng_-78kAD{oh_=K zn*wopK3cG}MBJf%6=}9YouD;zyWbjRt%A#pWc1zb3@FB`_Q~~UI!uvse(FQfl zUt=Qy2DSjwpzAUJ048~^;@Yo{C56R_8nZEeF}vm)0xoYe0y|tYI!>Y(d}mSro0`z; zeb6Eg*(a2{5Ypj8S$-_~L)+IlozZn|Iak`$jQKd63hldhts0=m>k~HC&`@|~;XaG6 zLVxC))8>^?13P*mV#ydlkC0V6AWK(BjWpqu| zbh7#bkKuL<kv5;Emm4zkF;X>rfbzAc7!Z)i};f=*bypYUD zho5-B5n;)FP(nzq8FG3TH?7l0vS{G}G9@~zxY>CqbX^mb$|JncS3I_2RD@?I9bz>LbX13A0N_LQmd(!3AxqmR_;3bJavc81%v z)Q~pDm0d1VrVe~>X?GOUOz94e6Nbt|fe6(S@cN64Gy6{i*TPukTmfvgPR>+qe>)@w z8mS6=rvR0~cqVfEWFsL|kZ3t~m-iV}va(IjJ;Hh4R9uISa6;@9d{D+7CwskGx!7MGZ6|rdE_I{cMD}-` zoi0%doDSznN-Evavf!_d@UNJt*Fl;hNrnVT2Fal8iBh(LU^l>8I1%x!q=6A@zO6O} zs0R@~z(6E;t~6L7tclb6A}zwwIvS;W`?F>>P)INWt6N9r4JbH*;&^6B!lHNAY+v3R zwCVoTTSL`1XtRZ_9vWH*(HcV?PImcNBOtbC4{U(v-HA~xMdpP8<);Xv0y_e1i%t|f zdyL`MtgjoC^Z-wGt@&6(9Wx>;qYcYwopK7H4iejT?T|>BSm)-fV&7yB;ANW4ZRzzc z?^;uh#-bDq@QjjBiIf-00TSw~)V;r?BHNEpDb(dLsJ_Z!zT7<{oC-V^NTEs|MeD0- zzuH~jmz>@&JaYIW>X&?~S>~+R!;wQOq|+{tI&#vV^n%|7ksh!vXzONlSb4zc!X;}> zMaUjix==sr4oMiHxL@~MPL%PrMzU{DPuz`9zWln9XnqKqNo3TZc;22OZ{ zy(90FLmd!qHIv!b-q){c(0@VYnzE(k5#rf~N5m{u-X za_J$`vM`7Bh@_`N%&n~35!O^m^pyWGR65?W@EH_fG}veT4I>@L72iny$1yuwBopv> zsSxe4Htw2+2f`M-+7|iva$OjEp*e=6r{J`{W_IyMTo#x0Yayp+V8z~17Hx&~6G%t? zN=#7bc$BWFl&qzMvU^iRl>Rvj(_`fR9T%ZBYX1?fg((%9FgbGrBl_7^rRQW9GA*@E zLN~c4F@W|oNmH$kHZ)4U$u(P4S;GSPDy671d;6L8z}?RfSb0PHN)PsKViOm_PLB-7 z+-+jjpC&oGWj(BQ{|L#DFOC3+-%fvGOOx^u^Ysxsq)Ox4^;}rM$!;(?`m@wtkXb~%u$Zx% za#IBD9hq=no-2H90jB}1^>TfWp)=Sb1v9w#UAHvYbn1PpHFbB+hwSXWK(ta=^8VN< z^j!PhT^ZXf#;?$ZWkn?(vJ20u-_SsGO1os)z;s=hI)d6iN-4mC9>EtcU@Mybflo@| z82lRHB)FEu4k@P9W+a)>t{^Jl;)gL&tWZBy(gWmfXX8XiUdnU>LtbceRd2RogiprV zK3KHRpSd5n#Hy5wQ!-Fg;{(9?K%pRuAEZwPR-E)JGeljq?MUmP=K$zkEO46*td&DL z%C4c|+^C204zq3rsTdE?%Y;lc1vKitClZ79P)GU-k`VCL5(kX_>5D{)C18r$^duj) zab$~pZ#$FLi^ihhytr80x6p2DsA3IsHPguaQ&s4izcL;7qGj1rPQM)4uc!I=d^j7S zs{`eqUlX0}s<8@_Iij-NBLD<2BE3VJ&k4Z6H;z?!7!7-XeeC-aX{Tl6ml!93m*cFJ z#Z5Q7fr}UC|2wXN*{|KEWPZ(V^*agnsVlrYkAd651IAl&yHxt9OnMCJBht5xn*lR2&NabYN zSWC^|d16K9!d@LjLiX4uEhz;%>2G#@i;bdI;t=8bK>y@P)WT!mDr~z}pG- zRg0M$Qpz0mbKF!xENTw8!Wwu{`9|04Gou}nTQ_L@`rl58B6UT^4~-?*}V`fYfKSaDIH zavlsK6XsL9-WmdH$C72oMpwJp)?;)Z4K6Es0B$SXP*QhM!gvpdUyI?}p1c2yYhY~r z_VvRqI~hi$_97U@cE5#Z{Zhy&EqB*`vAMpf?Ya?h{;uuk-}E1T!ah4kx_Q*9mOjl* zv62c1x-eMCSfQ*b3b|P6*~#_2>fN2y=iJQy-I$q_TIV>AHLGvxzY#v#{w}OBR>mny zZ+4AXVq%F7d*h&{U!c8&&KUXS@X->Bu@pTF71|eeQVYw8ns~h`7|n?)2@d35c_1Jn zeG)5*kFZ<}MejgYN(?7Nw?Mod)k5v*wm{$@osr)Ywv-QvXpeI;3Qku^T}zo`go?co z|65!$tORilITCe4GfhNoqaj~NtO|@obiA%Tub@&qQ)*Sn14oz#=<2osGcxe*+@PL< zyx=_nR&*Un8g$Iu#el1FV8xS6kKlqt6Q_nLmsoyCCicctlpM=xVMApO3V7u00mxNJ zn8H5H7~1cY0)_}KJSfc2QSG+HDoQlkX^Iwi_%Qb4&1XPlDw$%cwf-dlhzTK+<_D-) z&P@=34aLr)@%x%0WcLNFBZ4im4biAYc zX48#WytT#YP@@jEfGgaR&J#HZzJa@HjxyMYHe{pLPnxkn;~Nj*Rk*wS5*frI0o^@# z&G3U*-hF=Y_v1Euf&ZeY$+hsoi~%M`iq}OU5nnKjI6qCo7#tk{_f3pIO(8(pMmgCr#+;(8d(-5n@oY{gBKSFB;sfY zEGd8%M6}wgw88w$*dURSw+YzI2N!gycd}~V$*T@AlPt*-f=web80-YsRGL; zIurEoITNgt(oy6p0G%)TAq})jmI~qDOTd#8SWUAuE(*k}kk&NIGfR#?MWZ&@WgOiL z>$#C7>im5ft}NgVUz#o-;GS~3h`u>vuPTQ6J_?slXE&+uSm7V8X2xqGN*g32wQVF? z60uDVd}|BtzXW}IHl+O9$Y${gL@oN<={bc5POfF*UaM4*ulAX=jeCFG9716kCF{ap z+Aa!D*;gIV6MjhUJ)8P&!?O}G@h+kF9lXMn@bE1hm7VR%NpI0p(h7q@gb zs40V7?1#wanDpa((WWtV447#&s#OHJWeK>i<+;H67mI#8cP#nvB-$#8&oY@Q_cX1> z#729EG?sBvSe1t$UC3o?5BSvkVN@w(QQ4cW%3w&{E71?HvJrUEs@C5uiGi2-#9RzC zw0R)RSq1PMNN=!DdusVZwDksjyaAQbNru6UwUWxld@ldSWo?0&)`;Xs$LTI|<=N_s z*4BCzi%Pnt37TSLENizfSMFGy!FQt!OTgaGufi;Y{r$=cJS)FXBg|11{Y)6 z&FoDw-n6}+505Cb=XILmcU3v0TbML}3&IJnbKY?t6@!3@-XG)E17_uq1tu zz$~wy7yG89CHH-vtG}q6Z~ttOmW){@%R~RrHPL3}aSux$jl5%aPq}sjvD-AQns@b7 zY@Oc;tRc(`c(&eQsK@oDdmBD-*rPabNn z(VZVY5nz7{q0q`4KJLomsMOu|s7*#%-xXTM-Iq0IbER!m(6>i7*+fAfS`~--GwXqM z4ca)XqKhhrI<(1CRvrYaF?C+w%ux-FklJA!x)gsK+>>%M>?Cm`XxbwUj;EAE@Q-G= z5cFv(Qwcw7h#q)bu5EK58r1nZ6^FodqAYE;KnPkOE*EDluO!khZFyZZGn4S2qu$k&M8jDj8T_CbL0QU?r8R{_G)Wt1$pHq>0cP3sbJb9fA#aCxY+I-RDFonr20^=HoUCZRYU z3;Wx@Q{b+BZ2dl{1zxcqS5d}TP9^VEZo``(0%P+4>^Ho?uXD2Rd}SjDvjSCkh2VrA zKWEMFMooUWGVS_sQoH(GX9QMhVu*UMH=Y!B(2b48^*fnH@gfxbGf<8rF%}3qZBgv? zh(JU+*63i>>V+rSOX()d6M}awEy>N7L-;9D0cY+eL%cJ})#Owz>4SDuWjsapJukYm z#U|itkDzOryOj(#d47LERC;) zr?00mlOxu-u}_c>)3d=1nWQ1_>F0k02%Z<)U=_eaKsaOFH4zrLYa*;@;Akf7-~g~P z1n-xT%i0(jSUv$dfNPE!IynMu{+t&lDe21Kfn)7m%JJ%C)HSiGPUMys&0o#k$Pl1AFx2#-J9Qk{BW?yJ&d`)AH4#W6I1ps&M36?pz z;*EEoPlL}Wyd}~t&>61YcyLUW`L*Z@r$ihqOO<>>P87W7%w)RnriPH5#PubXD(#Qt zb=`}6I@RDHQpY=kNa_A{ANlk2h1!-L-XsS9{Yde^7JZx&lBt*$XJa_U*{MPcyegB@ zLiCqy>-sZ1zHFGjnK%FwzcjhG6;2~wQj-;X$(393Gf(VA30y8mnsPt6v5LGPJu3eu zY%}lS@YZ2aSN!T?5YGnE75@r$2_iPZ7L`-9i-c%-06Byv)+f~T;|Gd|m55Y+$g%Bm zPj}UPswtB5NxC%9CW$b6C5-v-S_M4W{9XsSP#qo;3y`eTAPWR3Kpk!&Td%m;xeD(J zkgb$2pVc5gT>4^o<`c@;15!fPdzkh}4{kYM1SD4KDK~XdJLN?dXcN3q2h=!JPqqSs`ZYWO$j+JfDLj)AlVFaGoLZ`FsNhYa`KNgLG*%}AYs=;H z-Q%gTlisM@(w$LOiPoC~Zg644D-NihWG4QGg)6mba_C<| z;@RIbtg|gW6G~C0*G;5-D_|-`wZ2&m1fZD<%P|7sCJmNjGcn=gW2)16WU#O`laDax zK8Ni+Aoi>@VK=3s;#}xhR^9Jzw%MFc&x8*v?<7KQc~eC$6!C7}T1I4g>`)FZ;6Rnwc-Ku+?+S~*U6eo2GC z#py)*DBdbx(@JH~ypn7wmCD#+D?O9fB53UEWb`Rx5qG*P9;QEqBx0pe!g%R;g<1|W zMu{%gG1KRqtpu76i)yF|p#XiLn}Zmhwi8>MGujfX&N?{@xCESOraYg32W<;>eAK%n z={*s@RQHJgpeK#FTvnKc6_gCq#JuoUie}W< zt!_}JcJdvs(L`=w;$Bzoa@0VGU*b&#h-6ubG#6sWaT z*4e@S?>9bJF?xvi88VQ^@r zKb^NY2to+SU}2lC7kk*#5^CKI%J*psqC;BRr_+8)Xi7@g5@;Nvy3eEf#ln6AX4h~MMTk5c4t}yc06aIsgVKpin*eIuxsE?F&)z#b;yzjfuy#dfqX{bNPrN@_B>{_9E zTA9)oOozvwO4b|3^;LmSq(^Y$uRpK4e~~g3$WV`$-BNHg_JV8Bv@!_>w9>pL(8W8T zSG4bRrDxA@u=P5Iq+vU_@wG*u!cg_2hU(^|WjF(DGEeyX?=kLU(a;!+whGaG=fSNk z*d?J`ge}AuLkq8o<>B87rYJ=#c@W4vb7cAbZL+a|P3JNNTkMid`+4ty!bj+3z=Hu0 z2k~HtdJ9WD2XZ{)`#7phzt{sp23-LLii+4_=Z+?tI+p-T*MNe$odqR$OZ^4Ug5CuT z>i1p^xbmEkI^S@5AhehRFD01*!L@ABtj*r?4~-95ub}R0(7Iwut*5`#qILDD6W_+Y z7)hdJCyOScg7TgL3J2FgP@G{DM3nY%3J5%E4=gG53uob>YW;S3YOCMKEWp2y_pULd z=p=qD$*^aBEj`$6MpY$1=Rss08VHvfrz0aIPuO$uvA14Y@(@0v%R)ODP2>dYu%KdV z3le_(DM~MIPhf?ZG*^A{jL?E72-d;zxY6Q_sWG>^d_+41@mMh)5P!H8)>l(`oU75yjMi=)QZ5O0~QIy0S`KRD5!4!wV>5V?kFP{XPF5va? z8WGZv+8|*>b6RX+2UjA5NFOwz5p0Xk%wVPkH~B_fO|%-3SAXru`l;Bvj)VC1llyI#qf&7Wa-Y(RzE&hY z#c`VnHONe7V=Y8iCAFyTYmIZ+o7?S*PF%lCmTuSQ%Jo#!vaWf%RI1FfrKD#hkY^wk z>Ol?BIebHZxO^o#6XIxE5=%gk`%B3fsR3KJd{z1=UolnL zxVJG*lrB{j4QrEo1?2fkWeE@8QtFVo#bYKD-BTwXlsAn+NIb#ykk;2~i}Z^tL*(2) zDEj^l>+ymTQdwjrNTKb<0x2!h66mc&hT9y_TjZ^<6q!w3JlFH^F9%r}bVg%n`#$SA z&?V##X#;j9KdvHYJ;nlu*FKt&fVUnaw~l6VR7w7Mh6<%OUk2tF0U`-YdRCIEo2*N0JceWvAO{% z05P^$9S&j+i1P&7jd02s11a{qeAFhKXYn|Z#^q<%L~&7E#{x}TCh%f9zL9B;_`cnq%wnr{i$aybv{USMj{H&n;e zC~91brnUfLfZ$-d$uYF~3IP{V_iN_BMk)+?D8L>gm}S$!?t& zQlV)1kc4Sz^kx9=TMR`7EF>s4=Y{5@Phqsy>A;-)7co^s1!;p=U*}pMhm{+p@Vufq zatXMEDqvV#Y82v96zT<7!oqk$@r_WmroUiUA0ETO)P?^L+pKL?*#5@C#oGCq1U=5Q zA0g$CZ~r`Dhx2h-IFJTaeCVSSfwE;Ai~U4%Mq7m$8A^hr2vx1wxKsjlVJ*taD2inZ zTzJ!$3*)*Mowg_q)qb6JF*!R=E}uk`Izeuu4*gX`kp(D<1DCh^tm&)Ddt~J}Qxsnjwv(tX8 zvyX!L<$1uTZ4B=@8GX|K7p-NHRI&kObG=6SV0YmbkOV-TRnI zO|*+T>1{%)>Y&?HHZ}6B)M-B$(%6o>e)DT`N>B^fzZz(E#-_Zl+AUBz!y!nVaDOy2 z$3u6pg1+`qnWld>CufRs*74%yV;3YT)s1-)(cMSoXga~Vsd(BP^rPAa)$jC(-*v@% z37zH!198UphLe}-S3Rsm`BEDOKWWc0w{xqA*NctylQ_1U7V-~4#VrQ*?E^Rv8KvWdt1NJtqcSn{#j*j6w z_1fbstu}x`G<;}0Qkh1vRW!SfaI804LpSoumU$ORzJWX)cqNKhju>)fk(kqM3Ml&A z!2Gp=M0KTb2SOfg6AZ!n)LNnKv9DJsEvO069M7@{505>ElahKg5amp<}T8K&fK;h(?6 zD8mw1UY2+wk3w(U>HbZF1W!;bJwh(oaCX7syZ3Sf5xDMzI?8(|Toe&WF(R&fcQ+c3yu={`!G8FXR6UiyIUh!wW8&E1JhsV_F+0ryRogcJ z=mjDX`rf1N0|SyXNpzx^Ga$E{xZ0rjA#wUl`H)|yF6#O1-j|5DzIW3t#yt+7 zcNg7}SUGs7>rG7>bWO7Kff`(5%~@f&g(PraPAi=D6r5Zft>_!#dM0X0J+$2_BNH?R zoa|$Frq!Oc@hvp^n3_f=wL8pkIYe%I^NNz0o<~a;t!-9IusL$bf5@y~j^P}uJSmA`P$b6?hqshH+!(Lfw%ZzV&R@ zSeM4K%Zh$TpIJvl3*Y+435$*J^=n5yy{_hfE7>NG#EjgVvP#5-e(CKh=sppX^maAE zNX<@{IQl-T&J*XUGd?M*u+U5u(r+=mRT<)1Vz2x=5(;T>kq3-Km|}E3Yx(Hz7#Fh- zz1n~3Ra5b{ZofBz<>0=~(tV~a7j=@I={B{}SvEEpZ~--V8|+jXB-+>wb+%*PSrdZd z7M{LZGk~yc&-P~2ym$d(y&q9q~N)W7GI1>>$$4YC(l9;BI13c~kj3e=Ud&dSCF}&uf?M zQd!GHyq=ro4Wh7xiYat>cl(8HtY7Wh&9m~CO^d~rM$q3WUk>W0gg4=VV7}+B=s|xE zyE2=a+GER^wZ<-ONb~odKoM*{ON^<6vCMC38HjZPl4594l@+cg4VO?`I&Mo&us#aV z&!-u6$QGLAU*#cd%#fN1kMNt$1mqiRebD;4A5quK z7G|4$JX+^DnL|IBlVhRQcziEzlnlzG*w-%kD?5Go)@k3XN?84TAp`fR>uYF~{~Kf29!G+~dPVdddEX}m_7oomyD(yDIatk7$|^h&!doNXehDBkck zGHZHZw^gsxnR%8Mcd6cQ*_(*8?TI!o8~%Cr!~0;J=2knihLxO6xsTalBrM@Q^UNyj zVZwsht9y$YVubn_ZZF&fuy~>$Y6f9uA@PKi>23z+Q7{K@vT87eZ_m5Z9YJQD%FARh zv|zV|_NH?_O}CC$;*4S~@fX=kPp}X**M^)lUdx}$t*&sF_aybYoUtxbJ6e@BL}bl1 z!gT6u4CD@44+*4-XGo_UwnuSDFq<3Yni%th`w)asPuN!fv`@Vk1Q{p(l+*v!dyUnU z@o%Of@J0AD0uM(%Sh-G71j(L& z#P>w2frh%`Q@B-Vy)lew@)RRbW1*xiX#VUh!RrokQKezDMl(Pi7&LpTQ4WmY{j%mR z>8x+w^%Q|N=rgn$>1|JlTu_p;q~`Q0G8B^T$>eeq+Te)oVD#ZgMAFQ$_)mrzjB|g` zYS5--U%iJr+>7rW=v1SQV+cxz6!kgQ!XCkoVvHC1QeKbF9MWkg!Dv_QAffz)dg8!k zQuE^sz}g^`R)c``sZ6UDkCt|Y0SPUFV}87$sgh-)j|KOnk>d17D!hRm^A=XVt5jh> zMLY7^-f@~ojO8e$4?w2mp$dkaKo?OHsn3i~zb0SkIrsVb$m2nO#Xx9kGwk)6!4yOg z?W?Bf8f3#FIu_n8C|AH{1iDH6^kk#6ZboKqIJf=jSvq;s`D^5j0A?78kZwAX1j!|? z(Ro#^<*qj68no=MqN`!UyC{&DG>|2Urxzf2d<_NMv`I8MT!f0TR}vyyIanCmY~t>P zuspc1JS|BN^x{Pmr{`zp?V)1mH{!WDQe>FU)D^N4h_)qgYCDy(NQI`tsiKN* z^<&J-v3;7VsAjVwtwbGO<*WB+#)?m0!8ba$B{?vfrtw>+A=x918Gc4%Rzxucj&tQS!w@i}(J^sJ zKFQ=gIFhUdz7R;=5Xpcxr~b0W)oYr+jId!P$MPYlSqn4GDWT{fvr(V(8v(p~mc2vF$K-#w&EfsA&V3V^Wqp-ulGl!{yL& z*6TF`2H;Ub8CW7d@LsE;%sohS2y_ToSXhW%SYPqNs&~`YVE;h_*ne>CCHR$Y^xYq} z`k!q?Y-}9CTk!_A*Ac49jt2IQ|2xup8^BHXJ?B^ONKpX~Fu`BA4}xL;7T~&H2^(HR z7&+d^l?!%KID`Ac-+?`)t!-Zg4^(p`2neZPz*xZRrGEwXZxT`6mhqYRh@di9xu#$_ zf0Z!|>@>d<_J(Z2_NGo&;M_i9u0{acpH7(DVB_Q{?2=%xI`Arx^A{QAkpDf{KPa-E z>5xbYY@f%75D?cHjepWP_`&pVCAygu@wOOpFpM@Iz-%9YMY-NQ_(_@Ikdc3j@S}bf zIrEQ2>}?Dx#Y-9;u$uD0&*5LYLnHQYV+fmoyPY`D-oa7X$?#9J{WUBq$T_qO+!a{C zU0(R7T;QuW`2P*|haw&R8qQ9&^BFd{(}#mQz4R||W#B0E-_)cCz{JKL@UO(w4)}~-B+Zuo!lK*p3+_vwbLeSM9 zcxy@@0|Mf@B<)XPqWbL?$lOuy@HX&zPIW>NSoCf%_^&E=1;_UPrpo1j4h~>pf7lrO z5CA_;9RYuB>T>q|-DWWEG8p$)fs?_x)_xQBPe2y~d%%xjbO-RwTI*sz)eOFx1i#V$ z6YxJ7_h!-V>mu$yiH7?>LjI$eH>)52I&zhH|0Cv)p8VJ5yjeWw7Fg;&-9{+J-k1 z3jc}_r}+;Ee<<$%uLN*ghMP%NuM-phq-O@di*VN)`DQ*($)6zLs{-SH!uj_JTyINv zGm|9PBsVD6m-#wDbwr@(7#Ptd0VKP$@Z?ZKK`T%;BWE2 zE#lwhfV|y+n;CnqbNc-xb<5vrz+djm-u0AN@MNdN!< literal 0 HcmV?d00001 diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..6637ced --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1 @@ +distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip \ No newline at end of file diff --git a/mvnw b/mvnw new file mode 100755 index 0000000..6ecc150 --- /dev/null +++ b/mvnw @@ -0,0 +1,236 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # + # Look for the Apple JDKs first to preserve the existing behaviour, and then look + # for the new JDKs provided by Oracle. + # + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then + # + # Oracle JDKs + # + export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then + # + # Apple JDKs + # + export JAVA_HOME=`/usr/libexec/java_home` + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + local basedir=$(pwd) + local wdir=$(pwd) + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + wdir=$(cd "$wdir/.."; pwd) + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +# avoid using MAVEN_CMD_LINE_ARGS below since that would loose parameter escaping in $@ +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 0000000..8bb8275 --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,146 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +set MAVEN_CMD_LINE_ARGS=%MAVEN_CONFIG% %* + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" + +set WRAPPER_JAR=""%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +# avoid using MAVEN_CMD_LINE_ARGS below since that would loose parameter escaping in %* +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% From 4d32514d6772421ee1b13d6465401c00fdf64ce7 Mon Sep 17 00:00:00 2001 From: John Martel Date: Thu, 2 Mar 2017 14:55:50 -0500 Subject: [PATCH 09/59] Increase Mongo stub server timeout for tests Tests sometimes timeout on Travis, so this tries to fix the issue by allowing more time to complete operations. --- .../dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java b/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java index 3f0fbc6..2f2b544 100644 --- a/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java +++ b/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java @@ -101,7 +101,7 @@ private void useStartedMongoServer() { } private void initializeMongoClient() { - MongoClientOptions options = MongoClientOptions.builder().serverSelectionTimeout(100).build(); + MongoClientOptions options = MongoClientOptions.builder().serverSelectionTimeout(300).build(); mongoClient = new MongoClient(new ServerAddress(serverAddress), options); } From 9513164e013db4691f5d38aa54acc32e849de287 Mon Sep 17 00:00:00 2001 From: John Martel Date: Wed, 8 Mar 2017 20:58:42 -0500 Subject: [PATCH 10/59] Fix loading coverage data from saved state --- README.md | 15 ++++-- pom.xml | 13 ++++- .../java/dk/dma/ais/coverage/AisCoverage.java | 19 ++++++-- .../dma/ais/coverage/data/ICoverageData.java | 3 +- .../dma/ais/coverage/data/OnlyMemoryData.java | 43 +++++++++++++++-- .../java/dk/dma/ais/coverage/data/Source.java | 5 +- .../persistence/CoverageDataMarshaller.java | 7 +-- .../persistence/DatabaseInstance.java | 9 ++-- .../MemoryOnlyDatabaseInstance.java | 9 ++-- .../MongoCoverageDataMarshaller.java | 44 ++++++++++++----- .../persistence/MongoDatabaseInstance.java | 26 +++++----- .../persistence/PersisterService.java | 12 ++++- .../resources/coverage-mongodb-sample.xml | 1 + .../ais/coverage/data/OnlyMemoryDataTest.java | 48 +++++++++++++++++++ .../dma/ais/coverage/fixture/CellFixture.java | 16 +++++-- .../MemoryOnlyDatabaseInstanceTest.java | 9 ++-- .../MongoCoverageDataMarshallerTest.java | 45 ++++++++++++----- .../MongoDatabaseInstanceTest.java | 19 +++++--- .../persistence/PersisterServiceTest.java | 6 +-- 19 files changed, 268 insertions(+), 81 deletions(-) create mode 100644 src/test/java/dk/dma/ais/coverage/data/OnlyMemoryDataTest.java diff --git a/README.md b/README.md index 55aba70..444a333 100644 --- a/README.md +++ b/README.md @@ -13,19 +13,24 @@ AisCoverage is a tool for calculating how well AIS receivers (sources) cover a g ## Building ## - mvn clean install + ./mvnw clean install ## Developing in Eclipse ## M2 Eclipse plugin or - mvn eclipse:eclipse + ./mvnw eclipse:eclipse -### Rest API ### +## Rest API ## /coverage/rest/* + +## Storing Coverage Data ## +Only MongoDB is supported at the moment. Mongo v3.4.2 or above needs to be installed and authentication is not yet supported. + +Data (atthe highest detailed level) is persisted by a background thread at regular (configurable) intervals. The default is 60 minutes. -### Distribution ### +## Distribution ## A distributable zip file is found [here](http://fuka.dk/snapshots/AisCoverage-0.2.zip).
Be aware: As it contains executable files, your browser may post a warning when you download the file.

@@ -42,7 +47,7 @@ When running tests over longer periods using mongodb you might experience some i The current release makes it possible to see the coverage within a limited timespan, down to a single hour, and with 1 hour intervals.
The data is currently only persisted in memory, so might make an out of memory error, if run over longer amounts of time.
-A sample of how satalite data coverage will be handled is possible, by pressing ctrl, and dragging the mouse over the area of interest. +A sample of how satellite data coverage will be handled is possible, by pressing ctrl, and dragging the mouse over the area of interest.
Examples of configuration files can be found here:
diff --git a/pom.xml b/pom.xml index 27470f3..249ed4a 100644 --- a/pom.xml +++ b/pom.xml @@ -109,18 +109,27 @@ 2.7.11 test + + org.hamcrest + hamcrest-library + 1.3 + test + de.bwaldvogel mongo-java-server 1.6.0 test - org.apache.commons commons-lang3 3.5 - test + + + org.apache.commons + commons-collections4 + 4.1 diff --git a/src/main/java/dk/dma/ais/coverage/AisCoverage.java b/src/main/java/dk/dma/ais/coverage/AisCoverage.java index 6429d44..fb9118a 100644 --- a/src/main/java/dk/dma/ais/coverage/AisCoverage.java +++ b/src/main/java/dk/dma/ais/coverage/AisCoverage.java @@ -14,9 +14,14 @@ */ package dk.dma.ais.coverage; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; import java.util.List; +import java.util.Map; import java.util.function.Consumer; +import org.apache.commons.collections4.IterableUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,10 +91,18 @@ public void accept(AisPacket packet) { } private void loadCoverageDataFromDatabase() { - List loadedCoverageData = databaseInstance.loadLatestSavedCoverageData(); + Map> loadedCoverageData = databaseInstance.loadLatestSavedCoverageData(); - for (Cell cell : loadedCoverageData) { - handler.getDataHandler().updateCell(cell); + for (String sourceId : loadedCoverageData.keySet()) { + for (Cell cell : loadedCoverageData.get(sourceId)) { + handler.getDataHandler().updateCell(sourceId, cell); + + List timeSpanIds = IterableUtils.toList(cell.getFixedWidthSpans().keySet()); + Collections.sort(timeSpanIds); + if (Helper.firstMessage != null && (Helper.firstMessage.getTime() > timeSpanIds.get(0))) { + Helper.firstMessage = new Date(timeSpanIds.get(0)); + } + } } } diff --git a/src/main/java/dk/dma/ais/coverage/data/ICoverageData.java b/src/main/java/dk/dma/ais/coverage/data/ICoverageData.java index d378e50..f4b5044 100644 --- a/src/main/java/dk/dma/ais/coverage/data/ICoverageData.java +++ b/src/main/java/dk/dma/ais/coverage/data/ICoverageData.java @@ -30,7 +30,7 @@ public interface ICoverageData { void updateShip(Ship ship); Cell createCell(String sourceMmsi, double lat, double lon); Cell getCell(String sourceMmsi, double lat, double lon); - void updateCell(Cell c); + void updateCell(String sourceId, Cell newCell); List getCells(QueryParams params); Source getSource(String sourceId); Source createSource(String sourceId); @@ -38,5 +38,4 @@ public interface ICoverageData { void incrementReceivedSignals(String sourceMmsi, double lat, double lon, Date timestamp); void incrementMissingSignals(String sourceMmsi, double lat, double lon, Date timestamp); void trimWindow(Date trimPoint); - } diff --git a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java index 8bc3dbb..4e7f625 100644 --- a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java +++ b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java @@ -69,13 +69,44 @@ public Cell getCell(String sourceMmsi, double lat, double lon) { } @Override - public void updateCell(Cell newCell) { - Source superSource = getSource(AbstractCalculator.SUPERSOURCE_MMSI); - Cell oldCell = superSource.getCell(newCell.getLatitude(), newCell.getLongitude()); + public void updateCell(String sourceId, Cell newCell) { + Source source = getSource(sourceId); + if (source == null) { + source = createSource(sourceId); + } + + Cell oldCell = source.getCell(newCell.getLatitude(), newCell.getLongitude()); if (oldCell == null) { - oldCell = superSource.createCell(newCell.getLatitude(), newCell.getLongitude()); + source.addCell(newCell); + } else { + updateExistingCellFromNewCell(oldCell, newCell); + } + } + + private void updateExistingCellFromNewCell(Cell oldCell, Cell newCell) { + oldCell.addNOofMissingSignals(newCell.getNOofMissingSignals()); + oldCell.addReceivedSignals(newCell.getNOofReceivedSignals()); + + updateCellTimeSpans(oldCell, newCell); + } + + private void updateCellTimeSpans(Cell oldCell, Cell newCell) { + for (Entry newTimeSpan : newCell.getFixedWidthSpans().entrySet()) { + if (!oldCell.getFixedWidthSpans().containsKey(newTimeSpan.getKey())) { + oldCell.getFixedWidthSpans().put(newTimeSpan.getKey(), newTimeSpan.getValue()); + } else { + TimeSpan oldTimeSpan = oldCell.getFixedWidthSpans().get(newTimeSpan.getKey()); + + TimeSpan updatedTimeSpan = new TimeSpan(oldTimeSpan.getFirstMessage()); + updatedTimeSpan.setLastMessage(oldTimeSpan.getLastMessage()); + updatedTimeSpan.setMessageCounterSat(oldTimeSpan.getMessageCounterSat() + newTimeSpan.getValue().getMessageCounterSat()); + updatedTimeSpan.setMessageCounterTerrestrial(oldTimeSpan.getMessageCounterTerrestrial() + newTimeSpan.getValue().getMessageCounterTerrestrial()); + updatedTimeSpan.setMessageCounterTerrestrialUnfiltered(oldTimeSpan.getMessageCounterTerrestrialUnfiltered() + newTimeSpan.getValue().getMessageCounterTerrestrialUnfiltered()); + updatedTimeSpan.setMissingSignals(oldTimeSpan.getMissingSignals() + newTimeSpan.getValue().getMissingSignals()); + + oldCell.getFixedWidthSpans().put(newTimeSpan.getKey(), updatedTimeSpan); + } } - oldCell.setFixedWidthSpans(newCell.getFixedWidthSpans()); } @Override @@ -201,6 +232,7 @@ public void incrementReceivedSignals(String sourceMmsi, double lat, cell.getFixedWidthSpans().put(id.getTime(), ts); } ts.setMessageCounterTerrestrial(ts.getMessageCounterTerrestrial() + 1); + cell.incrementNOofReceivedSignals(); } @Override @@ -219,6 +251,7 @@ public void incrementMissingSignals(String sourceMmsi, double lat, cell.getFixedWidthSpans().put(id.getTime(), ts); } ts.incrementMissingSignals(); + cell.incrementNOofMissingSignals(); } public CustomMessage packetToCustomMessage(AisPacket packet) { diff --git a/src/main/java/dk/dma/ais/coverage/data/Source.java b/src/main/java/dk/dma/ais/coverage/data/Source.java index f4cc5c1..e23db7e 100644 --- a/src/main/java/dk/dma/ais/coverage/data/Source.java +++ b/src/main/java/dk/dma/ais/coverage/data/Source.java @@ -19,7 +19,6 @@ import java.util.concurrent.ConcurrentHashMap; import dk.dma.ais.coverage.Helper; -import dk.dma.ais.coverage.data.Ship.ShipClass; public class Source implements Serializable { @@ -102,6 +101,10 @@ public Cell createTempCell(double latitude, double longitude, int multiplication return cell; } + public void addCell(Cell cell) { + grid.put(cell.getId(), cell); + } + public Map getGrid() { return grid; } diff --git a/src/main/java/dk/dma/ais/coverage/persistence/CoverageDataMarshaller.java b/src/main/java/dk/dma/ais/coverage/persistence/CoverageDataMarshaller.java index dc58489..ff9c0f7 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/CoverageDataMarshaller.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/CoverageDataMarshaller.java @@ -1,7 +1,8 @@ package dk.dma.ais.coverage.persistence; import java.time.ZonedDateTime; -import java.util.List; +import java.util.Collection; +import java.util.Map; import dk.dma.ais.coverage.data.Cell; @@ -10,7 +11,7 @@ */ interface CoverageDataMarshaller { - T marshall(List coverageData, ZonedDateTime dataTimestamp); + T marshall(Map> coverageData, ZonedDateTime dataTimestamp); - List unmarshall(T coverageData); + Map> unmarshall(T coverageData); } diff --git a/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java b/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java index 8ca3236..9a62e88 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/DatabaseInstance.java @@ -1,6 +1,7 @@ package dk.dma.ais.coverage.persistence; -import java.util.List; +import java.util.Collection; +import java.util.Map; import dk.dma.ais.coverage.configuration.DatabaseConfiguration; import dk.dma.ais.coverage.data.Cell; @@ -33,13 +34,13 @@ public interface DatabaseInstance extends AutoCloseable { * Saves the coverage data stored in in-memory cells to the underlying database. * * @param coverageData - * the coverage data stored in a {@link List} of {@link Cell}, as provided by {@link dk.dma.ais.coverage.data.ICoverageData} + * the coverage data stored as a {@link Collection} of {@link Cell} by source identifier * @return * the result of the save operation * @throws DatabaseConnectionException * when connection to the database server is impossible for any reason */ - PersistenceResult save(List coverageData); + PersistenceResult save(Map> coverageData); - List loadLatestSavedCoverageData(); + Map> loadLatestSavedCoverageData(); } diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java b/src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java index bc0647e..d15b307 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstance.java @@ -1,7 +1,8 @@ package dk.dma.ais.coverage.persistence; +import java.util.Collection; import java.util.Collections; -import java.util.List; +import java.util.Map; import dk.dma.ais.coverage.configuration.DatabaseConfiguration; import dk.dma.ais.coverage.data.Cell; @@ -20,13 +21,13 @@ public void createDatabase() { } @Override - public PersistenceResult save(List coverageData) { + public PersistenceResult save(Map> coverageData) { return null; } @Override - public List loadLatestSavedCoverageData() { - return Collections.emptyList(); + public Map> loadLatestSavedCoverageData() { + return Collections.emptyMap(); } @Override diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java b/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java index 49430f0..1815046 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java @@ -4,6 +4,8 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; @@ -11,6 +13,7 @@ import org.bson.Document; +import dk.dma.ais.coverage.Helper; import dk.dma.ais.coverage.data.Cell; import dk.dma.ais.coverage.data.TimeSpan; @@ -20,7 +23,7 @@ class MongoCoverageDataMarshaller implements CoverageDataMarshaller { @Override - public Document marshall(List coverageData, ZonedDateTime dataTimestamp) { + public Document marshall(Map> coverageData, ZonedDateTime dataTimestamp) { Document coverageDataDocument = new Document(); coverageDataDocument.put("dataTimestamp", dataTimestamp.withZoneSameInstant(ZoneId.of("UTC")).format(DateTimeFormatter.ISO_DATE_TIME)); @@ -34,18 +37,23 @@ public Document marshall(List coverageData, ZonedDateTime dataTimestamp) { return coverageDataDocument; } - private void marshallCells(List coverageData, List> grid) { - for (Cell cell : coverageData) { - Map savedCell = marshallCell(cell); - grid.add(savedCell); + private void marshallCells(Map> coverageData, List> grid) { + for (String sourceId : coverageData.keySet()) { + for (Cell cell : coverageData.get(sourceId)) { + Map savedCell = marshallCell(sourceId, cell); + grid.add(savedCell); + } } } - private Map marshallCell(Cell cell) { + private Map marshallCell(String sourceId, Cell cell) { Map savedCell = new LinkedHashMap<>(); + savedCell.put("sourceId", sourceId); savedCell.put("cellId", cell.getId()); savedCell.put("latitude", cell.getLatitude()); savedCell.put("longitude", cell.getLongitude()); + savedCell.put("numberOfReceivedSignals", cell.getNOofReceivedSignals()); + savedCell.put("numberOfMissingSignals", cell.getNOofMissingSignals()); Map> fixedWidthTimeSpans = marshallCellTimeSpans(cell); savedCell.put("timespans", fixedWidthTimeSpans); @@ -71,30 +79,42 @@ private Map> marshallCellTimeSpans(Cell cell) { } @Override - public List unmarshall(Document coverageData) { - List unmarshalledCoverageData = new ArrayList<>(); + public Map> unmarshall(Document coverageData) { + Map> unmarshalledCoverageData = new LinkedHashMap<>(); if (coverageData != null) { - unmarshallCells(coverageData, unmarshalledCoverageData); + unmarshalledCoverageData.putAll(unmarshallCells(coverageData)); } return unmarshalledCoverageData; } - private void unmarshallCells(Document coverageData, List unmarshalledCoverageData) { + private Map> unmarshallCells(Document coverageData) { + Map> unmarshalledCoverageData = new LinkedHashMap<>(); Object cells = coverageData.get("cells"); if (cells != null && (cells instanceof List)) { List> grid = (List>) cells; for (Map cell : grid) { Cell unmarshalledCell = unmarshallCell(cell); - unmarshalledCoverageData.add(unmarshalledCell); + String sourceId = (String) cell.get("sourceId"); + if (unmarshalledCoverageData.containsKey(sourceId)) { + unmarshalledCoverageData.get(sourceId).add(unmarshalledCell); + } else { + unmarshalledCoverageData.put(sourceId, new ArrayList<>(Arrays.asList(unmarshalledCell))); + } } } + + return unmarshalledCoverageData; } private Cell unmarshallCell(Map cell) { - Cell unmarshalledCell = new Cell((double) cell.get("latitude"), (double) cell.get("longitude"), (String) cell.get("cellId")); + double latitude = Helper.roundLat((double) cell.get("latitude"), 1); + double longitude = Helper.roundLon((double) cell.get("longitude"), 1); + Cell unmarshalledCell = new Cell(latitude, longitude, (String) cell.get("cellId")); + unmarshalledCell.addReceivedSignals(((Integer) cell.get("numberOfReceivedSignals")).intValue()); + unmarshalledCell.addNOofMissingSignals(((Integer) cell.get("numberOfMissingSignals")).intValue()); Map> fixedWidthTimeSpans = (Map>) cell.get("timespans"); Map unmarshalledTimeSpans = unmarshallTimeSpans(fixedWidthTimeSpans); diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java b/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java index d561b1f..4e0edd9 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java @@ -2,8 +2,9 @@ import java.time.ZoneId; import java.time.ZonedDateTime; -import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -17,7 +18,6 @@ import com.mongodb.ServerAddress; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoDatabase; -import dk.dma.ais.coverage.Helper; import dk.dma.ais.coverage.configuration.DatabaseConfiguration; import dk.dma.ais.coverage.data.Cell; @@ -110,7 +110,7 @@ private boolean collectionExists(MongoDatabase database) { } @Override - public PersistenceResult save(List coverageData) { + public PersistenceResult save(Map> coverageData) { requireOpenConnection(); Document coverageDataDocument = marshaller.marshall(coverageData, ZonedDateTime.now(ZoneId.of("UTC"))); @@ -118,6 +118,7 @@ public PersistenceResult save(List coverageData) { try { client.getDatabase(databaseName).getCollection(COVERAGE_DATA).insertOne(coverageDataDocument); + LOG.info("Saved [{}] cells with timestamp [{}]", numberOfSavedCells, coverageDataDocument.get("dataTimestamp")); return PersistenceResult.success(numberOfSavedCells); } catch (MongoException e) { logAndTransformException(e); @@ -127,10 +128,10 @@ public PersistenceResult save(List coverageData) { } @Override - public List loadLatestSavedCoverageData() { + public Map> loadLatestSavedCoverageData() { requireOpenConnection(); - List latestCoverageData = new ArrayList<>(); + Map> latestCoverageData = new LinkedHashMap<>(); try { Document orderByDataTimestamp = new Document(); @@ -139,17 +140,20 @@ public List loadLatestSavedCoverageData() { FindIterable foundDocuments = client.getDatabase(databaseName).getCollection(COVERAGE_DATA).find().sort(orderByDataTimestamp).limit(1); if (foundDocuments.iterator().hasNext()) { Document document = foundDocuments.iterator().next(); - latestCoverageData.addAll(marshaller.unmarshall(document)); + latestCoverageData.putAll(marshaller.unmarshall(document)); + + long loadedCells = 0L; + for (Collection cells : latestCoverageData.values()) { + loadedCells = loadedCells + cells.size(); + } + + LOG.info("Loaded [{}] cells from previously saved state at [{}]", loadedCells, document.get("dataTimestamp")); } } catch (MongoException e) { logAndTransformException(e); } - if (!latestCoverageData.isEmpty()) { - Helper.firstMessage = latestCoverageData.get(0).getFixedWidthSpans().values().iterator().next().getFirstMessage(); - } - - return Collections.unmodifiableList(latestCoverageData); + return Collections.unmodifiableMap(latestCoverageData); } void setMongoClientOptions(MongoClientOptions mongoClientOptions) { diff --git a/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java index 14ad8a1..02b2171 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java @@ -1,5 +1,8 @@ package dk.dma.ais.coverage.persistence; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -7,7 +10,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import dk.dma.ais.coverage.data.Cell; import dk.dma.ais.coverage.data.ICoverageData; +import dk.dma.ais.coverage.data.Source; /** * Performs asynchronous save operation to configured database. @@ -65,7 +70,12 @@ private class SaveOperation implements Runnable { @Override public void run() { - PersistenceResult persistenceResult = databaseInstance.save(coverageData.getCells(null)); + Map> cellsBySource = new LinkedHashMap<>(); + for (Source source : coverageData.getSources()) { + cellsBySource.put(source.getIdentifier(), source.getGrid().values()); + } + + PersistenceResult persistenceResult = databaseInstance.save(cellsBySource); if (PersistenceResult.Status.SUCCESS.equals(persistenceResult.getStatus())) { LOG.info("Saved [{}] cells to MongoDB database", persistenceResult.getWrittenCells()); diff --git a/src/main/resources/coverage-mongodb-sample.xml b/src/main/resources/coverage-mongodb-sample.xml index e82f0f3..b679095 100644 --- a/src/main/resources/coverage-mongodb-sample.xml +++ b/src/main/resources/coverage-mongodb-sample.xml @@ -33,6 +33,7 @@ localhost nordicCoverage 27017 + 30 diff --git a/src/test/java/dk/dma/ais/coverage/data/OnlyMemoryDataTest.java b/src/test/java/dk/dma/ais/coverage/data/OnlyMemoryDataTest.java new file mode 100644 index 0000000..b4e69ea --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/data/OnlyMemoryDataTest.java @@ -0,0 +1,48 @@ +package dk.dma.ais.coverage.data; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Date; + +import org.junit.Before; +import org.junit.Test; + +import dk.dma.ais.coverage.Helper; +import dk.dma.ais.coverage.calculator.AbstractCalculator; +import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; +import dk.dma.ais.coverage.fixture.CellFixture; + +public class OnlyMemoryDataTest { + + private ICoverageData coverageData; + private Cell aCell; + private Date now; + + @Before + public void setUp() throws Exception { + Helper.conf = new AisCoverageConfiguration(); + + coverageData = new OnlyMemoryData(); + aCell = CellFixture.createCellWithNoTimeSpan(); + coverageData.updateCell(AbstractCalculator.SUPERSOURCE_MMSI, aCell); + now = new Date(ZonedDateTime.now(ZoneId.of("UTC")).toInstant().getEpochSecond()); + } + + @Test + public void whenIncrementMissingSignals_thenCellGlobalMissingSignalsAreIncremented() { + coverageData.incrementMissingSignals(AbstractCalculator.SUPERSOURCE_MMSI, aCell.getLatitude(), aCell.getLongitude(), now); + + assertThat(aCell.getNOofMissingSignals(), is(equalTo(1))); + } + + @Test + public void whenIncrementReceivedSignals_thenCellGlobalReceivedSignalsAreIncremented() { + coverageData.incrementReceivedSignals(AbstractCalculator.SUPERSOURCE_MMSI, aCell.getLatitude(), aCell.getLongitude(), now); + + assertThat(aCell.getNOofReceivedSignals(), is(equalTo(1))); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java b/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java index e63c6fe..700ec6a 100644 --- a/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java +++ b/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java @@ -8,6 +8,8 @@ import org.apache.commons.lang3.RandomUtils; +import dk.dma.ais.coverage.Helper; +import dk.dma.ais.coverage.calculator.geotools.SphereProjection; import dk.dma.ais.coverage.data.Cell; import dk.dma.ais.coverage.data.TimeSpan; @@ -23,13 +25,21 @@ public static Cell createCellWithTimeSpans() { } private static Cell createCell() { - double latitude = RandomUtils.nextDouble(); - double longitude = RandomUtils.nextDouble(); - String cellId = latitude + "_" + longitude; + double latitude = randomLatitude(); + double longitude = randomLongitude(latitude); + String cellId = Helper.getCellId(latitude, longitude, 1); return new Cell(latitude, longitude, cellId); } + public static double randomLatitude() { + return Helper.roundLat(SphereProjection.metersToLatDegree(RandomUtils.nextDouble()), 1); + } + + public static double randomLongitude(double latitude) { + return Helper.roundLon(SphereProjection.metersToLonDegree(latitude, RandomUtils.nextDouble()), 1); + } + private static Map createTimeSpans() { Map fixedWidthTimeSpans = new LinkedHashMap<>(); diff --git a/src/test/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstanceTest.java b/src/test/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstanceTest.java index 8f0d905..bcd58e2 100644 --- a/src/test/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstanceTest.java +++ b/src/test/java/dk/dma/ais/coverage/persistence/MemoryOnlyDatabaseInstanceTest.java @@ -1,11 +1,12 @@ package dk.dma.ais.coverage.persistence; -import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertThat; -import java.util.List; +import java.util.Collection; +import java.util.Map; import org.junit.Test; @@ -17,7 +18,7 @@ public class MemoryOnlyDatabaseInstanceTest { public void whenSave_thenNullPersistenceResultIsReturned() { DatabaseInstance databaseInstance = new MemoryOnlyDatabaseInstance(); - PersistenceResult persistenceResult = databaseInstance.save(emptyList()); + PersistenceResult persistenceResult = databaseInstance.save(emptyMap()); assertThat(persistenceResult, is(nullValue())); } @@ -26,7 +27,7 @@ public void whenSave_thenNullPersistenceResultIsReturned() { public void whenLoadLatestSavedCoverageData_thenEmptyListIsReturned() { DatabaseInstance databaseInstance = new MemoryOnlyDatabaseInstance(); - List coverageData = databaseInstance.loadLatestSavedCoverageData(); + Map> coverageData = databaseInstance.loadLatestSavedCoverageData(); assertThat(coverageData.isEmpty(), is(true)); } diff --git a/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java b/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java index c862cdd..026b8fb 100644 --- a/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java +++ b/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java @@ -2,30 +2,46 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.closeTo; import static org.junit.Assert.assertThat; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.bson.Document; +import org.junit.After; import org.junit.Before; import org.junit.Test; +import dk.dma.ais.coverage.Helper; +import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; import dk.dma.ais.coverage.data.Cell; import dk.dma.ais.coverage.data.TimeSpan; import dk.dma.ais.coverage.fixture.CellFixture; public class MongoCoverageDataMarshallerTest { + private static final double DELTA_WHEN_COMPARING_DOUBLE = 0.0001D; private MongoCoverageDataMarshaller marshaller; @Before public void setUp() throws Exception { marshaller = new MongoCoverageDataMarshaller(); + + Helper.conf = new AisCoverageConfiguration(); + } + + @After + public void tearDown() throws Exception { + Helper.conf = null; + } @Test @@ -36,7 +52,7 @@ public void givenNoCoverageData_whenMarshall_thenDocumentWithEmptyCellsArrayAndT assertThatDocumentHasEmptyCellsAndDataTimestamp(marshalledCoverageData, now); - marshalledCoverageData = marshaller.marshall(Collections.emptyList(), now); + marshalledCoverageData = marshaller.marshall(Collections.emptyMap(), now); assertThatDocumentHasEmptyCellsAndDataTimestamp(marshalledCoverageData, now); } @@ -56,8 +72,10 @@ public void givenManyCells_whenMarshall_thenDocumentWithTheseCellsAndTimestampIs Cell secondCell = CellFixture.createCellWithNoTimeSpan(); cells.add(firstCell); cells.add(secondCell); + Map> cellsBySourceId = new LinkedHashMap<>(); + cellsBySourceId.put("default", cells); - Document marshalledCoverageData = marshaller.marshall(cells, now); + Document marshalledCoverageData = marshaller.marshall(cellsBySourceId, now); ZonedDateTime dataTimestamp = ZonedDateTime.parse((String) marshalledCoverageData.get("dataTimestamp"), DateTimeFormatter.ISO_DATE_TIME); assertThat(dataTimestamp, is(equalTo(now))); @@ -96,7 +114,7 @@ public void givenManyCells_whenMarshall_thenDocumentWithTheseCellsAndTimestampIs @Test public void givenNoCoverageData_whenUnmarshall_thenEmptyListIsReturned() { - List coverageData = marshaller.unmarshall(null); + Map> coverageData = marshaller.unmarshall(null); assertThat(coverageData.isEmpty(), is(true)); coverageData = marshaller.unmarshall(new Document()); @@ -116,16 +134,19 @@ public void givenManyCells_whenUnmarshall_thenTheseCellsAreReturned() { Cell secondCell = CellFixture.createCellWithNoTimeSpan(); cells.add(firstCell); cells.add(secondCell); - Document marshalledCoverageData = marshaller.marshall(cells, now); + Map> cellsBySourceId = new LinkedHashMap<>(); + cellsBySourceId.put("default", cells); + Document marshalledCoverageData = marshaller.marshall(cellsBySourceId, now); - List unmarshalledCoverageData = marshaller.unmarshall(marshalledCoverageData); + Map> unmarshalledCoverageData = marshaller.unmarshall(marshalledCoverageData); - assertThat(unmarshalledCoverageData.size(), is(equalTo(2))); + assertThat(unmarshalledCoverageData.get("default").size(), is(equalTo(2))); - Cell firstUnmarshalledCell = unmarshalledCoverageData.get(0); + Iterator cellsFromDefaultSource = unmarshalledCoverageData.get("default").iterator(); + Cell firstUnmarshalledCell = cellsFromDefaultSource.next(); assertThat(firstUnmarshalledCell.getId(), is(equalTo(firstCell.getId()))); - assertThat(firstUnmarshalledCell.getLatitude(), is(equalTo(firstCell.getLatitude()))); - assertThat(firstUnmarshalledCell.getLongitude(), is(equalTo(firstCell.getLongitude()))); + assertThat(firstUnmarshalledCell.getLatitude(), is(closeTo(firstCell.getLatitude(), DELTA_WHEN_COMPARING_DOUBLE))); + assertThat(firstUnmarshalledCell.getLongitude(), is(closeTo(firstCell.getLongitude(), DELTA_WHEN_COMPARING_DOUBLE))); Map firstUnmarshalledCellFixedWidthSpans = firstUnmarshalledCell.getFixedWidthSpans(); assertThat(firstUnmarshalledCellFixedWidthSpans.size(), is(equalTo(2))); @@ -142,10 +163,10 @@ public void givenManyCells_whenUnmarshall_thenTheseCellsAreReturned() { assertThat(unmarshalledTimeSpan.getMissingSignals(), is(equalTo(expectedTimeSpan.getMissingSignals()))); } - Cell secondUnmarshalledCell = unmarshalledCoverageData.get(1); + Cell secondUnmarshalledCell = cellsFromDefaultSource.next(); assertThat(secondUnmarshalledCell.getId(), is(equalTo(secondCell.getId()))); - assertThat(secondUnmarshalledCell.getLatitude(), is(equalTo(secondCell.getLatitude()))); - assertThat(secondUnmarshalledCell.getLongitude(), is(equalTo(secondCell.getLongitude()))); + assertThat(secondUnmarshalledCell.getLatitude(), is(closeTo(secondCell.getLatitude(), DELTA_WHEN_COMPARING_DOUBLE))); + assertThat(secondUnmarshalledCell.getLongitude(), is(closeTo(secondCell.getLongitude(), DELTA_WHEN_COMPARING_DOUBLE))); assertThat(secondUnmarshalledCell.getFixedWidthSpans().isEmpty(), is(true)); } diff --git a/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java b/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java index 2f2b544..1431fd2 100644 --- a/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java +++ b/src/test/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstanceTest.java @@ -12,8 +12,10 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import org.bson.Document; import org.junit.After; @@ -28,6 +30,8 @@ import com.mongodb.client.MongoIterable; import de.bwaldvogel.mongo.MongoServer; import de.bwaldvogel.mongo.backend.memory.MemoryBackend; +import dk.dma.ais.coverage.Helper; +import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; import dk.dma.ais.coverage.configuration.DatabaseConfiguration; import dk.dma.ais.coverage.data.Cell; import dk.dma.ais.coverage.fixture.CellFixture; @@ -50,6 +54,7 @@ public class MongoDatabaseInstanceTest { public void setUp() throws Exception { marshaller = mock(CoverageDataMarshaller.class); databaseInstance = new MongoDatabaseInstance(marshaller); + Helper.conf = new AisCoverageConfiguration(); } @After @@ -57,6 +62,7 @@ public void tearDown() throws Exception { databaseInstance.close(); closeMongoClient(); shutdownMongoServer(); + Helper.conf = null; } private void closeMongoClient() { @@ -156,7 +162,7 @@ public void givenACell_whenSave_thenCellIsPersistedInCoverageDataCollection() { Cell aCell = CellFixture.createCellWithTimeSpans(); - PersistenceResult result = databaseInstance.save(Arrays.asList(aCell)); + PersistenceResult result = databaseInstance.save(Collections.singletonMap("default", Arrays.asList(aCell))); assertThat(result.getStatus(), is(equalTo(PersistenceResult.Status.SUCCESS))); assertThat(result.getWrittenCells(), is(1L)); @@ -174,7 +180,7 @@ public void givenDatabaseInstanceIsNotOpened_whenCreateDatabase_thenIllegalState public void givenDatabaseInstanceIsNotOpened_whenSave_thenIllegalStateExceptionIsThrown() { thrown.expect(IllegalStateException.class); - databaseInstance.save(Collections.emptyList()); + databaseInstance.save(Collections.emptyMap()); } @Test @@ -190,7 +196,7 @@ public void givenNoSavedCoverageData_whenLoadLatestSavedCoverageData_thenEmptyLi useStartedMongoServer(); preCreateCollection(); - List cells = databaseInstance.loadLatestSavedCoverageData(); + Map> cells = databaseInstance.loadLatestSavedCoverageData(); assertThat(cells.isEmpty(), is(true)); } @@ -205,14 +211,15 @@ public void givenSavedCoverageData_whenLoadLatestSavedCoverageData_thenThisDataI Cell anotherCell = CellFixture.createCellWithNoTimeSpan(); List cells = Arrays.asList(aCell, anotherCell); MongoCoverageDataMarshaller marshaller = new MongoCoverageDataMarshaller(); - Document document = marshaller.marshall(cells, ZonedDateTime.now(ZoneId.of("UTC"))); + Document document = marshaller.marshall(Collections.singletonMap("default", cells), ZonedDateTime.now(ZoneId.of("UTC"))); mongoClient.getDatabase(NORDIC_COVERAGE_DATABASE).getCollection(COVERAGE_DATA_COLLECTION).insertOne(document); databaseInstance.setMarshaller(marshaller); - List loadedCoverageData = databaseInstance.loadLatestSavedCoverageData(); + Map> loadedCoverageData = databaseInstance.loadLatestSavedCoverageData(); - assertThat(loadedCoverageData.size(), is(equalTo(2))); + assertThat(loadedCoverageData.size(), is(equalTo(1))); + assertThat(loadedCoverageData.get("default").size(), is(equalTo(2))); } } diff --git a/src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java b/src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java index ab1ed02..caf4a45 100644 --- a/src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java +++ b/src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java @@ -3,7 +3,7 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; -import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -18,7 +18,7 @@ public class PersisterServiceTest { @Test public void whenStart_thenSaveIsInvokedOnDatabaseInstanceAtInterval() throws InterruptedException { DatabaseInstance databaseInstance = mock(DatabaseInstance.class); - when(databaseInstance.save(anyList())).thenReturn(PersistenceResult.success(1)); + when(databaseInstance.save(anyMap())).thenReturn(PersistenceResult.success(1)); ICoverageData coverageData = mock(ICoverageData.class); PersisterService persisterService = new PersisterService(databaseInstance, coverageData); persisterService.intervalInSeconds(1L); @@ -27,7 +27,7 @@ public void whenStart_thenSaveIsInvokedOnDatabaseInstanceAtInterval() throws Int Thread.sleep(5 * 1000); persisterService.stop(); - verify(databaseInstance, atLeast(4)).save(anyList()); + verify(databaseInstance, atLeast(4)).save(anyMap()); } @Test From 50c6bc7ab4d4bc09fbbd70f34c4104422f7b1e96 Mon Sep 17 00:00:00 2001 From: John Martel Date: Thu, 9 Mar 2017 19:59:56 -0500 Subject: [PATCH 11/59] Set system first message timestamp on load This allows the user to see the coverage as it was when the coverage data was last saved. --- .../java/dk/dma/ais/coverage/AisCoverage.java | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/AisCoverage.java b/src/main/java/dk/dma/ais/coverage/AisCoverage.java index fb9118a..046b607 100644 --- a/src/main/java/dk/dma/ais/coverage/AisCoverage.java +++ b/src/main/java/dk/dma/ais/coverage/AisCoverage.java @@ -14,17 +14,6 @@ */ package dk.dma.ais.coverage; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; - -import org.apache.commons.collections4.IterableUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import dk.dma.ais.bus.AisBus; import dk.dma.ais.bus.consumer.DistributerConsumer; import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; @@ -38,6 +27,16 @@ import dk.dma.ais.reader.AisReader; import net.jcip.annotations.GuardedBy; import net.jcip.annotations.ThreadSafe; +import org.apache.commons.collections4.IterableUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; /** @@ -97,15 +96,23 @@ private void loadCoverageDataFromDatabase() { for (Cell cell : loadedCoverageData.get(sourceId)) { handler.getDataHandler().updateCell(sourceId, cell); - List timeSpanIds = IterableUtils.toList(cell.getFixedWidthSpans().keySet()); - Collections.sort(timeSpanIds); - if (Helper.firstMessage != null && (Helper.firstMessage.getTime() > timeSpanIds.get(0))) { - Helper.firstMessage = new Date(timeSpanIds.get(0)); - } + adjustSystemEarliestMessageFromCell(cell); } } } + private void adjustSystemEarliestMessageFromCell(Cell cell) { + List timeSpanIds = IterableUtils.toList(cell.getFixedWidthSpans().keySet()); + Collections.sort(timeSpanIds); + Long earliestTimeSpanId = timeSpanIds.get(0); + + if (Helper.firstMessage == null) { + Helper.firstMessage = new Date(earliestTimeSpanId.longValue()); + } else if (Helper.firstMessage.getTime() > earliestTimeSpanId) { + Helper.firstMessage = new Date(earliestTimeSpanId); + } + } + private void createPersisterService() { persisterService = new PersisterService(databaseInstance, handler.getDataHandler()); persisterService.intervalInSeconds(conf.getDatabaseConfiguration().getPersistenceIntervalInSeconds()); From 46f4093c58d60ff6d73b55766f44356320edec79 Mon Sep 17 00:00:00 2001 From: John Martel Date: Thu, 9 Mar 2017 20:35:40 -0500 Subject: [PATCH 12/59] Use minutes as the persistence interval unit After discussion with the CCG engineer, it has been agreed that minutes are fine-grained enough for their needs. --- .../java/dk/dma/ais/coverage/AisCoverage.java | 2 +- .../configuration/DatabaseConfiguration.java | 10 +- .../persistence/PersisterService.java | 38 +++--- .../resources/coverage-mongodb-sample.xml | 2 +- .../DatabaseConfigurationTest.java | 8 +- .../persistence/PersisterServiceTest.java | 119 ++++++++++++++++-- 6 files changed, 137 insertions(+), 42 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/AisCoverage.java b/src/main/java/dk/dma/ais/coverage/AisCoverage.java index 046b607..ae7136b 100644 --- a/src/main/java/dk/dma/ais/coverage/AisCoverage.java +++ b/src/main/java/dk/dma/ais/coverage/AisCoverage.java @@ -115,7 +115,7 @@ private void adjustSystemEarliestMessageFromCell(Cell cell) { private void createPersisterService() { persisterService = new PersisterService(databaseInstance, handler.getDataHandler()); - persisterService.intervalInSeconds(conf.getDatabaseConfiguration().getPersistenceIntervalInSeconds()); + persisterService.intervalInMinutes(conf.getDatabaseConfiguration().getPersistenceIntervalInMinutes()); } public void start() { diff --git a/src/main/java/dk/dma/ais/coverage/configuration/DatabaseConfiguration.java b/src/main/java/dk/dma/ais/coverage/configuration/DatabaseConfiguration.java index 1d9aaee..a57dbae 100644 --- a/src/main/java/dk/dma/ais/coverage/configuration/DatabaseConfiguration.java +++ b/src/main/java/dk/dma/ais/coverage/configuration/DatabaseConfiguration.java @@ -19,7 +19,7 @@ public class DatabaseConfiguration { private String dbName = "nordicCoverage"; private String addr = "localhost"; private int port = 5000; - private int persistenceIntervalInSeconds = 60; + private int persistenceIntervalInMinutes = 60; public String getType() { return type; @@ -53,11 +53,11 @@ public void setPort(int port) { this.port = port; } - public int getPersistenceIntervalInSeconds() { - return persistenceIntervalInSeconds; + public int getPersistenceIntervalInMinutes() { + return persistenceIntervalInMinutes; } - public void setPersistenceIntervalInSeconds(int persistenceIntervalInSeconds) { - this.persistenceIntervalInSeconds = persistenceIntervalInSeconds; + public void setPersistenceIntervalInMinutes(int persistenceIntervalInMinutes) { + this.persistenceIntervalInMinutes = persistenceIntervalInMinutes; } } diff --git a/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java index 02b2171..96671ee 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java @@ -1,5 +1,11 @@ package dk.dma.ais.coverage.persistence; +import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.data.ICoverageData; +import dk.dma.ais.coverage.data.Source; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; @@ -7,13 +13,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import dk.dma.ais.coverage.data.Cell; -import dk.dma.ais.coverage.data.ICoverageData; -import dk.dma.ais.coverage.data.Source; - /** * Performs asynchronous save operation to configured database. */ @@ -22,8 +21,8 @@ public class PersisterService { private final DatabaseInstance databaseInstance; private final ICoverageData coverageData; - private long persistenceIntervalInSeconds = 60; - private ScheduledExecutorService executor; + private long persistenceIntervalInMinutes = 60; + private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); public PersisterService(DatabaseInstance databaseInstance, ICoverageData coverageData) { this.databaseInstance = databaseInstance; @@ -31,10 +30,9 @@ public PersisterService(DatabaseInstance databaseInstance, ICoverageData coverag } public void start() { - LOG.info("Starting PersisterService, persisting every [{}] seconds", persistenceIntervalInSeconds); + LOG.info("Starting PersisterService, persisting every [{}] minutes", persistenceIntervalInMinutes); - executor = Executors.newScheduledThreadPool(1); - executor.scheduleAtFixedRate(new SaveOperation(), persistenceIntervalInSeconds, persistenceIntervalInSeconds, TimeUnit.SECONDS); + executor.scheduleAtFixedRate(new SaveOperation(), persistenceIntervalInMinutes, persistenceIntervalInMinutes, TimeUnit.MINUTES); LOG.info("PersisterService started"); } @@ -44,9 +42,9 @@ public void stop() { executor.shutdown(); try { - if (!executor.awaitTermination(persistenceIntervalInSeconds, TimeUnit.SECONDS)) { + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { executor.shutdownNow(); - if (!executor.awaitTermination(persistenceIntervalInSeconds, TimeUnit.SECONDS)) { + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { LOG.warn("PersisterService thread pool did not terminate cleanly"); } } @@ -58,12 +56,16 @@ public void stop() { LOG.info("PersisterService stopped"); } - public void intervalInSeconds(long persistenceIntervalInSeconds) { - this.persistenceIntervalInSeconds = persistenceIntervalInSeconds; + public void intervalInMinutes(long persistenceIntervalInMinutes) { + this.persistenceIntervalInMinutes = persistenceIntervalInMinutes; + } + + long getIntervalInMinutes() { + return persistenceIntervalInMinutes; } - long getIntervalInSeconds() { - return persistenceIntervalInSeconds; + void setExecutor(ScheduledExecutorService executor) { + this.executor = executor; } private class SaveOperation implements Runnable { diff --git a/src/main/resources/coverage-mongodb-sample.xml b/src/main/resources/coverage-mongodb-sample.xml index b679095..ba5790b 100644 --- a/src/main/resources/coverage-mongodb-sample.xml +++ b/src/main/resources/coverage-mongodb-sample.xml @@ -33,7 +33,7 @@ localhost nordicCoverage 27017 - 30 + 60 diff --git a/src/test/java/dk/dma/ais/coverage/configuration/DatabaseConfigurationTest.java b/src/test/java/dk/dma/ais/coverage/configuration/DatabaseConfigurationTest.java index 0c850c3..4dee186 100644 --- a/src/test/java/dk/dma/ais/coverage/configuration/DatabaseConfigurationTest.java +++ b/src/test/java/dk/dma/ais/coverage/configuration/DatabaseConfigurationTest.java @@ -1,17 +1,17 @@ package dk.dma.ais.coverage.configuration; +import org.junit.Test; + import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; -import org.junit.Test; - public class DatabaseConfigurationTest { @Test - public void whenNewInstance_thenPersistenceIntervalInSecondsDefaultsTo60() { + public void whenNewInstance_thenPersistenceIntervalInMinutesDefaultsTo60() { DatabaseConfiguration configuration = new DatabaseConfiguration(); - assertThat(configuration.getPersistenceIntervalInSeconds(), is(equalTo(60))); + assertThat(configuration.getPersistenceIntervalInMinutes(), is(equalTo(60))); } } diff --git a/src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java b/src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java index caf4a45..63aa5ce 100644 --- a/src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java +++ b/src/test/java/dk/dma/ais/coverage/persistence/PersisterServiceTest.java @@ -1,17 +1,18 @@ package dk.dma.ais.coverage.persistence; +import dk.dma.ais.coverage.data.ICoverageData; +import org.apache.commons.lang3.NotImplementedException; +import org.junit.Test; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.*; + import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.Mockito.atLeast; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import org.junit.Test; - -import dk.dma.ais.coverage.data.ICoverageData; +import static org.mockito.Mockito.*; public class PersisterServiceTest { @@ -21,21 +22,113 @@ public void whenStart_thenSaveIsInvokedOnDatabaseInstanceAtInterval() throws Int when(databaseInstance.save(anyMap())).thenReturn(PersistenceResult.success(1)); ICoverageData coverageData = mock(ICoverageData.class); PersisterService persisterService = new PersisterService(databaseInstance, coverageData); - persisterService.intervalInSeconds(1L); + ScheduledExecutorService executor = new RunImmediatelyFiveTimesExecutorServive(); + persisterService.setExecutor(executor); persisterService.start(); - Thread.sleep(5 * 1000); persisterService.stop(); - verify(databaseInstance, atLeast(4)).save(anyMap()); + verify(databaseInstance, times(5)).save(anyMap()); } @Test - public void whenNewInstance_thenIntervalInSecondsDefaultsTo60() { + public void whenNewInstance_thenIntervalInMinutesDefaultsTo60() { DatabaseInstance databaseInstance = mock(DatabaseInstance.class); ICoverageData coverageData = mock(ICoverageData.class); PersisterService persisterService = new PersisterService(databaseInstance, coverageData); - assertThat(persisterService.getIntervalInSeconds(), is(equalTo(60L))); + assertThat(persisterService.getIntervalInMinutes(), is(equalTo(60L))); + } + + private static class RunImmediatelyFiveTimesExecutorServive implements ScheduledExecutorService { + private static final String NOT_IMPLEMENTED = "Not implemented"; + + @Override + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) { + throw new NotImplementedException(NOT_IMPLEMENTED); + } + + @Override + public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { + throw new NotImplementedException(NOT_IMPLEMENTED); + } + + @Override + public ScheduledFuture scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) { + for (int i = 0; i < 5; i++) { + command.run(); + } + return null; + } + + @Override + public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) { + throw new NotImplementedException(NOT_IMPLEMENTED); + } + + @Override + public void shutdown() { + + } + + @Override + public List shutdownNow() { + return null; + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return false; + } + + @Override + public Future submit(Callable task) { + throw new NotImplementedException(NOT_IMPLEMENTED); + } + + @Override + public Future submit(Runnable task, T result) { + throw new NotImplementedException(NOT_IMPLEMENTED); + } + + @Override + public Future submit(Runnable task) { + throw new NotImplementedException(NOT_IMPLEMENTED); + } + + @Override + public List> invokeAll(Collection> tasks) throws InterruptedException { + throw new NotImplementedException(NOT_IMPLEMENTED); + } + + @Override + public List> invokeAll(Collection> tasks, long timeout, TimeUnit unit) throws InterruptedException { + throw new NotImplementedException(NOT_IMPLEMENTED); + } + + @Override + public T invokeAny(Collection> tasks) throws InterruptedException, ExecutionException { + throw new NotImplementedException(NOT_IMPLEMENTED); + } + + @Override + public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + throw new NotImplementedException(NOT_IMPLEMENTED); + } + + @Override + public void execute(Runnable command) { + throw new NotImplementedException(NOT_IMPLEMENTED); + } } } From 67c4a346c89ab528acd74187aa9aaae64f4d6349 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sat, 11 Mar 2017 12:41:12 -0500 Subject: [PATCH 13/59] Decrease logging verbosity for the MongoDB driver --- src/main/resources/log4j.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/log4j.xml b/src/main/resources/log4j.xml index 6248868..75ed97a 100644 --- a/src/main/resources/log4j.xml +++ b/src/main/resources/log4j.xml @@ -28,6 +28,9 @@ + + + @@ -36,4 +39,3 @@ - \ No newline at end of file From 6db027c521d1182b646900d1a43f71b62c8a8e93 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sat, 11 Mar 2017 12:41:53 -0500 Subject: [PATCH 14/59] Log database operations timing --- src/main/java/dk/dma/ais/coverage/AisCoverage.java | 7 +++++++ .../dk/dma/ais/coverage/persistence/PersisterService.java | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/src/main/java/dk/dma/ais/coverage/AisCoverage.java b/src/main/java/dk/dma/ais/coverage/AisCoverage.java index ae7136b..f993b95 100644 --- a/src/main/java/dk/dma/ais/coverage/AisCoverage.java +++ b/src/main/java/dk/dma/ais/coverage/AisCoverage.java @@ -31,6 +31,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; +import java.time.Instant; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -90,7 +92,10 @@ public void accept(AisPacket packet) { } private void loadCoverageDataFromDatabase() { + Instant start = Instant.now(); Map> loadedCoverageData = databaseInstance.loadLatestSavedCoverageData(); + Instant databaseEnd = Instant.now(); + LOG.info("Loading coverage data from database took [{}] ms", Duration.between(start, databaseEnd).toMillis()); for (String sourceId : loadedCoverageData.keySet()) { for (Cell cell : loadedCoverageData.get(sourceId)) { @@ -99,6 +104,8 @@ private void loadCoverageDataFromDatabase() { adjustSystemEarliestMessageFromCell(cell); } } + Instant end = Instant.now(); + LOG.info("Loading coverage data and converting to memory structure took [{}] ms", Duration.between(start, end).toMillis()); } private void adjustSystemEarliestMessageFromCell(Cell cell) { diff --git a/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java index 96671ee..7068b67 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java @@ -6,6 +6,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; +import java.time.Instant; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; @@ -72,18 +74,22 @@ private class SaveOperation implements Runnable { @Override public void run() { + Instant start = Instant.now(); Map> cellsBySource = new LinkedHashMap<>(); for (Source source : coverageData.getSources()) { cellsBySource.put(source.getIdentifier(), source.getGrid().values()); } PersistenceResult persistenceResult = databaseInstance.save(cellsBySource); + Instant end = Instant.now(); if (PersistenceResult.Status.SUCCESS.equals(persistenceResult.getStatus())) { LOG.info("Saved [{}] cells to MongoDB database", persistenceResult.getWrittenCells()); } else { LOG.info("Failed saving cells to MongoDB database"); } + + LOG.info("Save operation took [{}] ms", Duration.between(start, end).toMillis()); } } } From 6d4374eac7868c695570dd3d0ddc4e5e07f2aacd Mon Sep 17 00:00:00 2001 From: John Martel Date: Sat, 11 Mar 2017 13:51:32 -0500 Subject: [PATCH 15/59] Use latest release versions of the DMA libraries --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 249ed4a..728e7a1 100644 --- a/pom.xml +++ b/pom.xml @@ -46,17 +46,17 @@ dk.dma.ais.lib ais-lib-communication - 2.2-SNAPSHOT + 2.2 dk.dma.commons dma-commons-app - 0.3-SNAPSHOT + 0.3 dk.dma.enav enav-model - 0.5-SNAPSHOT + 0.5 org.slf4j From 4018bc6aa8024a83b4173c66b93619944b5208b1 Mon Sep 17 00:00:00 2001 From: John Martel Date: Mon, 13 Mar 2017 19:55:31 -0400 Subject: [PATCH 16/59] Use single thread scheduled pool to persist data The thread pool created with `Executors#newSingleThreadScheduledExecutor()` ensured that if the thread dies, a new one will be created for future task execution. --- .../java/dk/dma/ais/coverage/persistence/PersisterService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java index 7068b67..0ced69d 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java @@ -24,7 +24,7 @@ public class PersisterService { private final DatabaseInstance databaseInstance; private final ICoverageData coverageData; private long persistenceIntervalInMinutes = 60; - private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + private ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); public PersisterService(DatabaseInstance databaseInstance, ICoverageData coverageData) { this.databaseInstance = databaseInstance; From 63fd115b163ea29d854c29a2fa41afec2f2f6253 Mon Sep 17 00:00:00 2001 From: John Martel Date: Tue, 21 Mar 2017 20:24:40 -0400 Subject: [PATCH 17/59] Log persistence errors --- .../dma/ais/coverage/persistence/PersisterService.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java index 0ced69d..fcdd2d3 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/PersisterService.java @@ -74,16 +74,22 @@ private class SaveOperation implements Runnable { @Override public void run() { + LOG.info("Starting save operation..."); Instant start = Instant.now(); Map> cellsBySource = new LinkedHashMap<>(); for (Source source : coverageData.getSources()) { cellsBySource.put(source.getIdentifier(), source.getGrid().values()); } - PersistenceResult persistenceResult = databaseInstance.save(cellsBySource); + PersistenceResult persistenceResult = null; + try { + persistenceResult = databaseInstance.save(cellsBySource); + } catch (RuntimeException e) { + LOG.error("Error while saving coverage data", e); + } Instant end = Instant.now(); - if (PersistenceResult.Status.SUCCESS.equals(persistenceResult.getStatus())) { + if ((persistenceResult != null) && PersistenceResult.Status.SUCCESS.equals(persistenceResult.getStatus())) { LOG.info("Saved [{}] cells to MongoDB database", persistenceResult.getWrittenCells()); } else { LOG.info("Failed saving cells to MongoDB database"); From 5c73e508ab116497a64a90318521d6cf867eccfa Mon Sep 17 00:00:00 2001 From: John Martel Date: Wed, 22 Mar 2017 19:43:02 -0400 Subject: [PATCH 18/59] Compress persisted coverage data --- pom.xml | 5 ++ .../persistence/MarshallingException.java | 13 ++++ .../MongoCoverageDataMarshaller.java | 60 ++++++++++++++++--- .../persistence/MongoDatabaseInstance.java | 28 ++++----- .../MongoCoverageDataMarshallerTest.java | 56 +++++++++++------ 5 files changed, 122 insertions(+), 40 deletions(-) create mode 100644 src/main/java/dk/dma/ais/coverage/persistence/MarshallingException.java diff --git a/pom.xml b/pom.xml index 728e7a1..f72635f 100644 --- a/pom.xml +++ b/pom.xml @@ -131,6 +131,11 @@ commons-collections4 4.1 + + commons-io + commons-io + 2.5 + diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MarshallingException.java b/src/main/java/dk/dma/ais/coverage/persistence/MarshallingException.java new file mode 100644 index 0000000..b7a162e --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/persistence/MarshallingException.java @@ -0,0 +1,13 @@ +package dk.dma.ais.coverage.persistence; + +import java.io.IOException; + +/** + * Indicates an exception while marshalling coverage data. + */ +public class MarshallingException extends RuntimeException { + + public MarshallingException(String message, IOException cause) { + super(message, cause); + } +} diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java b/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java index 1815046..7ef49be 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java @@ -1,21 +1,29 @@ package dk.dma.ais.coverage.persistence; +import dk.dma.ais.coverage.Helper; +import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.data.TimeSpan; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.bson.Document; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Collection; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; - -import org.bson.Document; - -import dk.dma.ais.coverage.Helper; -import dk.dma.ais.coverage.data.Cell; -import dk.dma.ais.coverage.data.TimeSpan; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; /** * A {@link CoverageDataMarshaller} that marshalls coverage data in a format usable to store in a MongoDB database. @@ -28,11 +36,16 @@ public Document marshall(Map> coverageData, ZonedDateTi coverageDataDocument.put("dataTimestamp", dataTimestamp.withZoneSameInstant(ZoneId.of("UTC")).format(DateTimeFormatter.ISO_DATE_TIME)); + Document documentToBeZipped = new Document(); List> grid = new ArrayList<>(); if (coverageData != null) { marshallCells(coverageData, grid); } - coverageDataDocument.put("cells", grid); + documentToBeZipped.put("cells", grid); + + String compressedCells = compressCellsData(documentToBeZipped); + coverageDataDocument.put("compressedCells", compressedCells); + coverageDataDocument.put("numberOfCells", grid.size()); return coverageDataDocument; } @@ -78,6 +91,20 @@ private Map> marshallCellTimeSpans(Cell cell) { return fixedWidthTimeSpans; } + private String compressCellsData(Document documentToBeZipped) { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + + try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream);) { + gzipOutputStream.write(documentToBeZipped.toJson().getBytes(StandardCharsets.US_ASCII)); + gzipOutputStream.close(); + + byte[] bytes = byteArrayOutputStream.toByteArray(); + return new String(Base64.getEncoder().encode(bytes), StandardCharsets.US_ASCII); + } catch (IOException e) { + throw new MarshallingException("Could not compress cells data", e); + } + } + @Override public Map> unmarshall(Document coverageData) { Map> unmarshalledCoverageData = new LinkedHashMap<>(); @@ -91,7 +118,8 @@ public Map> unmarshall(Document coverageData) { private Map> unmarshallCells(Document coverageData) { Map> unmarshalledCoverageData = new LinkedHashMap<>(); - Object cells = coverageData.get("cells"); + Document decompressedCells = decompressCells(coverageData); + Object cells = decompressedCells.get("cells"); if (cells != null && (cells instanceof List)) { List> grid = (List>) cells; @@ -109,6 +137,22 @@ private Map> unmarshallCells(Document coverageData) { return unmarshalledCoverageData; } + private Document decompressCells(Document marshalledCoverageData) { + String base64 = (String) marshalledCoverageData.get("compressedCells"); + if (!StringUtils.isBlank(base64)) { + byte[] gzippedData = Base64.getDecoder().decode(base64.getBytes(StandardCharsets.US_ASCII)); + + try (GZIPInputStream gzipInputStream = new GZIPInputStream(new ByteArrayInputStream(gzippedData))) { + String jsonData = IOUtils.toString(gzipInputStream, StandardCharsets.US_ASCII); + return Document.parse(jsonData); + } catch (IOException e) { + throw new MarshallingException("Could not unmarshall compressed cells data", e); + } + } + + return new Document(); + } + private Cell unmarshallCell(Map cell) { double latitude = Helper.roundLat((double) cell.get("latitude"), 1); double longitude = Helper.roundLon((double) cell.get("longitude"), 1); diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java b/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java index 4e0edd9..81dc5d1 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/MongoDatabaseInstance.java @@ -1,17 +1,5 @@ package dk.dma.ais.coverage.persistence; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.bson.Document; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.mongodb.MongoClient; import com.mongodb.MongoClientOptions; import com.mongodb.MongoException; @@ -20,6 +8,16 @@ import com.mongodb.client.MongoDatabase; import dk.dma.ais.coverage.configuration.DatabaseConfiguration; import dk.dma.ais.coverage.data.Cell; +import org.bson.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; /** * {@link DatabaseInstance} implementation that keeps data in a MongoDB database. @@ -114,13 +112,13 @@ public PersistenceResult save(Map> coverageData) { requireOpenConnection(); Document coverageDataDocument = marshaller.marshall(coverageData, ZonedDateTime.now(ZoneId.of("UTC"))); - long numberOfSavedCells = ((List>) coverageDataDocument.get("cells")).size(); + long numberOfSavedCells = ((Integer) coverageDataDocument.get("numberOfCells")); try { client.getDatabase(databaseName).getCollection(COVERAGE_DATA).insertOne(coverageDataDocument); LOG.info("Saved [{}] cells with timestamp [{}]", numberOfSavedCells, coverageDataDocument.get("dataTimestamp")); return PersistenceResult.success(numberOfSavedCells); - } catch (MongoException e) { + } catch (MongoException | MarshallingException e) { logAndTransformException(e); } @@ -149,7 +147,7 @@ public Map> loadLatestSavedCoverageData() { LOG.info("Loaded [{}] cells from previously saved state at [{}]", loadedCells, document.get("dataTimestamp")); } - } catch (MongoException e) { + } catch (MongoException | MarshallingException e) { logAndTransformException(e); } diff --git a/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java b/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java index 026b8fb..9fd0890 100644 --- a/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java +++ b/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java @@ -1,31 +1,37 @@ package dk.dma.ais.coverage.persistence; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.closeTo; -import static org.junit.Assert.assertThat; +import dk.dma.ais.coverage.Helper; +import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; +import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.data.TimeSpan; +import dk.dma.ais.coverage.fixture.CellFixture; +import org.apache.commons.io.IOUtils; +import org.bson.Document; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Base64; import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.zip.GZIPInputStream; -import org.bson.Document; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import dk.dma.ais.coverage.Helper; -import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; -import dk.dma.ais.coverage.data.Cell; -import dk.dma.ais.coverage.data.TimeSpan; -import dk.dma.ais.coverage.fixture.CellFixture; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.closeTo; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; public class MongoCoverageDataMarshallerTest { private static final double DELTA_WHEN_COMPARING_DOUBLE = 0.0001D; @@ -41,7 +47,6 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { Helper.conf = null; - } @Test @@ -60,7 +65,23 @@ public void givenNoCoverageData_whenMarshall_thenDocumentWithEmptyCellsArrayAndT private void assertThatDocumentHasEmptyCellsAndDataTimestamp(Document marshalledCoverageData, ZonedDateTime now) { ZonedDateTime dataTimestamp = ZonedDateTime.parse((String) marshalledCoverageData.get("dataTimestamp"), DateTimeFormatter.ISO_DATE_TIME); assertThat(dataTimestamp, is(equalTo(now))); - assertThat(((List>) marshalledCoverageData.get("cells")).isEmpty(), is(true)); + + Document decompressedCells = decompressCells(marshalledCoverageData); + assertThat(((List>) decompressedCells.get("cells")).isEmpty(), is(true)); + } + + private Document decompressCells(Document marshalledCoverageData) { + String base64 = (String) marshalledCoverageData.get("compressedCells"); + byte[] gzippedData = Base64.getDecoder().decode(base64.getBytes(StandardCharsets.US_ASCII)); + + try (GZIPInputStream gzipInputStream = new GZIPInputStream(new ByteArrayInputStream(gzippedData))) { + String jsonData = IOUtils.toString(gzipInputStream, StandardCharsets.US_ASCII); + return Document.parse(jsonData); + } catch (IOException e) { + fail("Failed decompressing cells: " + e.getMessage()); + } + + return null; } @Test @@ -80,7 +101,8 @@ public void givenManyCells_whenMarshall_thenDocumentWithTheseCellsAndTimestampIs ZonedDateTime dataTimestamp = ZonedDateTime.parse((String) marshalledCoverageData.get("dataTimestamp"), DateTimeFormatter.ISO_DATE_TIME); assertThat(dataTimestamp, is(equalTo(now))); - List> grid = (List>) marshalledCoverageData.get("cells"); + Document decompressedCells = decompressCells(marshalledCoverageData); + List> grid = (List>) decompressedCells.get("cells"); assertThat(grid.size(), is(equalTo(2))); Map firstMarshalledCell = grid.get(0); From d0109a1ee671eee815d7e16a83b54e5ea8432afe Mon Sep 17 00:00:00 2001 From: John Martel Date: Fri, 24 Mar 2017 22:04:57 -0400 Subject: [PATCH 19/59] Reduce message filtering due to SoG The previous filter would filter out too many messages, creating gaps in the canadian coverage. During winter, in the Saint-Lawrence Seaway, ships could be slowed down dramatically or stopped due to ice, but there is still a need to receive the messages and calculate coverage for these. --- .../ais/coverage/calculator/AbstractCalculator.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/calculator/AbstractCalculator.java b/src/main/java/dk/dma/ais/coverage/calculator/AbstractCalculator.java index a8c6e18..c7640b2 100644 --- a/src/main/java/dk/dma/ais/coverage/calculator/AbstractCalculator.java +++ b/src/main/java/dk/dma/ais/coverage/calculator/AbstractCalculator.java @@ -14,11 +14,6 @@ */ package dk.dma.ais.coverage.calculator; -import java.io.Serializable; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import dk.dma.ais.coverage.AisCoverageGUI; import dk.dma.ais.coverage.calculator.geotools.SphereProjection; import dk.dma.ais.coverage.data.CustomMessage; @@ -30,6 +25,12 @@ import dk.dma.ais.message.AisMessage5; import dk.dma.ais.message.ShipTypeCargo; import dk.dma.ais.message.ShipTypeCargo.ShipType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Serializable; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * See CoverageCalculator and DensityPlotCalculator for examples of how to extend this class. When a calculator is added to an @@ -137,7 +138,7 @@ public double getExpectedTransmittingFrequency(double sog, boolean rotating, Shi */ public boolean filterMessage(CustomMessage customMessage) { - if (customMessage.getSog() < 3 || customMessage.getSog() > 50) { + if (customMessage.getSog() < 0 || customMessage.getSog() > 150) { return true; } if (customMessage.getCog() == 360) { From 787684ede0b89769238a5ece928d2350e0f38ffd Mon Sep 17 00:00:00 2001 From: John Martel Date: Fri, 24 Mar 2017 22:07:14 -0400 Subject: [PATCH 20/59] Bump version number --- coverage-sample.xml | 48 +++++++++++++++++++++++++++++++++++++++++++++ pom.xml | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 coverage-sample.xml diff --git a/coverage-sample.xml b/coverage-sample.xml new file mode 100644 index 0000000..85c4bc6 --- /dev/null +++ b/coverage-sample.xml @@ -0,0 +1,48 @@ + + + + + + + 802 + 804 + 810 + CanAISShore + + + 205.193.156.26:9001 + + + + UNFILTERED + + + + + 0.0225225225 + 0.0386812541 + + + MongoDB + localhost + nordicCoverage + 27017 + 2 + + + + + + + + + 8090 + web + /coverage + + 720 + diff --git a/pom.xml b/pom.xml index f72635f..5f44978 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ dk.dma.ais ais-coverage - 0.2.1-CCG + 0.2.2-CCG AIS coverage analyzer AIS coverage analyzer From 98a15d548b3f15a964646eb30a909c8fe5fa11bf Mon Sep 17 00:00:00 2001 From: John Martel Date: Mon, 13 Mar 2017 19:58:10 -0400 Subject: [PATCH 21/59] Bump version number --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5f44978..1a8d09e 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ dk.dma.ais ais-coverage - 0.2.2-CCG + 0.3.0-CCG AIS coverage analyzer AIS coverage analyzer From 426b4627f545758c10ba22c8c30eb68c1f93a490 Mon Sep 17 00:00:00 2001 From: John Martel Date: Mon, 13 Mar 2017 19:58:45 -0400 Subject: [PATCH 22/59] Use SNAPSHOT version of dma-ais libraries --- pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1a8d09e..cb31b33 100644 --- a/pom.xml +++ b/pom.xml @@ -46,7 +46,7 @@ dk.dma.ais.lib ais-lib-communication - 2.2 + 2.4-SNAPSHOT dk.dma.commons @@ -139,6 +139,11 @@ + + dma-snapshots + Dma Snapshot Repository + http://repository-dma.forge.cloudbees.com/snapshot/ + dma-releases Dma Release Repository From 4ea76a52bd5addd267708efcff6d4eb730046703 Mon Sep 17 00:00:00 2001 From: John Martel Date: Mon, 13 Mar 2017 21:29:41 -0400 Subject: [PATCH 23/59] Support VSI data in cells --- .../java/dk/dma/ais/coverage/data/Cell.java | 84 ++++++++++++++++--- .../dk/dma/ais/coverage/data/TimeSpan.java | 41 +++++++-- .../dk/dma/ais/coverage/data/CellTest.java | 71 ++++++++++++++++ .../dma/ais/coverage/data/TimeSpanTest.java | 58 +++++++++++++ .../dma/ais/coverage/fixture/CellFixture.java | 18 ++-- 5 files changed, 245 insertions(+), 27 deletions(-) create mode 100644 src/test/java/dk/dma/ais/coverage/data/CellTest.java create mode 100644 src/test/java/dk/dma/ais/coverage/data/TimeSpanTest.java diff --git a/src/main/java/dk/dma/ais/coverage/data/Cell.java b/src/main/java/dk/dma/ais/coverage/data/Cell.java index 5921c92..3c687cb 100644 --- a/src/main/java/dk/dma/ais/coverage/data/Cell.java +++ b/src/main/java/dk/dma/ais/coverage/data/Cell.java @@ -21,9 +21,10 @@ import java.util.Map; public class Cell { - private int NOofReceivedSignals; private int NOofMissingSignals; + private int numberOfVsiMessages; + private int averageSignalStrength; private double latitude; private double longitude; private List timeSpans; @@ -55,19 +56,29 @@ public Cell(double lat, double lon, String id) { this.longitude = lon; } - public void incrementNOofReceivedSignals() { + public synchronized void incrementNOofReceivedSignals() { NOofReceivedSignals++; } - public void incrementNOofMissingSignals() { + public synchronized void incrementNOofMissingSignals() { NOofMissingSignals++; } - public long getTotalNumberOfMessages() { + public synchronized void incrementNumberOfVsiMessages(int signalStrength) { + int incrementedNumberOfVsiMessages = numberOfVsiMessages + 1; + averageSignalStrength = computeAverageSignalStrength(signalStrength, incrementedNumberOfVsiMessages); + numberOfVsiMessages = incrementedNumberOfVsiMessages; + } + + private int computeAverageSignalStrength(int signalStrength, int incrementedNumberOfVsiMessages) { + return Math.floorDiv((numberOfVsiMessages * averageSignalStrength) + signalStrength, incrementedNumberOfVsiMessages); + } + + public synchronized long getTotalNumberOfMessages() { return NOofReceivedSignals + NOofMissingSignals; } - public double getCoverage() { + public synchronized double getCoverage() { return (double) NOofReceivedSignals / (double) getTotalNumberOfMessages(); } @@ -91,11 +102,11 @@ public String getId() { return this.latitude + "_" + this.longitude; } - public int getNOofReceivedSignals(Date starttime, Date endTime) { + public synchronized int getNOofReceivedSignals(Date starttime, Date endTime) { int result = 0; Collection spans = fixedWidthSpans.values(); - for (TimeSpan timeSpan : spans) { + for (TimeSpan timeSpan : spans) { if (timeSpan.getFirstMessage().getTime() >= starttime.getTime() && timeSpan.getLastMessage().getTime() <= endTime.getTime()) { @@ -106,36 +117,83 @@ public int getNOofReceivedSignals(Date starttime, Date endTime) { return result; } - public int getNOofMissingSignals(Date starttime, Date endTime) { + public synchronized int getNOofMissingSignals(Date starttime, Date endTime) { int result = 0; Collection spans = fixedWidthSpans.values(); + for (TimeSpan timeSpan : spans) { if (timeSpan.getFirstMessage().getTime() >= starttime.getTime() && timeSpan.getLastMessage().getTime() <= endTime.getTime()) { result = result + timeSpan.getMissingSignals(); } } + return result; } - public int getNOofReceivedSignals() { + public synchronized int getNOofReceivedSignals() { return this.NOofReceivedSignals; } - public int getNOofMissingSignals() { + public synchronized int getNOofMissingSignals() { return this.NOofMissingSignals; } - public void addReceivedSignals(int amount) { + public synchronized void addReceivedSignals(int amount) { this.NOofReceivedSignals += amount; } - public void addNOofMissingSignals(int amount) { + public synchronized void addNOofMissingSignals(int amount) { this.NOofMissingSignals += amount; } - public void setNoofMissingSignals(int amount) { + public synchronized void setNoofMissingSignals(int amount) { this.NOofMissingSignals = amount; } + public synchronized int getNumberOfVsiMessages() { + return numberOfVsiMessages; + } + + public synchronized int getAverageSignalStrength() { + return averageSignalStrength; + } + + public synchronized int getNumberOfVsiMessages(Date startTime, Date endTime) { + int result = 0; + Collection spans = fixedWidthSpans.values(); + + for (TimeSpan timeSpan : spans) { + if (timeSpan.getFirstMessage().getTime() >= startTime.getTime() + && timeSpan.getLastMessage().getTime() <= endTime.getTime()) { + + result = result + timeSpan.getVsiMessageCounter(); + } + } + + return result; + } + + public synchronized int getAverageSignalStrength(Date startTime, Date endTime) { + int summedAverageSignalStrength = 0; + int numberOfVsiMessages = getNumberOfVsiMessages(startTime, endTime); + Collection spans = fixedWidthSpans.values(); + + for (TimeSpan timeSpan : spans) { + if (timeSpan.getFirstMessage().getTime() >= startTime.getTime() + && timeSpan.getLastMessage().getTime() <= endTime.getTime()) { + + int currentTimeSpanTotalSignalStrength = timeSpan.getAverageSignalStrength() * timeSpan.getVsiMessageCounter(); + summedAverageSignalStrength = summedAverageSignalStrength + currentTimeSpanTotalSignalStrength; + } + } + + return Math.floorDiv(summedAverageSignalStrength, numberOfVsiMessages); + } + + public synchronized void addVsiMessages(int numberOfVsiMessages, int averageSignalStrength) { + int incrementedNumberOfVsiMessages = this.numberOfVsiMessages + numberOfVsiMessages; + this.averageSignalStrength = computeAverageSignalStrength(numberOfVsiMessages * averageSignalStrength, incrementedNumberOfVsiMessages); + this.numberOfVsiMessages = incrementedNumberOfVsiMessages; + } } diff --git a/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java b/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java index e10c86f..5314245 100644 --- a/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java +++ b/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java @@ -26,6 +26,8 @@ public class TimeSpan { private int messageCounterTerrestrial; private int missingSignals; private int messageCounterTerrestrialUnfiltered; + private int vsiMessageCounter; + private int averageSignalStrength; public int getMessageCounterTerrestrialUnfiltered() { return messageCounterTerrestrialUnfiltered; @@ -100,24 +102,51 @@ public void setMessageCounterSat(int messageCounter) { this.messageCounterSat = messageCounter; } - public void add(TimeSpan span2) { - this.setMessageCounterSat(this.getMessageCounterSat() + span2.getMessageCounterSat()); - this.setMessageCounterTerrestrial(this.getMessageCounterTerrestrial() + span2.getMessageCounterTerrestrial()); - this.addMessageCounterTerrestrialUnfiltered(span2.getMessageCounterTerrestrialUnfiltered()); - for (String s : span2.distinctShipsSat.keySet()) { + public int getVsiMessageCounter() { + return vsiMessageCounter; + } + + public void setVsiMessageCounter(int vsiMessageCounter) { + this.vsiMessageCounter = vsiMessageCounter; + } + + public int getAverageSignalStrength() { + return averageSignalStrength; + } + + public void setAverageSignalStrength(int averageSignalStrength) { + this.averageSignalStrength = averageSignalStrength; + } + + public void add(TimeSpan other) { + this.setMessageCounterSat(this.getMessageCounterSat() + other.getMessageCounterSat()); + this.setMessageCounterTerrestrial(this.getMessageCounterTerrestrial() + other.getMessageCounterTerrestrial()); + this.addMessageCounterTerrestrialUnfiltered(other.getMessageCounterTerrestrialUnfiltered()); + this.setAverageSignalStrength(sumAverageSignalStrength(other)); + this.setVsiMessageCounter(this.getVsiMessageCounter() + other.getVsiMessageCounter()); + for (String s : other.distinctShipsSat.keySet()) { this.distinctShipsSat.put(s, true); } - for (String s : span2.distinctShipsTerrestrial.keySet()) { + for (String s : other.distinctShipsTerrestrial.keySet()) { this.distinctShipsTerrestrial.put(s, true); } } + private int sumAverageSignalStrength(TimeSpan other) { + int thisAggregatedSignalStrength = this.getVsiMessageCounter() * this.getAverageSignalStrength(); + int otherAggregatedSignalStrength = other.getVsiMessageCounter() * other.getAverageSignalStrength(); + int summedVsiMessageCounters = this.getVsiMessageCounter() + other.getVsiMessageCounter(); + return Math.floorDiv(thisAggregatedSignalStrength + otherAggregatedSignalStrength, summedVsiMessageCounters); + } + public TimeSpan copy() { TimeSpan copy = new TimeSpan(this.getFirstMessage()); copy.setLastMessage(this.getLastMessage()); copy.setMessageCounterSat(this.getMessageCounterSat()); copy.setMessageCounterTerrestrial(this.getMessageCounterTerrestrial()); copy.setMessageCounterTerrestrialUnfiltered(this.messageCounterTerrestrialUnfiltered); + copy.setVsiMessageCounter(this.getVsiMessageCounter()); + copy.setAverageSignalStrength(this.getAverageSignalStrength()); for (String s : this.distinctShipsSat.keySet()) { copy.distinctShipsSat.put(s, true); } diff --git a/src/test/java/dk/dma/ais/coverage/data/CellTest.java b/src/test/java/dk/dma/ais/coverage/data/CellTest.java new file mode 100644 index 0000000..3c79431 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/data/CellTest.java @@ -0,0 +1,71 @@ +package dk.dma.ais.coverage.data; + +import dk.dma.ais.coverage.Helper; +import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; +import dk.dma.ais.coverage.fixture.CellFixture; +import org.junit.Before; +import org.junit.Test; + +import java.util.Iterator; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class CellTest { + + @Before + public void setUp() throws Exception { + Helper.conf = new AisCoverageConfiguration(); + } + + @Test + public void whenIncrementNumberOfVsiMessages_thenNumberOfVsiMessagesHasOneAdded() { + Cell aCell = CellFixture.createCellWithNoTimeSpan(); + + aCell.incrementNumberOfVsiMessages(-10); + aCell.incrementNumberOfVsiMessages(-20); + + assertThat(aCell.getNumberOfVsiMessages(), is(equalTo(2))); + assertThat(aCell.getAverageSignalStrength(), is(equalTo(-15))); + } + + @Test + public void whenAddVsiMessages_thenNumberOfVsiMessagesAreSummedAndAverageSignalStrengthIsRecomputed() { + Cell aCell = CellFixture.createCellWithNoTimeSpan(); + + aCell.incrementNumberOfVsiMessages(-10); + aCell.incrementNumberOfVsiMessages(-20); + + aCell.addVsiMessages(3, -12); + + assertThat(aCell.getNumberOfVsiMessages(), is(equalTo(5))); + assertThat(aCell.getAverageSignalStrength(), is(equalTo(-14))); + } + + @Test + public void whenGettNumberOfVsiMessagesWithTimestamps_thenNumberOfVsiMessagesFromMatchingTimespansAreReturned() { + Cell aCell = CellFixture.createCellWithTimeSpans(); + TimeSpan firstTimespan = aCell.getFixedWidthSpans().values().iterator().next(); + + int numberOfVsiMessages = aCell.getNumberOfVsiMessages(firstTimespan.getFirstMessage(), firstTimespan.getLastMessage()); + + assertThat(numberOfVsiMessages, is(equalTo(firstTimespan.getVsiMessageCounter()))); + } + + @Test + public void whengetAverageSignalStrength_thenAverageSignalStrengthFromMatchingTimespansAreReturned() { + Cell aCell = CellFixture.createCellWithTimeSpans(); + Iterator timeSpanIterator = aCell.getFixedWidthSpans().values().iterator(); + TimeSpan firstTimespan = timeSpanIterator.next(); + TimeSpan secondTimespan = timeSpanIterator.next(); + + int averageSignalStrength = aCell.getAverageSignalStrength(firstTimespan.getFirstMessage(), secondTimespan.getLastMessage()); + + int firstTimespanAverageSignalStrength = firstTimespan.getAverageSignalStrength() * firstTimespan.getVsiMessageCounter(); + int secondTimespanAverageSignalStrength = secondTimespan.getAverageSignalStrength() * secondTimespan.getVsiMessageCounter(); + int totalVsiMessagesForBothTimespans = firstTimespan.getVsiMessageCounter() + secondTimespan.getVsiMessageCounter(); + int expectedAverageSignalStrength = Math.floorDiv(firstTimespanAverageSignalStrength + secondTimespanAverageSignalStrength, totalVsiMessagesForBothTimespans); + assertThat(averageSignalStrength, is(equalTo(expectedAverageSignalStrength))); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/data/TimeSpanTest.java b/src/test/java/dk/dma/ais/coverage/data/TimeSpanTest.java new file mode 100644 index 0000000..6dc40a1 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/data/TimeSpanTest.java @@ -0,0 +1,58 @@ +package dk.dma.ais.coverage.data; + +import org.junit.Before; +import org.junit.Test; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Date; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class TimeSpanTest { + private Date now; + private TimeSpan aTimeSpan; + private TimeSpan anotherTimeSpan; + + @Before + public void setUp() throws Exception { + now = new Date(ZonedDateTime.now(ZoneId.of("UTC")).toInstant().toEpochMilli()); + aTimeSpan = new TimeSpan(now); + anotherTimeSpan = new TimeSpan(now); + } + + @Test + public void whenAdd_thenVsiMessageCounterIsSumOfCurrentAndAddedTimeSpan() { + aTimeSpan.setVsiMessageCounter(5); + anotherTimeSpan.setVsiMessageCounter(5); + + aTimeSpan.add(anotherTimeSpan); + + assertThat(aTimeSpan.getVsiMessageCounter(), is(equalTo(10))); + } + + @Test + public void whenAdd_thenAverageSignalStrengthComputesAddedTimeSpan() { + aTimeSpan.setVsiMessageCounter(9); + aTimeSpan.setAverageSignalStrength(0); + anotherTimeSpan.setVsiMessageCounter(1); + anotherTimeSpan.setAverageSignalStrength(-100); + + aTimeSpan.add(anotherTimeSpan); + + assertThat(aTimeSpan.getAverageSignalStrength(), is(equalTo(-10))); + } + + @Test + public void whenCopy_thenCopyHasSameVsiMessageCounterAndAverageSignalStrength() { + aTimeSpan.setVsiMessageCounter(9); + aTimeSpan.setAverageSignalStrength(-100); + + TimeSpan copy = aTimeSpan.copy(); + + assertThat(copy.getVsiMessageCounter(), is(equalTo(9))); + assertThat(copy.getAverageSignalStrength(), is(equalTo(-100))); + } +} diff --git a/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java b/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java index 700ec6a..8646e34 100644 --- a/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java +++ b/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java @@ -1,18 +1,17 @@ package dk.dma.ais.coverage.fixture; +import dk.dma.ais.coverage.Helper; +import dk.dma.ais.coverage.calculator.geotools.SphereProjection; +import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.data.TimeSpan; +import org.apache.commons.lang3.RandomUtils; + import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.Date; import java.util.LinkedHashMap; import java.util.Map; -import org.apache.commons.lang3.RandomUtils; - -import dk.dma.ais.coverage.Helper; -import dk.dma.ais.coverage.calculator.geotools.SphereProjection; -import dk.dma.ais.coverage.data.Cell; -import dk.dma.ais.coverage.data.TimeSpan; - public class CellFixture { public static Cell createCellWithTimeSpans() { @@ -44,12 +43,15 @@ private static Map createTimeSpans() { Map fixedWidthTimeSpans = new LinkedHashMap<>(); for (int i = 0; i <= 1; i++) { - long timespanKey = ZonedDateTime.now(ZoneId.of("UTC")).toInstant().toEpochMilli() + i; + long timespanKey = ZonedDateTime.now(ZoneId.of("UTC")).toInstant().toEpochMilli() + (i * 10000); TimeSpan timespan = new TimeSpan(new Date(timespanKey)); + timespan.setLastMessage(new Date(timespanKey + ((i + 1) * 10000))); timespan.setMessageCounterSat(RandomUtils.nextInt()); timespan.setMessageCounterTerrestrial(RandomUtils.nextInt()); timespan.setMessageCounterTerrestrialUnfiltered(RandomUtils.nextInt()); timespan.setMissingSignals(RandomUtils.nextInt()); + timespan.setVsiMessageCounter(RandomUtils.nextInt(1, 10)); + timespan.setAverageSignalStrength(Math.negateExact(RandomUtils.nextInt(10, 60))); fixedWidthTimeSpans.put(timespanKey, timespan); } From 598a53f83eecc25e523cbe951dd6b6de1618b9c5 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sat, 25 Mar 2017 11:16:38 -0400 Subject: [PATCH 24/59] Do not filter out messages due to Course Over Ground The canadian data shows that many messages are received with a CoG of 360, which is basically the same as 0. Since we do not filter CoGs of 0, I fail to see why we would filter out those messages with 360. --- .../dk/dma/ais/coverage/calculator/AbstractCalculator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/calculator/AbstractCalculator.java b/src/main/java/dk/dma/ais/coverage/calculator/AbstractCalculator.java index c7640b2..d9b4d06 100644 --- a/src/main/java/dk/dma/ais/coverage/calculator/AbstractCalculator.java +++ b/src/main/java/dk/dma/ais/coverage/calculator/AbstractCalculator.java @@ -139,9 +139,7 @@ public double getExpectedTransmittingFrequency(double sog, boolean rotating, Shi public boolean filterMessage(CustomMessage customMessage) { if (customMessage.getSog() < 0 || customMessage.getSog() > 150) { - return true; - } - if (customMessage.getCog() == 360) { + LOG.info("filtering message due to SoG: {}, lat: {}, long: {}", customMessage.getSog(), customMessage.getLatitude(), customMessage.getLongitude()); return true; } @@ -156,12 +154,14 @@ public boolean filterMessage(CustomMessage customMessage) { double distance = projection.distBetweenPoints(firstMessage.getLongitude(), firstMessage.getLatitude(), lastMessage.getLongitude(), lastMessage.getLatitude()); if (distance > 2000) { + LOG.info("filtering message due to distance: {}, lat: {}, long: {}", distance, customMessage.getLatitude(), customMessage.getLongitude()); return true; } // Filter message based on time between first and last message double timeDifference = this.getTimeDifference(firstMessage, lastMessage); if (timeDifference > 1200) { + LOG.info("filtering message due to time difference: {}, lat: {}, long: {}", timeDifference, customMessage.getLatitude(), customMessage.getLongitude()); return true; } From 2a9afe6476c460468858d0755920e27dbe038f73 Mon Sep 17 00:00:00 2001 From: Daniel Levesque Date: Sat, 25 Mar 2017 12:34:20 -0400 Subject: [PATCH 25/59] Make sure we only process messages when message buffer is full instead of processing 10000 times the same message --- src/main/java/dk/dma/ais/coverage/CoverageHandler.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/CoverageHandler.java b/src/main/java/dk/dma/ais/coverage/CoverageHandler.java index 1ec5803..d2325e5 100644 --- a/src/main/java/dk/dma/ais/coverage/CoverageHandler.java +++ b/src/main/java/dk/dma/ais/coverage/CoverageHandler.java @@ -38,7 +38,8 @@ */ public class CoverageHandler { private static final Logger LOG = LoggerFactory.getLogger(CoverageHandler.class); - + public static final int MESSAGE_BUFFER_SIZE = 10000; + private List calculators = new ArrayList(); private ICoverageData dataHandler; private AisCoverageConfiguration conf; @@ -49,8 +50,10 @@ public class CoverageHandler { @Override protected boolean removeEldestEntry(Map.Entry eldest) { - process(eldest.getValue()); - return this.size() > 10000; + if (this.size() > MESSAGE_BUFFER_SIZE) { + process(eldest.getValue()); + } + return this.size() > MESSAGE_BUFFER_SIZE; } }; From 1f3f39944eac67bf281ce3faeabb75e7ad420df2 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sat, 25 Mar 2017 17:42:48 -0400 Subject: [PATCH 26/59] Support VSI messages --- pom.xml | 2 +- .../dk/dma/ais/coverage/CoverageHandler.java | 23 +-- .../calculator/TerrestrialCalculator.java | 33 ++-- .../java/dk/dma/ais/coverage/data/Cell.java | 12 +- .../dma/ais/coverage/data/CustomMessage.java | 25 ++- .../dma/ais/coverage/data/ICoverageData.java | 8 +- .../dma/ais/coverage/data/OnlyMemoryData.java | 142 ++++++++++-------- .../dk/dma/ais/coverage/data/TimeSpan.java | 10 ++ .../MongoCoverageDataMarshaller.java | 16 ++ .../dk/dma/ais/coverage/data/CellTest.java | 4 +- .../dma/ais/coverage/data/TimeSpanTest.java | 11 ++ .../dma/ais/coverage/fixture/CellFixture.java | 3 +- .../MongoCoverageDataMarshallerTest.java | 20 +++ 13 files changed, 205 insertions(+), 104 deletions(-) diff --git a/pom.xml b/pom.xml index cb31b33..b0958d3 100644 --- a/pom.xml +++ b/pom.xml @@ -46,7 +46,7 @@ dk.dma.ais.lib ais-lib-communication - 2.4-SNAPSHOT + 2.4-CCG-SNAPSHOT dk.dma.commons diff --git a/src/main/java/dk/dma/ais/coverage/CoverageHandler.java b/src/main/java/dk/dma/ais/coverage/CoverageHandler.java index d2325e5..2aad68a 100644 --- a/src/main/java/dk/dma/ais/coverage/CoverageHandler.java +++ b/src/main/java/dk/dma/ais/coverage/CoverageHandler.java @@ -14,15 +14,6 @@ */ package dk.dma.ais.coverage; -import java.util.ArrayList; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import dk.dma.ais.coverage.calculator.AbstractCalculator; import dk.dma.ais.coverage.calculator.SatCalculator; import dk.dma.ais.coverage.calculator.TerrestrialCalculator; @@ -32,6 +23,14 @@ import dk.dma.ais.coverage.data.OnlyMemoryData; import dk.dma.ais.coverage.data.Ship; import dk.dma.ais.packet.AisPacket; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; /** * Handler for received AisPackets @@ -117,7 +116,7 @@ public void receiveUnfiltered(AisPacket packet) { //extract relevant information from packet CustomMessage message = dataHandler.packetToCustomMessage(packet); - if(message == null){ + if (message == null) { return; } @@ -128,7 +127,9 @@ public void receiveUnfiltered(AisPacket packet) { if (existing == null) { doubletBuffer.put(key, message); }else{ - existing.addSourceMMSI(message.getSourceList().iterator().next()); + if (message.getSourceList().iterator().hasNext()) { + existing.addSourceMMSI(message.getSourceList().iterator().next()); + } } } diff --git a/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java b/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java index e4fe1a4..0eea6b4 100644 --- a/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java +++ b/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java @@ -14,15 +14,6 @@ */ package dk.dma.ais.coverage.calculator; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - import dk.dma.ais.coverage.AisCoverage; import dk.dma.ais.coverage.Helper; import dk.dma.ais.coverage.configuration.AisCoverageConfiguration; @@ -30,20 +21,20 @@ import dk.dma.ais.coverage.data.CustomMessage; import dk.dma.ais.coverage.data.QueryParams; import dk.dma.ais.coverage.data.Ship; -import dk.dma.ais.coverage.data.Ship.ShipClass; -import dk.dma.ais.coverage.data.Source; -import dk.dma.ais.coverage.data.Source.ReceiverType; -import dk.dma.ais.coverage.data.Source_UserProvided; import dk.dma.ais.coverage.event.AisEvent; -import dk.dma.ais.coverage.event.AisEvent.Event; import dk.dma.ais.coverage.event.IAisEventListener; import dk.dma.ais.coverage.export.data.ExportCell; import dk.dma.ais.coverage.export.data.JSonCoverageMap; import dk.dma.ais.coverage.export.data.JsonConverter; -import dk.dma.ais.message.AisMessage; -import dk.dma.ais.message.AisMessage4; -import dk.dma.ais.message.AisPositionMessage; -import dk.dma.ais.proprietary.IProprietarySourceTag; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * This calculator expects a filtered data stream! (No doublets) The stream must not be downsampled! @@ -103,7 +94,11 @@ private boolean checkDoublets(CustomMessage m) { * This is called whenever a message is received */ public void calculate(CustomMessage message) { - + if (message.isVsi()) { + dataHandler.incrementReceivedVsiMessage(AbstractCalculator.SUPERSOURCE_MMSI, message.getLatitude(), message.getLongitude(), message.getTimestamp(), message.getSignalStrength()); + approveMessage(message); + return; + } Ship ship = dataHandler.getShip(message.getShipMMSI()); diff --git a/src/main/java/dk/dma/ais/coverage/data/Cell.java b/src/main/java/dk/dma/ais/coverage/data/Cell.java index 3c687cb..890d044 100644 --- a/src/main/java/dk/dma/ais/coverage/data/Cell.java +++ b/src/main/java/dk/dma/ais/coverage/data/Cell.java @@ -71,7 +71,11 @@ public synchronized void incrementNumberOfVsiMessages(int signalStrength) { } private int computeAverageSignalStrength(int signalStrength, int incrementedNumberOfVsiMessages) { - return Math.floorDiv((numberOfVsiMessages * averageSignalStrength) + signalStrength, incrementedNumberOfVsiMessages); + if (incrementedNumberOfVsiMessages > 0) { + return Math.floorDiv((numberOfVsiMessages * averageSignalStrength) + signalStrength, incrementedNumberOfVsiMessages); + } else { + return 0; + } } public synchronized long getTotalNumberOfMessages() { @@ -188,7 +192,11 @@ public synchronized int getAverageSignalStrength(Date startTime, Date endTime) { } } - return Math.floorDiv(summedAverageSignalStrength, numberOfVsiMessages); + if (numberOfVsiMessages > 0) { + return Math.floorDiv(summedAverageSignalStrength, numberOfVsiMessages); + } else { + return 0; + } } public synchronized void addVsiMessages(int numberOfVsiMessages, int averageSignalStrength) { diff --git a/src/main/java/dk/dma/ais/coverage/data/CustomMessage.java b/src/main/java/dk/dma/ais/coverage/data/CustomMessage.java index 231565a..0069d8c 100644 --- a/src/main/java/dk/dma/ais/coverage/data/CustomMessage.java +++ b/src/main/java/dk/dma/ais/coverage/data/CustomMessage.java @@ -14,16 +14,14 @@ */ package dk.dma.ais.coverage.data; +import dk.dma.ais.message.AisMessage; +import dk.dma.ais.packet.AisPacketTags.SourceType; + import java.io.Serializable; import java.util.Date; import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; import java.util.Set; -import dk.dma.ais.message.AisMessage; -import dk.dma.ais.packet.AisPacketTags.SourceType; - /** * Used for storing information relevant for the calculators */ @@ -41,6 +39,8 @@ public class CustomMessage implements Serializable { private long timeSinceLastMsg; private String key; private SourceType sourceType; + private boolean vsi; + private int signalStrength; public SourceType getSourceType() { return sourceType; @@ -125,4 +125,19 @@ public void setOriginalMessage(AisMessage originalMessage) { this.originalMessage = originalMessage; } + public void setVsi(boolean vsi) { + this.vsi = vsi; + } + + public boolean isVsi() { + return vsi; + } + + public void setSignalStrength(int signalStrength) { + this.signalStrength = signalStrength; + } + + public int getSignalStrength() { + return signalStrength; + } } diff --git a/src/main/java/dk/dma/ais/coverage/data/ICoverageData.java b/src/main/java/dk/dma/ais/coverage/data/ICoverageData.java index f4b5044..e1649f5 100644 --- a/src/main/java/dk/dma/ais/coverage/data/ICoverageData.java +++ b/src/main/java/dk/dma/ais/coverage/data/ICoverageData.java @@ -14,13 +14,13 @@ */ package dk.dma.ais.coverage.data; +import dk.dma.ais.coverage.data.Ship.ShipClass; +import dk.dma.ais.packet.AisPacket; + import java.util.Collection; import java.util.Date; import java.util.List; -import dk.dma.ais.coverage.data.Ship.ShipClass; -import dk.dma.ais.packet.AisPacket; - public interface ICoverageData { CustomMessage packetToCustomMessage(AisPacket packet); @@ -37,5 +37,7 @@ public interface ICoverageData { Collection getSources(); void incrementReceivedSignals(String sourceMmsi, double lat, double lon, Date timestamp); void incrementMissingSignals(String sourceMmsi, double lat, double lon, Date timestamp); + void incrementReceivedVsiMessage(String sourceMmsi, double latitude, double longitude, Date timestamp, int signalStrength); void trimWindow(Date trimPoint); + } diff --git a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java index 4e7f625..feec588 100644 --- a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java +++ b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java @@ -14,19 +14,6 @@ */ package dk.dma.ais.coverage.data; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import dk.dma.ais.coverage.AisCoverage; import dk.dma.ais.coverage.Helper; import dk.dma.ais.coverage.calculator.AbstractCalculator; @@ -41,6 +28,18 @@ import dk.dma.ais.packet.AisPacketTags.SourceType; import dk.dma.ais.proprietary.IProprietarySourceTag; import dk.dma.enav.model.geometry.Position; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; public class OnlyMemoryData implements ICoverageData { private static final Logger LOG = LoggerFactory @@ -103,6 +102,14 @@ private void updateCellTimeSpans(Cell oldCell, Cell newCell) { updatedTimeSpan.setMessageCounterTerrestrial(oldTimeSpan.getMessageCounterTerrestrial() + newTimeSpan.getValue().getMessageCounterTerrestrial()); updatedTimeSpan.setMessageCounterTerrestrialUnfiltered(oldTimeSpan.getMessageCounterTerrestrialUnfiltered() + newTimeSpan.getValue().getMessageCounterTerrestrialUnfiltered()); updatedTimeSpan.setMissingSignals(oldTimeSpan.getMissingSignals() + newTimeSpan.getValue().getMissingSignals()); + int oldVsiMessageCounter = oldTimeSpan.getVsiMessageCounter(); + int newVsiMessageCounter = newTimeSpan.getValue().getVsiMessageCounter(); + if (oldVsiMessageCounter + newVsiMessageCounter > 0) { + updatedTimeSpan.setVsiMessageCounter(oldVsiMessageCounter + newVsiMessageCounter); + updatedTimeSpan.setAverageSignalStrength( + ((oldVsiMessageCounter * oldTimeSpan.getAverageSignalStrength()) + (newVsiMessageCounter * newTimeSpan.getValue().getAverageSignalStrength())) + / updatedTimeSpan.getVsiMessageCounter()); + } oldCell.getFixedWidthSpans().put(newTimeSpan.getKey(), updatedTimeSpan); } @@ -126,8 +133,8 @@ private List getCells() { cells.add(cell); } } - } + return cells; } @@ -179,20 +186,15 @@ private List getCells(double latStart, double lonStart, Collection bscells = source.getGrid().values(); for (Cell cell : bscells) { - if (Helper.isInsideBox(cell, latStart, lonStart, latEnd, - lonEnd)) { - Cell tempCell = cellMultiplicationSource.getCell( - cell.getLatitude(), cell.getLongitude()); + if (Helper.isInsideBox(cell, latStart, lonStart, latEnd, lonEnd)) { + Cell tempCell = cellMultiplicationSource.getCell(cell.getLatitude(), cell.getLongitude()); if (tempCell == null) { - tempCell = cellMultiplicationSource.createCell( - cell.getLatitude(), cell.getLongitude()); + tempCell = cellMultiplicationSource.createCell(cell.getLatitude(), cell.getLongitude()); } - tempCell.addNOofMissingSignals((int) cell - .getNOofMissingSignals(starttime, endtime)); - tempCell.addReceivedSignals(cell - .getNOofReceivedSignals(starttime, endtime)); + tempCell.addNOofMissingSignals((int) cell.getNOofMissingSignals(starttime, endtime)); + tempCell.addReceivedSignals(cell.getNOofReceivedSignals(starttime, endtime)); + tempCell.addVsiMessages(cell.getNumberOfVsiMessages(starttime, endtime), cell.getAverageSignalStrength(starttime, endtime)); } - } // add cells for particular source to cell-list. @@ -218,31 +220,23 @@ public List getCells(QueryParams params) { } @Override - public void incrementReceivedSignals(String sourceMmsi, double lat, - double lon, Date timestamp) { - Cell cell = getCell(sourceMmsi, lat, lon); - if (cell == null) { - cell = createCell(sourceMmsi, lat, lon); - } - Date id = Helper.getFloorDate(timestamp); - TimeSpan ts = cell.getFixedWidthSpans().get(id.getTime()); - if (ts == null) { - ts = new TimeSpan(id); - ts.setLastMessage(Helper.getCeilDate(timestamp)); - cell.getFixedWidthSpans().put(id.getTime(), ts); - } + public void incrementReceivedSignals(String sourceMmsi, double lat, double lon, Date timestamp) { + Cell cell = getCellFromCoordinates(sourceMmsi, lat, lon); + TimeSpan ts = getTimeSpanFromTimestamp(timestamp, cell); + ts.setMessageCounterTerrestrial(ts.getMessageCounterTerrestrial() + 1); cell.incrementNOofReceivedSignals(); } - @Override - public void incrementMissingSignals(String sourceMmsi, double lat, - double lon, Date timestamp) { - + private Cell getCellFromCoordinates(String sourceMmsi, double lat, double lon) { Cell cell = getCell(sourceMmsi, lat, lon); if (cell == null) { cell = createCell(sourceMmsi, lat, lon); } + return cell; + } + + private TimeSpan getTimeSpanFromTimestamp(Date timestamp, Cell cell) { Date id = Helper.getFloorDate(timestamp); TimeSpan ts = cell.getFixedWidthSpans().get(id.getTime()); if (ts == null) { @@ -250,10 +244,27 @@ public void incrementMissingSignals(String sourceMmsi, double lat, ts.setLastMessage(Helper.getCeilDate(timestamp)); cell.getFixedWidthSpans().put(id.getTime(), ts); } + return ts; + } + + @Override + public void incrementMissingSignals(String sourceMmsi, double lat, double lon, Date timestamp) { + Cell cell = getCellFromCoordinates(sourceMmsi, lat, lon); + TimeSpan ts = getTimeSpanFromTimestamp(timestamp, cell); + ts.incrementMissingSignals(); cell.incrementNOofMissingSignals(); } + @Override + public void incrementReceivedVsiMessage(String sourceMmsi, double latitude, double longitude, Date timestamp, int signalStrength) { + Cell cell = getCellFromCoordinates(sourceMmsi, latitude, longitude); + TimeSpan ts = getTimeSpanFromTimestamp(timestamp, cell); + + ts.incrementNumberOfVsiMessages(signalStrength); + cell.incrementNumberOfVsiMessages(signalStrength); + } + public CustomMessage packetToCustomMessage(AisPacket packet) { AisMessage aisMessage = packet.tryGetAisMessage(); @@ -353,25 +364,36 @@ public CustomMessage packetToCustomMessage(AisPacket packet) { } } - // Extract ship - Ship ship = getShip(aisMessage.getUserId()); - if (ship == null) { - ship = createShip(aisMessage.getUserId(), shipClass); - } + if (packet.isVsi()) { + CustomMessage newMessage = new CustomMessage(); + newMessage.setVsi(true); + newMessage.setSignalStrength(packet.getVsi().getSignalStrength()); + newMessage.setLatitude(posMessage.getPos().getGeoLocation().getLatitude()); + newMessage.setLongitude(posMessage.getPos().getGeoLocation().getLongitude()); + newMessage.setTimestamp(packet.getVsi().getTimestamp()); - CustomMessage newMessage = new CustomMessage(); - newMessage.setCog((double) posMessage.getCog() / 10); - newMessage.setSog((double) posMessage.getSog() / 10); - newMessage.setLatitude(posMessage.getPos().getGeoLocation() - .getLatitude()); - newMessage.setLongitude(posMessage.getPos().getGeoLocation() - .getLongitude()); - newMessage.setTimestamp(timestamp); - newMessage.addSourceMMSI(baseId); - newMessage.setShipMMSI(aisMessage.getUserId()); - newMessage.setSourceType(sourceType); - - return newMessage; + return newMessage; + } else { + // Extract ship + Ship ship = getShip(aisMessage.getUserId()); + if (ship == null) { + ship = createShip(aisMessage.getUserId(), shipClass); + } + + CustomMessage newMessage = new CustomMessage(); + newMessage.setCog((double) posMessage.getCog() / 10); + newMessage.setSog((double) posMessage.getSog() / 10); + newMessage.setLatitude(posMessage.getPos().getGeoLocation() + .getLatitude()); + newMessage.setLongitude(posMessage.getPos().getGeoLocation() + .getLongitude()); + newMessage.setTimestamp(timestamp); + newMessage.addSourceMMSI(baseId); + newMessage.setShipMMSI(aisMessage.getUserId()); + newMessage.setSourceType(sourceType); + + return newMessage; + } } @Override diff --git a/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java b/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java index 5314245..21d13ab 100644 --- a/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java +++ b/src/main/java/dk/dma/ais/coverage/data/TimeSpan.java @@ -118,6 +118,16 @@ public void setAverageSignalStrength(int averageSignalStrength) { this.averageSignalStrength = averageSignalStrength; } + public synchronized void incrementNumberOfVsiMessages(int signalStrength) { + int incrementedNumberOfVsiMessages = vsiMessageCounter + 1; + averageSignalStrength = computeAverageSignalStrength(signalStrength, incrementedNumberOfVsiMessages); + vsiMessageCounter = incrementedNumberOfVsiMessages; + } + + private int computeAverageSignalStrength(int signalStrength, int incrementedNumberOfVsiMessages) { + return Math.floorDiv((vsiMessageCounter * averageSignalStrength) + signalStrength, incrementedNumberOfVsiMessages); + } + public void add(TimeSpan other) { this.setMessageCounterSat(this.getMessageCounterSat() + other.getMessageCounterSat()); this.setMessageCounterTerrestrial(this.getMessageCounterTerrestrial() + other.getMessageCounterTerrestrial()); diff --git a/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java b/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java index 7ef49be..1b42f4e 100644 --- a/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java +++ b/src/main/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshaller.java @@ -67,6 +67,8 @@ private Map marshallCell(String sourceId, Cell cell) { savedCell.put("longitude", cell.getLongitude()); savedCell.put("numberOfReceivedSignals", cell.getNOofReceivedSignals()); savedCell.put("numberOfMissingSignals", cell.getNOofMissingSignals()); + savedCell.put("numberOfVsiMessages", cell.getNumberOfVsiMessages()); + savedCell.put("averageSignalStrength", cell.getAverageSignalStrength()); Map> fixedWidthTimeSpans = marshallCellTimeSpans(cell); savedCell.put("timespans", fixedWidthTimeSpans); @@ -84,6 +86,8 @@ private Map> marshallCellTimeSpans(Cell cell) { messages.put("messageCounterTerrestrial", fixedWidthTimeSpan.getValue().getMessageCounterTerrestrial()); messages.put("messageCounterTerrestrialUnfiltered", fixedWidthTimeSpan.getValue().getMessageCounterTerrestrialUnfiltered()); messages.put("missingSignals", fixedWidthTimeSpan.getValue().getMissingSignals()); + messages.put("vsiMessageCounter", fixedWidthTimeSpan.getValue().getVsiMessageCounter()); + messages.put("averageSignalStrength", fixedWidthTimeSpan.getValue().getAverageSignalStrength()); fixedWidthTimeSpans.put(fixedWidthTimeSpan.getKey().toString(), messages); } @@ -160,6 +164,11 @@ private Cell unmarshallCell(Map cell) { unmarshalledCell.addReceivedSignals(((Integer) cell.get("numberOfReceivedSignals")).intValue()); unmarshalledCell.addNOofMissingSignals(((Integer) cell.get("numberOfMissingSignals")).intValue()); + Integer numberOfVsiMessages = (Integer) cell.get("numberOfVsiMessages"); + if (numberOfVsiMessages != null && numberOfVsiMessages > 0) { + unmarshalledCell.addVsiMessages(numberOfVsiMessages.intValue(), ((Integer) cell.get("averageSignalStrength")).intValue()); + } + Map> fixedWidthTimeSpans = (Map>) cell.get("timespans"); Map unmarshalledTimeSpans = unmarshallTimeSpans(fixedWidthTimeSpans); @@ -176,6 +185,13 @@ private Map unmarshallTimeSpans(Map> unmarshalledTimeSpan.setMessageCounterTerrestrial(timespan.getValue().get("messageCounterTerrestrial").intValue()); unmarshalledTimeSpan.setMessageCounterTerrestrialUnfiltered(timespan.getValue().get("messageCounterTerrestrialUnfiltered").intValue()); unmarshalledTimeSpan.setMissingSignals(timespan.getValue().get("missingSignals").intValue()); + + Integer numberOfVsiMessages = (Integer) timespan.getValue().get("vsiMessageCounter"); + if (numberOfVsiMessages != null) { + unmarshalledTimeSpan.setVsiMessageCounter(numberOfVsiMessages.intValue()); + unmarshalledTimeSpan.setAverageSignalStrength(((Integer) timespan.getValue().get("averageSignalStrength")).intValue()); + } + unmarshalledTimeSpans.put(Long.valueOf(timespan.getKey()), unmarshalledTimeSpan); } return unmarshalledTimeSpans; diff --git a/src/test/java/dk/dma/ais/coverage/data/CellTest.java b/src/test/java/dk/dma/ais/coverage/data/CellTest.java index 3c79431..16009fd 100644 --- a/src/test/java/dk/dma/ais/coverage/data/CellTest.java +++ b/src/test/java/dk/dma/ais/coverage/data/CellTest.java @@ -44,7 +44,7 @@ public void whenAddVsiMessages_thenNumberOfVsiMessagesAreSummedAndAverageSignalS } @Test - public void whenGettNumberOfVsiMessagesWithTimestamps_thenNumberOfVsiMessagesFromMatchingTimespansAreReturned() { + public void whenGetNumberOfVsiMessagesWithTimestamps_thenNumberOfVsiMessagesFromMatchingTimespansAreReturned() { Cell aCell = CellFixture.createCellWithTimeSpans(); TimeSpan firstTimespan = aCell.getFixedWidthSpans().values().iterator().next(); @@ -54,7 +54,7 @@ public void whenGettNumberOfVsiMessagesWithTimestamps_thenNumberOfVsiMessagesFro } @Test - public void whengetAverageSignalStrength_thenAverageSignalStrengthFromMatchingTimespansAreReturned() { + public void whenGetAverageSignalStrength_thenAverageSignalStrengthFromMatchingTimespansAreReturned() { Cell aCell = CellFixture.createCellWithTimeSpans(); Iterator timeSpanIterator = aCell.getFixedWidthSpans().values().iterator(); TimeSpan firstTimespan = timeSpanIterator.next(); diff --git a/src/test/java/dk/dma/ais/coverage/data/TimeSpanTest.java b/src/test/java/dk/dma/ais/coverage/data/TimeSpanTest.java index 6dc40a1..a4747ca 100644 --- a/src/test/java/dk/dma/ais/coverage/data/TimeSpanTest.java +++ b/src/test/java/dk/dma/ais/coverage/data/TimeSpanTest.java @@ -55,4 +55,15 @@ public void whenCopy_thenCopyHasSameVsiMessageCounterAndAverageSignalStrength() assertThat(copy.getVsiMessageCounter(), is(equalTo(9))); assertThat(copy.getAverageSignalStrength(), is(equalTo(-100))); } + + @Test + public void whenIncrementNumberOfVsiMessages_thenNumberOfVsiMessagesHasOneAdded() { + aTimeSpan.setVsiMessageCounter(1); + aTimeSpan.setAverageSignalStrength(-15); + + aTimeSpan.incrementNumberOfVsiMessages(-10); + + assertThat(aTimeSpan.getVsiMessageCounter(), is(equalTo(2))); + assertThat(aTimeSpan.getAverageSignalStrength(), is(equalTo(-13))); + } } diff --git a/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java b/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java index 8646e34..f343b19 100644 --- a/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java +++ b/src/test/java/dk/dma/ais/coverage/fixture/CellFixture.java @@ -28,7 +28,8 @@ private static Cell createCell() { double longitude = randomLongitude(latitude); String cellId = Helper.getCellId(latitude, longitude, 1); - return new Cell(latitude, longitude, cellId); + Cell cell = new Cell(latitude, longitude, cellId); + return cell; } public static double randomLatitude() { diff --git a/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java b/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java index 9fd0890..67a6aed 100644 --- a/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java +++ b/src/test/java/dk/dma/ais/coverage/persistence/MongoCoverageDataMarshallerTest.java @@ -109,6 +109,10 @@ public void givenManyCells_whenMarshall_thenDocumentWithTheseCellsAndTimestampIs assertThat(firstMarshalledCell.get("cellId"), is(equalTo(firstCell.getId()))); assertThat(firstMarshalledCell.get("latitude"), is(equalTo(firstCell.getLatitude()))); assertThat(firstMarshalledCell.get("longitude"), is(equalTo(firstCell.getLongitude()))); + assertThat(firstMarshalledCell.get("numberOfReceivedSignals"), is(equalTo(firstCell.getNOofReceivedSignals()))); + assertThat(firstMarshalledCell.get("numberOfMissingSignals"), is(equalTo(firstCell.getNOofMissingSignals()))); + assertThat(firstMarshalledCell.get("numberOfVsiMessages"), is(equalTo(firstCell.getNumberOfVsiMessages()))); + assertThat(firstMarshalledCell.get("averageSignalStrength"), is(equalTo(firstCell.getAverageSignalStrength()))); Map> firstMarshalledCellTimespans = (Map>) firstMarshalledCell.get("timespans"); assertThat(firstMarshalledCellTimespans.size(), is(equalTo(2))); @@ -123,12 +127,18 @@ public void givenManyCells_whenMarshall_thenDocumentWithTheseCellsAndTimestampIs assertThat(marshalledTimeSpan.get("messageCounterTerrestrial"), is(equalTo(expectedTimeSpan.getMessageCounterTerrestrial()))); assertThat(marshalledTimeSpan.get("messageCounterTerrestrialUnfiltered"), is(equalTo(expectedTimeSpan.getMessageCounterTerrestrialUnfiltered()))); assertThat(marshalledTimeSpan.get("missingSignals"), is(equalTo(expectedTimeSpan.getMissingSignals()))); + assertThat(marshalledTimeSpan.get("vsiMessageCounter"), is(equalTo(expectedTimeSpan.getVsiMessageCounter()))); + assertThat(marshalledTimeSpan.get("averageSignalStrength"), is(equalTo(expectedTimeSpan.getAverageSignalStrength()))); } Map secondMarshalledCell = grid.get(1); assertThat(secondMarshalledCell.get("cellId"), is(equalTo(secondCell.getId()))); assertThat(secondMarshalledCell.get("latitude"), is(equalTo(secondCell.getLatitude()))); assertThat(secondMarshalledCell.get("longitude"), is(equalTo(secondCell.getLongitude()))); + assertThat(secondMarshalledCell.get("numberOfReceivedSignals"), is(equalTo(secondCell.getNOofReceivedSignals()))); + assertThat(secondMarshalledCell.get("numberOfMissingSignals"), is(equalTo(secondCell.getNOofMissingSignals()))); + assertThat(secondMarshalledCell.get("numberOfVsiMessages"), is(equalTo(secondCell.getNumberOfVsiMessages()))); + assertThat(secondMarshalledCell.get("averageSignalStrength"), is(equalTo(secondCell.getAverageSignalStrength()))); Map> secondMarshalledCellTimespans = (Map>) secondMarshalledCell.get("timespans"); assertThat(secondMarshalledCellTimespans.isEmpty(), is(true)); @@ -169,6 +179,10 @@ public void givenManyCells_whenUnmarshall_thenTheseCellsAreReturned() { assertThat(firstUnmarshalledCell.getId(), is(equalTo(firstCell.getId()))); assertThat(firstUnmarshalledCell.getLatitude(), is(closeTo(firstCell.getLatitude(), DELTA_WHEN_COMPARING_DOUBLE))); assertThat(firstUnmarshalledCell.getLongitude(), is(closeTo(firstCell.getLongitude(), DELTA_WHEN_COMPARING_DOUBLE))); + assertThat(firstUnmarshalledCell.getNOofReceivedSignals(), is(equalTo(firstCell.getNOofReceivedSignals()))); + assertThat(firstUnmarshalledCell.getNOofMissingSignals(), is(equalTo(firstCell.getNOofMissingSignals()))); + assertThat(firstUnmarshalledCell.getNumberOfVsiMessages(), is(equalTo(firstCell.getNumberOfVsiMessages()))); + assertThat(firstUnmarshalledCell.getAverageSignalStrength(), is(equalTo(firstCell.getAverageSignalStrength()))); Map firstUnmarshalledCellFixedWidthSpans = firstUnmarshalledCell.getFixedWidthSpans(); assertThat(firstUnmarshalledCellFixedWidthSpans.size(), is(equalTo(2))); @@ -183,12 +197,18 @@ public void givenManyCells_whenUnmarshall_thenTheseCellsAreReturned() { assertThat(unmarshalledTimeSpan.getMessageCounterTerrestrial(), is(equalTo(expectedTimeSpan.getMessageCounterTerrestrial()))); assertThat(unmarshalledTimeSpan.getMessageCounterTerrestrialUnfiltered(), is(equalTo(expectedTimeSpan.getMessageCounterTerrestrialUnfiltered()))); assertThat(unmarshalledTimeSpan.getMissingSignals(), is(equalTo(expectedTimeSpan.getMissingSignals()))); + assertThat(unmarshalledTimeSpan.getVsiMessageCounter(), is(equalTo(expectedTimeSpan.getVsiMessageCounter()))); + assertThat(unmarshalledTimeSpan.getAverageSignalStrength(), is(equalTo(expectedTimeSpan.getAverageSignalStrength()))); } Cell secondUnmarshalledCell = cellsFromDefaultSource.next(); assertThat(secondUnmarshalledCell.getId(), is(equalTo(secondCell.getId()))); assertThat(secondUnmarshalledCell.getLatitude(), is(closeTo(secondCell.getLatitude(), DELTA_WHEN_COMPARING_DOUBLE))); assertThat(secondUnmarshalledCell.getLongitude(), is(closeTo(secondCell.getLongitude(), DELTA_WHEN_COMPARING_DOUBLE))); + assertThat(secondUnmarshalledCell.getNOofReceivedSignals(), is(equalTo(secondCell.getNOofReceivedSignals()))); + assertThat(secondUnmarshalledCell.getNOofMissingSignals(), is(equalTo(secondCell.getNOofMissingSignals()))); + assertThat(secondUnmarshalledCell.getNumberOfVsiMessages(), is(equalTo(secondCell.getNumberOfVsiMessages()))); + assertThat(secondUnmarshalledCell.getAverageSignalStrength(), is(equalTo(secondCell.getAverageSignalStrength()))); assertThat(secondUnmarshalledCell.getFixedWidthSpans().isEmpty(), is(true)); } From 18b4493c39416d12a83696e38af0b904a0e4cc9e Mon Sep 17 00:00:00 2001 From: Daniel Levesque Date: Sat, 25 Mar 2017 17:59:35 -0400 Subject: [PATCH 27/59] Add Coverage Type Filter in Coverage Settings Modify ExportCell and JsonConverter to support VSI messages Handle display of VSI messages in map --- .../ais/coverage/export/data/ExportCell.java | 2 + .../coverage/export/data/JsonConverter.java | 22 +- web/css/style.css | 4 +- web/index.html | 24 +- web/js/Copy of CoverageUI.js | 506 ------------------ web/js/CoverageUI.js | 196 ++++--- 6 files changed, 155 insertions(+), 599 deletions(-) delete mode 100644 web/js/Copy of CoverageUI.js diff --git a/src/main/java/dk/dma/ais/coverage/export/data/ExportCell.java b/src/main/java/dk/dma/ais/coverage/export/data/ExportCell.java index 9bd9b7d..565959a 100644 --- a/src/main/java/dk/dma/ais/coverage/export/data/ExportCell.java +++ b/src/main/java/dk/dma/ais/coverage/export/data/ExportCell.java @@ -23,6 +23,8 @@ public class ExportCell implements Serializable { public double lon; public long nrOfRecMes; public long nrOfMisMes; + public long numberOfVsiMessages; + public long averageSignalStrength; public String sourceMmsi; public double getCoverage() { diff --git a/src/main/java/dk/dma/ais/coverage/export/data/JsonConverter.java b/src/main/java/dk/dma/ais/coverage/export/data/JsonConverter.java index eb54de1..cb3b536 100644 --- a/src/main/java/dk/dma/ais/coverage/export/data/JsonConverter.java +++ b/src/main/java/dk/dma/ais/coverage/export/data/JsonConverter.java @@ -14,17 +14,13 @@ */ package dk.dma.ais.coverage.export.data; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import dk.dma.ais.coverage.calculator.AbstractCalculator; import dk.dma.ais.coverage.data.Cell; import dk.dma.ais.coverage.data.Source; import dk.dma.ais.coverage.data.TimeSpan; +import org.apache.commons.lang3.RandomUtils; public class JsonConverter { @@ -61,11 +57,13 @@ public static ExportCell toJsonCell(Cell cell, Cell superCell, Date starttime, D long expected = superCell.getNOofReceivedSignals() + superCell.getNOofMissingSignals(); // System.out.println(superCell.getNOofMissingSignals()); - ExportCell Jcell = new ExportCell(); - Jcell.lat = cell.getLatitude(); - Jcell.lon = cell.getLongitude(); - Jcell.nrOfMisMes = expected - cell.getNOofReceivedSignals(); - Jcell.nrOfRecMes = cell.getNOofReceivedSignals(); + ExportCell jsonCell = new ExportCell(); + jsonCell.lat = cell.getLatitude(); + jsonCell.lon = cell.getLongitude(); + jsonCell.nrOfMisMes = expected - cell.getNOofReceivedSignals(); + jsonCell.nrOfRecMes = cell.getNOofReceivedSignals(); + jsonCell.numberOfVsiMessages = cell.getNumberOfVsiMessages(); + jsonCell.averageSignalStrength = cell.getAverageSignalStrength(); // if(expected < Jcell.nrOfRecMes){ // System.out.println("supercell received="+superCell.getNOofReceivedSignals()); @@ -76,7 +74,7 @@ public static ExportCell toJsonCell(Cell cell, Cell superCell, Date starttime, D // System.out.println(); // } - return Jcell; + return jsonCell; } public static List toJsonTimeSpan(List timespans) { diff --git a/web/css/style.css b/web/css/style.css index 7b93ef3..68ebe99 100644 --- a/web/css/style.css +++ b/web/css/style.css @@ -228,8 +228,8 @@ hr { .colorbox { margin-left: 3px; float: left; - width: 20px; - height: 20px; + width: 15px; + height: 15px; } #redbox { diff --git a/web/index.html b/web/index.html index cfc642b..5f23c1c 100644 --- a/web/index.html +++ b/web/index.html @@ -31,7 +31,7 @@ var y=document.getElementById("expotype").options; exportType.value = y[x].text; } - + @@ -40,12 +40,6 @@ - - - - - -
@@ -58,9 +52,19 @@
Select all' - }); - $('#exportPanel').expandable({ - header: "Export" - }); - $('#shiptrackPanel').expandable({ - header: "Ship Tracking" - }); + var shiptrackactive = false; + var satChartMethod = 'satonly'; + + + this.setupUI = function () { + + // Set zoom panel positon + $(".olControlZoom").css('left', zoomPanelPositionLeft); + $(".olControlZoom").css('top', zoomPanelPositionTop); + + // Set loading panel positon + var x = $(document).width() / 2 - $("#loadingPanel").width() / 2; + $("#loadingPanel").css('left', x); + + // Update mouse location when moved + map.events.register("mousemove", map, function (e) { + var position = this.events.getMousePosition(e); + pixel = new OpenLayers.Pixel(position.x, position.y); + var lonLat = map.getLonLatFromPixel(pixel).transform( + map.getProjectionObject(), // from Spherical Mercator Projection + new OpenLayers.Projection("EPSG:4326") // to WGS 1984 + ); + $("#location").html(lonLat.lon.toFixed(4) + ", " + lonLat.lat.toFixed(4)); + }); + + // Init expandable panels + $('#thresholdPanel').expandable({ + header: "Coverage Settings" + }); + $('#featureDetailsPanel').expandable({ + header: "Source Details" + }); + $('#sourcesPanel').expandable({ + header: 'Sources
Select all' + }); + $('#exportPanel').expandable({ + header: "Export" + }); + $('#shiptrackPanel').expandable({ + header: "Ship Tracking" + }); // $('#bottomPanel').expandable({ // header: "Satellite Statistics", // maxHeight: "800px" @@ -68,588 +67,594 @@ function CoverageUI () { // header: "Sliding Window", // maxHeight: "800px" // }); - - //add check box listeners - $(document).on('change', ".sourceCheckbox", function(e) { - self.sources[this.id].enabled=$(this).is(':checked'); - self.refreshSourceList(); - self.refreshSourceDetails(); - self.changed=true; - }); - - $(document).on('change', "#selectall", function(e) { - var element = $(this); - $.each(self.sources, function(key, source) { - source.enabled=element.is(':checked'); - }); - self.refreshSourceList(); - self.refreshSourceDetails(); - self.changed=true; - }); - - $(document).on('change', "#coverageType", function(e) { - var maxThresholdToSave = self.maxThreshold; - var minThresholdToSave = self.minThreshold; - var thresholdUnitToSave = self.thresholdUnit; - - self.maxThreshold = self.savedMaxThreshold; - self.minThreshold = self.savedMinThreshold; - self.thresholdUnit = self.savedThresholdUnit; - - self.savedMaxThreshold = maxThresholdToSave; - self.savedMinThreshold = minThresholdToSave; - self.savedThresholdUnit = thresholdUnitToSave; + + //add check box listeners + $(document).on('change', ".sourceCheckbox", function (e) { + self.sources[this.id].enabled = $(this).is(':checked'); + self.refreshSourceList(); + self.refreshSourceDetails(); + self.changed = true; + }); + + $(document).on('change', "#selectall", function (e) { + var element = $(this); + $.each(self.sources, function (key, source) { + source.enabled = element.is(':checked'); + }); + self.refreshSourceList(); + self.refreshSourceDetails(); + self.changed = true; + }); + + $(document).on('change', "#coverageType", function (e) { + var maxThresholdToSave = self.maxThreshold; + var minThresholdToSave = self.minThreshold; + var thresholdUnitToSave = self.thresholdUnit; + + self.maxThreshold = self.savedMaxThreshold; + self.minThreshold = self.savedMinThreshold; + self.thresholdUnit = self.savedThresholdUnit; + + self.savedMaxThreshold = maxThresholdToSave; + self.savedMinThreshold = minThresholdToSave; + self.savedThresholdUnit = thresholdUnitToSave; self.drawSlider(); self.cleanupCellDetails(); - self.changed=true; + self.changed = true; }); - - //Close window listeners - $('.close').parent().hide(); - $('.close').click(function(){ - $(this).parent().fadeOut(100); - //remove sat rectangle + + //Close window listeners + $('.close').parent().hide(); + $('.close').click(function () { + $(this).parent().fadeOut(100); + //remove sat rectangle // boxLayer.removeFeatures([boxLayer.features[0]]); - boxLayer.removeAllFeatures(); - - }); - - //Open ship tracking chart listener - $('#shiptrackingOpenChartBtn').click(function(){ + boxLayer.removeAllFeatures(); + + }); + + //Open ship tracking chart listener + $('#shiptrackingOpenChartBtn').click(function () { // alert("open bar chart"); - self.updateShipTrackBarChart(); - $('#trackingBarchartWindow').fadeIn(100); - - }); - - //Draw ship track listener - $('#shiptrackingMapBtn').click(function(){ - console.log("draw ship track"); + self.updateShipTrackBarChart(); + $('#trackingBarchartWindow').fadeIn(100); + + }); + + //Draw ship track listener + $('#shiptrackingMapBtn').click(function () { + console.log("draw ship track"); // self.drawLine(55,6,58,9); - self.updateShipTrack(); - - }); - - //Chart method change - listener - $( "#chartMethodSelect" ).change(function() { - satChartMethod=$( this ).val(); - self.updateBarChart(); - }); - - //setup the threshold slider - self.drawSlider(); - - //setup min expected messages per cell slider - var minExpected = $("#minExpected"); - minExpected.html(self.minExpectedMessages); - $( "#filterSlider" ).slider({ - min: 0, - max: 1000, - value: self.minExpectedMessages, - slide: function(event, ui){ - self.minExpectedMessages=ui.value; - minExpected.html(ui.value); - }, - change: function(event, ui){ - self.changed = true; - } - }); - - //setup sliding window - self.updateSlidingWindow(); - - //setup - var exportMultiplicationDivHidden = $("#exportMultiHidden"); - var exportMultiplicationDiv = $("#exportMultiplicationFactor"); - exportMultiplicationDivHidden.val(self.exportMultiplicationFactor); - exportMultiplicationDiv.html(self.exportMultiplicationFactor); - $( "#multiplicationSlider" ).slider({ - min: 1, - max: 60, - value: self.exportMultiplicationFactor, - slide: function(event, ui){ - self.minExpectedMessages=ui.value; - exportMultiplicationDiv.html(ui.value); - exportMultiplicationDivHidden.val(ui.value); - } - }); - - //setting the loop function - var myTimeout; - function loopFunction () {; - if(self.changed){ - self.drawCoverage(); - //self.updateSlidingWindow(); - self.changed = false; - } - myTimeout = setTimeout(loopFunction, 2000); - } - myTimeout = setTimeout(loopFunction, 2000); - - //map changed listener - map.events.register('moveend', this, function (event) { - - //update export div - self.exportMultiplicationFactor = self.getMultiplicationFactor(); - exportMultiplicationDivHidden.val(self.exportMultiplicationFactor); - exportMultiplicationDiv.html(self.exportMultiplicationFactor); - $( "#multiplicationSlider" ).slider({value:self.getMultiplicationFactor()}); - - - //draw coverage - coverageUI.drawCoverage(); - //self.updateSlidingWindow(); - + self.updateShipTrack(); + + }); + + //Chart method change - listener + $("#chartMethodSelect").change(function () { + satChartMethod = $(this).val(); + self.updateBarChart(); + }); + + //setup the threshold slider + self.drawSlider(); + + //setup min expected messages per cell slider + var minExpected = $("#minExpected"); + minExpected.html(self.minExpectedMessages); + $("#filterSlider").slider({ + min: 0, + max: 1000, + value: self.minExpectedMessages, + slide: function (event, ui) { + self.minExpectedMessages = ui.value; + minExpected.html(ui.value); + }, + change: function (event, ui) { + self.changed = true; + } + }); + + //setup sliding window + self.updateSlidingWindow(); + + //setup + var exportMultiplicationDivHidden = $("#exportMultiHidden"); + var exportMultiplicationDiv = $("#exportMultiplicationFactor"); + exportMultiplicationDivHidden.val(self.exportMultiplicationFactor); + exportMultiplicationDiv.html(self.exportMultiplicationFactor); + $("#multiplicationSlider").slider({ + min: 1, + max: 60, + value: self.exportMultiplicationFactor, + slide: function (event, ui) { + self.minExpectedMessages = ui.value; + exportMultiplicationDiv.html(ui.value); + exportMultiplicationDivHidden.val(ui.value); + } + }); + + //setting the loop function + var myTimeout; + + function loopFunction() { + ; + if (self.changed) { + self.drawCoverage(); + //self.updateSlidingWindow(); + self.changed = false; + } + myTimeout = setTimeout(loopFunction, 2000); + } + + myTimeout = setTimeout(loopFunction, 2000); + + //map changed listener + map.events.register('moveend', this, function (event) { + + //update export div + self.exportMultiplicationFactor = self.getMultiplicationFactor(); + exportMultiplicationDivHidden.val(self.exportMultiplicationFactor); + exportMultiplicationDiv.html(self.exportMultiplicationFactor); + $("#multiplicationSlider").slider({value: self.getMultiplicationFactor()}); + + + //draw coverage + coverageUI.drawCoverage(); + //self.updateSlidingWindow(); + }); } - this.isVDMCategory = function() { + this.isVDMCategory = function () { return $('#coverageType').val() == "VDM"; } - this.drawSlider = function() { - $( "#slider-range" ).dragslider({ + this.drawSlider = function () { + $("#slider-range").dragslider({ range: true, min: self.isVDMCategory() ? 0 : -200, max: self.isVDMCategory() ? 100 : 0, rangeDrag: true, - values: [ self.minThreshold, self.maxThreshold ], - slide: function( event, ui ) { - self.minThreshold = ui.values[ 0 ]; - self.maxThreshold = ui.values[ 1 ]; - $( "#min-range" ).html( " < " + self.minThreshold + self.thresholdUnit + " <= " ); - $( "#max-range" ).html( " < " + self.maxThreshold + self.thresholdUnit + " <= " ); + values: [self.minThreshold, self.maxThreshold], + slide: function (event, ui) { + self.minThreshold = ui.values[0]; + self.maxThreshold = ui.values[1]; + $("#min-range").html(" < " + self.minThreshold + self.thresholdUnit + " <= "); + $("#max-range").html(" < " + self.maxThreshold + self.thresholdUnit + " <= "); }, - change: function(event, ui){ + change: function (event, ui) { self.changed = true; } }); - $( "#min-range" ).html( " < " + self.minThreshold + self.thresholdUnit + " <= " ); - $( "#max-range" ).html( " < " + self.maxThreshold + self.thresholdUnit + " <= " ); + $("#min-range").html(" < " + self.minThreshold + self.thresholdUnit + " <= "); + $("#max-range").html(" < " + self.maxThreshold + self.thresholdUnit + " <= "); } - this.updateShipTrack = function(){ - aisJsonClient.getShipTrack(selectedStartDate.getTime(), selectedEndDate.getTime(), $('#shiptrackingmmsi').val(), function(trackarray){ - shiptrackactive=true; - lineLayer.removeAllFeatures(); - var previousLat = null; - var previousLon = null; - var nextRed = false; - //For each timespan - $.each(trackarray, function(key, value) { - //For each lat-lon inside timespan - $.each(value.positions, function(key1, value1) { - if(previousLat != null){ - if(nextRed == true){ - self.drawLine(previousLat,previousLon,value1.lat,value1.lon, '#CC0000'); - nextRed=false; - }else{ - self.drawLine(previousLat,previousLon,value1.lat,value1.lon, '#006633'); - } - - } - previousLat=value1.lat; - previousLon=value1.lon; - }); - nextRed=true; - - }); - }); + this.updateShipTrack = function () { + aisJsonClient.getShipTrack(selectedStartDate.getTime(), selectedEndDate.getTime(), $('#shiptrackingmmsi').val(), function (trackarray) { + shiptrackactive = true; + lineLayer.removeAllFeatures(); + var previousLat = null; + var previousLon = null; + var nextRed = false; + //For each timespan + $.each(trackarray, function (key, value) { + //For each lat-lon inside timespan + $.each(value.positions, function (key1, value1) { + if (previousLat != null) { + if (nextRed == true) { + self.drawLine(previousLat, previousLon, value1.lat, value1.lon, '#CC0000'); + nextRed = false; + } else { + self.drawLine(previousLat, previousLon, value1.lat, value1.lon, '#006633'); + } + + } + previousLat = value1.lat; + previousLon = value1.lon; + }); + nextRed = true; + + }); + }); } - this.drawLine = function(lat1, lon1, lat2, lon2, color){ - var points = new Array( - new OpenLayers.Geometry.Point(lon1, lat1).transform(new OpenLayers.Projection("EPSG:4326"), map.getProjectionObject()), - new OpenLayers.Geometry.Point(lon2, lat2).transform(new OpenLayers.Projection("EPSG:4326"), map.getProjectionObject()) - ); - var line = new OpenLayers.Geometry.LineString(points); - - var style = { - strokeColor: color, + this.drawLine = function (lat1, lon1, lat2, lon2, color) { + var points = new Array( + new OpenLayers.Geometry.Point(lon1, lat1).transform(new OpenLayers.Projection("EPSG:4326"), map.getProjectionObject()), + new OpenLayers.Geometry.Point(lon2, lat2).transform(new OpenLayers.Projection("EPSG:4326"), map.getProjectionObject()) + ); + var line = new OpenLayers.Geometry.LineString(points); + + var style = { + strokeColor: color, // strokeOpacity: 1.0, - strokeWidth: 5 - }; + strokeWidth: 5 + }; - lineFeature = new OpenLayers.Feature.Vector(line, null, style); - lineLayer.addFeatures([lineFeature]); + lineFeature = new OpenLayers.Feature.Vector(line, null, style); + lineLayer.addFeatures([lineFeature]); } - this.updateSlidingWindow = function(){ - aisJsonClient.getStatus(function(data){ - var startDate = new Date(data.firstMessage); - startDate.setMinutes(0); - startDate.setSeconds(0); - startDate.setMilliseconds(0); - var endDate = new Date(data.lastMessage+(1000*60*60)); - endDate.setMinutes(0); - endDate.setSeconds(0); - endDate.setMilliseconds(0); - - self.setupSlidingWindow(startDate, endDate); + this.updateSlidingWindow = function () { + aisJsonClient.getStatus(function (data) { + var startDate = new Date(data.firstMessage); + startDate.setMinutes(0); + startDate.setSeconds(0); + startDate.setMilliseconds(0); + var endDate = new Date(data.lastMessage + (1000 * 60 * 60)); + endDate.setMinutes(0); + endDate.setSeconds(0); + endDate.setMilliseconds(0); + + self.setupSlidingWindow(startDate, endDate); // alert(startDate); // alert(endDate); // alert(data.firstMessage); - }); + }); } - - this.setupSlidingWindow = function(startDate, endDate){ - $('.slidingWindowLabel').hide(); - $( "#globalStarTime" ).html(self.formatDate(startDate)); - $( "#globalEndTime" ).html(self.formatDate(endDate)); - - - //Update export post form - $('#exportStartTime').val(startDate.getTime()); - $('#exportEndTime').val(endDate.getTime()); - - var timeDif = (endDate.getTime()-startDate.getTime())/1000/60/60; - var startTimeLabel = $( "#starTime" ); - var endTimeLabel = $( "#endTime" ); - var intervalLabel = $( "#interval" ); + + this.setupSlidingWindow = function (startDate, endDate) { + $('.slidingWindowLabel').hide(); + $("#globalStarTime").html(self.formatDate(startDate)); + $("#globalEndTime").html(self.formatDate(endDate)); + + + //Update export post form + $('#exportStartTime').val(startDate.getTime()); + $('#exportEndTime').val(endDate.getTime()); + + var timeDif = (endDate.getTime() - startDate.getTime()) / 1000 / 60 / 60; + var startTimeLabel = $("#starTime"); + var endTimeLabel = $("#endTime"); + var intervalLabel = $("#interval"); // var slidingWindowSlider = $( "#slidingWindowOuter" ).width(); - var leftOffset = 37; - var pixelInterval = $( "#slidingWindowOuter" ).width()/timeDif; - $( "#globalStarTime" ).css("left", leftOffset); - $( "#globalEndTime" ).css("left", leftOffset+(pixelInterval*timeDif)); - - - - var updateLabels = function(firstValue, lastValue){ - selectedStartDate = new Date(startDate.getTime()+firstValue*1000*60*60); - selectedEndDate = new Date(startDate.getTime()+lastValue*1000*60*60); - startTimeLabel.html(self.formatDate(selectedStartDate)); - startTimeLabel.css("left", (firstValue*pixelInterval)+leftOffset); - endTimeLabel.html(self.formatDate(selectedEndDate)); - endTimeLabel.css("left", (lastValue*pixelInterval)+leftOffset); - - var intervalSize = lastValue-firstValue; - if(intervalSize > 1){ - intervalLabel.html(intervalSize +" hours"); - }else{ - intervalLabel.html(intervalSize +" hour"); - } - intervalLabel.css("left", ((firstValue+(intervalSize/2))*pixelInterval)-25); - - - } - if (typeof defaultFirstValue === "undefined") { - defaultFirstValue = timeDif-6; - defaultSecondValue = timeDif; - if(timeDif < 6){ defaultFirstValue = 0; } - - }else{ - //console.log(selectedStartDate); - } - updateLabels(defaultFirstValue,defaultSecondValue); - - - $( "#slidingWindow" ).dragslider({ - range: true, - min: 0, - max: timeDif, - rangeDrag: true, - values: [ defaultFirstValue, defaultSecondValue ], - slide: function(event, ui){ - if(ui.values[1] - ui.values[0] < 1){ + var leftOffset = 37; + var pixelInterval = $("#slidingWindowOuter").width() / timeDif; + $("#globalStarTime").css("left", leftOffset); + $("#globalEndTime").css("left", leftOffset + (pixelInterval * timeDif)); + + + var updateLabels = function (firstValue, lastValue) { + selectedStartDate = new Date(startDate.getTime() + firstValue * 1000 * 60 * 60); + selectedEndDate = new Date(startDate.getTime() + lastValue * 1000 * 60 * 60); + startTimeLabel.html(self.formatDate(selectedStartDate)); + startTimeLabel.css("left", (firstValue * pixelInterval) + leftOffset); + endTimeLabel.html(self.formatDate(selectedEndDate)); + endTimeLabel.css("left", (lastValue * pixelInterval) + leftOffset); + + var intervalSize = lastValue - firstValue; + if (intervalSize > 1) { + intervalLabel.html(intervalSize + " hours"); + } else { + intervalLabel.html(intervalSize + " hour"); + } + intervalLabel.css("left", ((firstValue + (intervalSize / 2)) * pixelInterval) - 25); + + + } + if (typeof defaultFirstValue === "undefined") { + defaultFirstValue = timeDif - 6; + defaultSecondValue = timeDif; + if (timeDif < 6) { + defaultFirstValue = 0; + } + + } else { + //console.log(selectedStartDate); + } + updateLabels(defaultFirstValue, defaultSecondValue); + + + $("#slidingWindow").dragslider({ + range: true, + min: 0, + max: timeDif, + rangeDrag: true, + values: [defaultFirstValue, defaultSecondValue], + slide: function (event, ui) { + if (ui.values[1] - ui.values[0] < 1) { // do not allow change return false; } - self.canUpdate = true; - updateLabels(ui.values[ 0 ],ui.values[ 1 ]); - }, - change: function(event, ui){ - - //Update export post form - $('#exportStartTime').val(selectedStartDate.getTime()); - $('#exportEndTime').val(selectedEndDate.getTime()); - - //Update sat bar chart if it is visible - if ($('#barchartpanel').css('display') != 'none') { - self.updateBarChart(); - }else if($('#trackingBarchartWindow').css('display') != 'none'){ - self.updateShipTrackBarChart(); - } - if(shiptrackactive==true){ - self.updateShipTrack(); - } - if(self.canUpdate){ - self.changed = true; - self.canUpdate = false; - } - - } - }); - - showLabels = false; - $('#slidingWindowPanel').mouseover(function() { - if(showLabels==false){ - $('.slidingWindowLabel').fadeIn(100); - } - showLabels=true; - - }); - $('#slidingWindowPanel').mouseout(function() { - showLabels = false; - - //wait a bit and hide if showlabels is still true. - setTimeout(function(){ - if(showLabels==false){ - $('.slidingWindowLabel').fadeOut(100); - } - },1000); - }); - - console.log("sliding window updated"); + self.canUpdate = true; + updateLabels(ui.values[0], ui.values[1]); + }, + change: function (event, ui) { + + //Update export post form + $('#exportStartTime').val(selectedStartDate.getTime()); + $('#exportEndTime').val(selectedEndDate.getTime()); + + //Update sat bar chart if it is visible + if ($('#barchartpanel').css('display') != 'none') { + self.updateBarChart(); + } else if ($('#trackingBarchartWindow').css('display') != 'none') { + self.updateShipTrackBarChart(); + } + if (shiptrackactive == true) { + self.updateShipTrack(); + } + if (self.canUpdate) { + self.changed = true; + self.canUpdate = false; + } + + } + }); + + showLabels = false; + $('#slidingWindowPanel').mouseover(function () { + if (showLabels == false) { + $('.slidingWindowLabel').fadeIn(100); + } + showLabels = true; + + }); + $('#slidingWindowPanel').mouseout(function () { + showLabels = false; + + //wait a bit and hide if showlabels is still true. + setTimeout(function () { + if (showLabels == false) { + $('.slidingWindowLabel').fadeOut(100); + } + }, 1000); + }); + + console.log("sliding window updated"); } - this.formatDate = function(date){ - return date.toISOString(); + this.formatDate = function (date) { + return date.toISOString(); } - + /** * Adds all the layers that will contain graphic. */ - this.addLayers = function(){ - - // Get renderer - var renderer = OpenLayers.Util.getParameters(window.location.href).renderer; - renderer = (renderer) ? [renderer] : OpenLayers.Layer.Vector.prototype.renderers; - // renderer = ["Canvas", "SVG", "VML"]; - - // Add OpenStreetMap Layer - var osm = new OpenLayers.Layer.OSM( - "OSM", - "http://a.tile.openstreetmap.org/${z}/${x}/${y}.png", - { - 'layers':'basic', - 'isBaseLayer': true - } - ); - map.addLayer(osm); - - //polygon layer - polygonLayer = new OpenLayers.Layer.Vector("Polygon Layer",{ - styleMap: new OpenLayers.StyleMap({ - "default": new OpenLayers.Style({ - strokeColor: "black", - strokeOpacity: .7, - strokeWidth: 1, - fillColor: "${fillcolor}", - fillOpacity: .6, - cursor: "pointer" - }), - "select": new OpenLayers.Style({ - strokeColor: "blue", - strokeOpacity: .7, - strokeWidth: 4, - }) - }) - }); + this.addLayers = function () { + + // Get renderer + var renderer = OpenLayers.Util.getParameters(window.location.href).renderer; + renderer = (renderer) ? [renderer] : OpenLayers.Layer.Vector.prototype.renderers; + // renderer = ["Canvas", "SVG", "VML"]; + + // Add OpenStreetMap Layer + var osm = new OpenLayers.Layer.OSM( + "OSM", + "http://a.tile.openstreetmap.org/${z}/${x}/${y}.png", + { + 'layers': 'basic', + 'isBaseLayer': true + } + ); + map.addLayer(osm); + + //polygon layer + polygonLayer = new OpenLayers.Layer.Vector("Polygon Layer", { + styleMap: new OpenLayers.StyleMap({ + "default": new OpenLayers.Style({ + strokeColor: "black", + strokeOpacity: .7, + strokeWidth: 1, + fillColor: "${fillcolor}", + fillOpacity: .6, + cursor: "pointer" + }), + "select": new OpenLayers.Style({ + strokeColor: "blue", + strokeOpacity: .7, + strokeWidth: 4, + }) + }) + }); // polygonLayer.styleMap.styles['default'].defaultStyle={fillColor: 'blue'} - map.addLayers([polygonLayer]); - - //source layer - sourceLayer = new OpenLayers.Layer.Vector("Sources", { - styleMap: new OpenLayers.StyleMap({ - "default": new OpenLayers.Style({ - strokeColor: "black", - strokeOpacity: .7, - strokeWidth: 1, - fillColor: "${fillcolor}", - fillOpacity: .6, - cursor: "pointer" - }), - "select": new OpenLayers.Style({ - strokeColor: "blue", - strokeOpacity: .7, - strokeWidth: 4, - }) - }) - }); - map.addLayer(sourceLayer); - - //select control for source layer - selectControl = new OpenLayers.Control.SelectFeature([sourceLayer, polygonLayer], - { - clickout: true, - hover: false, - highlightOnly: false, - eventListeners: { - featurehighlighted: self.onFeatureSelect, - featureunhighlighted: self.onFeatureUnselect - } - - } - ); - selectControl.handlers.feature.stopDown = false; - map.addControl(selectControl); - selectControl.activate(); - - boxLayer = new OpenLayers.Layer.Vector("Box layer"); - map.addLayers([boxLayer]); + map.addLayers([polygonLayer]); + + //source layer + sourceLayer = new OpenLayers.Layer.Vector("Sources", { + styleMap: new OpenLayers.StyleMap({ + "default": new OpenLayers.Style({ + strokeColor: "black", + strokeOpacity: .7, + strokeWidth: 1, + fillColor: "${fillcolor}", + fillOpacity: .6, + cursor: "pointer" + }), + "select": new OpenLayers.Style({ + strokeColor: "blue", + strokeOpacity: .7, + strokeWidth: 4, + }) + }) + }); + map.addLayer(sourceLayer); + + //select control for source layer + selectControl = new OpenLayers.Control.SelectFeature([sourceLayer, polygonLayer], + { + clickout: true, + hover: false, + highlightOnly: false, + eventListeners: { + featurehighlighted: self.onFeatureSelect, + featureunhighlighted: self.onFeatureUnselect + } + + } + ); + selectControl.handlers.feature.stopDown = false; + map.addControl(selectControl); + selectControl.activate(); + + boxLayer = new OpenLayers.Layer.Vector("Box layer"); + map.addLayers([boxLayer]); // var currentBox = null; - var draw = new OpenLayers.Control.DrawFeature( - boxLayer, - OpenLayers.Handler.RegularPolygon, - { - featureAdded : function(feature) { - if(boxLayer.features.length > 1){ - boxLayer.removeFeatures([boxLayer.features[0]]); + var draw = new OpenLayers.Control.DrawFeature( + boxLayer, + OpenLayers.Handler.RegularPolygon, + { + featureAdded: function (feature) { + if (boxLayer.features.length > 1) { + boxLayer.removeFeatures([boxLayer.features[0]]); //// console.log("remove"); - } + } // currentBox = feature; - var g=boxLayer.features[0].clone().geometry; //get geometry of a featyre in your vector layer - var vertices = g.getVertices(); - - topleftPoint = vertices[1].transform( map.getProjectionObject(), - new OpenLayers.Projection("EPSG:4326")); - bottomRightPoint = vertices[3].transform( map.getProjectionObject(), - new OpenLayers.Projection("EPSG:4326")); - - self.updateBarChart(); - $('#barchartpanel').fadeIn(100); + var g = boxLayer.features[0].clone().geometry; //get geometry of a featyre in your vector layer + var vertices = g.getVertices(); + + topleftPoint = vertices[1].transform(map.getProjectionObject(), + new OpenLayers.Projection("EPSG:4326")); + bottomRightPoint = vertices[3].transform(map.getProjectionObject(), + new OpenLayers.Projection("EPSG:4326")); + + self.updateBarChart(); + $('#barchartpanel').fadeIn(100); // console.log("A point has been added"); - }, - handlerOptions: { - sides: 4, - irregular: true - } - },function(){alert('added')} - ); - map.addControl(draw); - - var isDown = false; - var box; - $(document).keydown(function (e) { - if(e.which == 17){ - if(isDown != true){ - isDown = true; - draw.activate(); + }, + handlerOptions: { + sides: 4, + irregular: true + } + }, function () { + alert('added') + } + ); + map.addControl(draw); + + var isDown = false; + var box; + $(document).keydown(function (e) { + if (e.which == 17) { + if (isDown != true) { + isDown = true; + draw.activate(); // $('.expandable').fadeOut(); - } - isDown = true; - - } - }); - $(document).keyup(function (e) { - if(e.which == 17){ - isDown = false; - draw.deactivate(); + } + isDown = true; + + } + }); + $(document).keyup(function (e) { + if (e.which == 17) { + isDown = false; + draw.deactivate(); // $('.expandable').fadeIn(); - } - }); - - lineLayer = new OpenLayers.Layer.Vector("Line Layer"); - map.addLayers([lineLayer]); + } + }); + + lineLayer = new OpenLayers.Layer.Vector("Line Layer"); + map.addLayers([lineLayer]); } - this.updateBarChart = function(){ - $('#trackingBarchartWindow').hide(); - $('#barchart').attr('src', 'rest/satExportPNG?random='+Math.random()+'&startTime='+selectedStartDate.getTime()+'&endTime='+selectedEndDate.getTime()+'&lat1='+topleftPoint.y+'&lon1='+topleftPoint.x+'&lat2='+bottomRightPoint.y+'&lon2='+bottomRightPoint.x+'&satChartMethod='+satChartMethod); + this.updateBarChart = function () { + $('#trackingBarchartWindow').hide(); + $('#barchart').attr('src', 'rest/satExportPNG?random=' + Math.random() + '&startTime=' + selectedStartDate.getTime() + '&endTime=' + selectedEndDate.getTime() + '&lat1=' + topleftPoint.y + '&lon1=' + topleftPoint.x + '&lat2=' + bottomRightPoint.y + '&lon2=' + bottomRightPoint.x + '&satChartMethod=' + satChartMethod); } - this.updateShipTrackBarChart = function(){ - $('#barchartpanel').hide(); - $('#trackingBarchartImg').attr('src', 'rest/shipTrackExportPNG?random='+Math.random()+'&startTime='+selectedStartDate.getTime()+'&endTime='+selectedEndDate.getTime()+'&shipmmsi='+$('#shiptrackingmmsi').val()); + this.updateShipTrackBarChart = function () { + $('#barchartpanel').hide(); + $('#trackingBarchartImg').attr('src', 'rest/shipTrackExportPNG?random=' + Math.random() + '&startTime=' + selectedStartDate.getTime() + '&endTime=' + selectedEndDate.getTime() + '&shipmmsi=' + $('#shiptrackingmmsi').val()); } - this.refreshSourceList = function(){ - var sourceContainer = $("#sourcesPanel > .panelContainer"); - sourceContainer.html(""); - var sourceshtml = ""; - $.each(self.sources, function(key, source) { - var checked = ""; - if(source.enabled){ - checked = 'checked="checked"'; - } - sourceshtml += '
'+source.name+'
'+source.type+'
'; - }); - sourceContainer.html(sourceshtml); - self.drawSources(); + this.refreshSourceList = function () { + var sourceContainer = $("#sourcesPanel > .panelContainer"); + sourceContainer.html(""); + var sourceshtml = ""; + $.each(self.sources, function (key, source) { + var checked = ""; + if (source.enabled) { + checked = 'checked="checked"'; + } + sourceshtml += '
' + source.name + '
' + source.type + '
'; + }); + sourceContainer.html(sourceshtml); + self.drawSources(); } - - this.refreshSourceDetails = function(){ - - if(self.selectedSource == null){ - return; - } - var checked=""; - if(self.selectedSource.enabled){ - checked='checked="checked"'; + + this.refreshSourceDetails = function () { + + if (self.selectedSource == null) { + return; + } + var checked = ""; + if (self.selectedSource.enabled) { + checked = 'checked="checked"'; } - $("#featureDetailsPanel > .panelContainer").html('
Id
'+ - '
'+feature.mmsi+'
'+ - '
Name
'+ - '
'+feature.name+'
'+ - '
Type
'+ - '
'+feature.type+'
'+ - '
Lat
'+ - '
'+feature.lat.toFixed(4)+'
'+ - '
Lon
'+ - '
'+feature.lon.toFixed(4)+'
'+ - '
Enabled
'+ - '
'); + $("#featureDetailsPanel > .panelContainer").html('
Id
' + + '
' + feature.mmsi + '
' + + '
Name
' + + '
' + feature.name + '
' + + '
Type
' + + '
' + feature.type + '
' + + '
Lat
' + + '
' + feature.lat.toFixed(4) + '
' + + '
Lon
' + + '
' + feature.lon.toFixed(4) + '
' + + '
Enabled
' + + '
'); } - - this.drawSources = function(){ - sourceLayer.removeAllFeatures(); - var drawTheSource = this.drawSource - $.each(this.sources, function(key, val) { - drawTheSource(val); - }); + + this.drawSources = function () { + sourceLayer.removeAllFeatures(); + var drawTheSource = this.drawSource + $.each(this.sources, function (key, val) { + drawTheSource(val); + }); } - - this.drawSource = function(val){ - var image; - if(val.enabled){ - image = 'img/marker.png'; - }else{ - image = 'img/marker2.png' - } - - var feature = new OpenLayers.Feature.Vector( - new OpenLayers.Geometry.Point( val.lon , val.lat ).transform( - new OpenLayers.Projection("EPSG:4326"), // transform from WGS 1984 - map.getProjectionObject() // to Spherical Mercator Projection - ), - {hmm:'100'}); + + this.drawSource = function (val) { + var image; + if (val.enabled) { + image = 'img/marker.png'; + } else { + image = 'img/marker2.png' + } + + var feature = new OpenLayers.Feature.Vector( + new OpenLayers.Geometry.Point(val.lon, val.lat).transform( + new OpenLayers.Projection("EPSG:4326"), // transform from WGS 1984 + map.getProjectionObject() // to Spherical Mercator Projection + ), + {hmm: '100'}); // {externalGraphic: image, graphicHeight: 21, graphicWidth: 16, cursor: "crosshair", fillColor: "#ffcc66", pointRadius: "10"}); - feature.mmsi = val.mmsi; - feature.name = val.name; - feature.type = val.type; - feature.lat = val.lat; - feature.lon = val.lon; - feature.source = val; - val.feature=feature; - - if(val.enabled){ - feature.style = { - pointRadius: "10", // sized according to type attribute - fillColor: "#980000 ", - strokeColor: "black", - strokeWidth: 2, - graphicZIndex: 1, - cursor: "pointer" - } - }else{ - feature.style = { - pointRadius: "10", // sized according to type attribute - fillColor: "white", - strokeColor: "black", - strokeWidth: 2, - graphicZIndex: 1, - cursor: "pointer" - } - } - if(self.selectedSource == val){ - feature.style["strokeColor"] = "blue"; - }else{ - feature.style["strokeColor"] = "black"; - } - - sourceLayer.addFeatures(feature); + feature.mmsi = val.mmsi; + feature.name = val.name; + feature.type = val.type; + feature.lat = val.lat; + feature.lon = val.lon; + feature.source = val; + val.feature = feature; + + if (val.enabled) { + feature.style = { + pointRadius: "10", // sized according to type attribute + fillColor: "#980000 ", + strokeColor: "black", + strokeWidth: 2, + graphicZIndex: 1, + cursor: "pointer" + } + } else { + feature.style = { + pointRadius: "10", // sized according to type attribute + fillColor: "white", + strokeColor: "black", + strokeWidth: 2, + graphicZIndex: 1, + cursor: "pointer" + } + } + if (self.selectedSource == val) { + feature.style["strokeColor"] = "blue"; + } else { + feature.style["strokeColor"] = "black"; + } + + sourceLayer.addFeatures(feature); } - + // this.loadSatStats = function(topleft, bottomright){ // //// var array = [{"fromTime":1374220785115,"toTime":1374220774501,"spanLength":1,"timeSinceLastSpan":0,"accumulatedTime":0,"signals":3,"distinctShips":1},{"fromTime":1374226711355,"toTime":1374227266676,"spanLength":9,"timeSinceLastSpan":98,"accumulatedTime":108,"signals":163,"distinctShips":7},{"fromTime":1374228081000,"toTime":1374228352000,"spanLength":4,"timeSinceLastSpan":13,"accumulatedTime":126,"signals":38,"distinctShips":6},{"fromTime":1374232884293,"toTime":1374233250297,"spanLength":6,"timeSinceLastSpan":75,"accumulatedTime":207,"signals":71,"distinctShips":6},{"fromTime":1374233909000,"toTime":1374234299000,"spanLength":6,"timeSinceLastSpan":10,"accumulatedTime":225,"signals":33,"distinctShips":7},{"fromTime":1374238552052,"toTime":1374239226409,"spanLength":11,"timeSinceLastSpan":70,"accumulatedTime":307,"signals":46,"distinctShips":6},{"fromTime":1374239872000,"toTime":1374239943000,"spanLength":1,"timeSinceLastSpan":10,"accumulatedTime":319,"signals":8,"distinctShips":5},{"fromTime":1374243070626,"toTime":1374243403490,"spanLength":5,"timeSinceLastSpan":52,"accumulatedTime":377,"signals":42,"distinctShips":6},{"fromTime":1374244812816,"toTime":1374245655000,"spanLength":14,"timeSinceLastSpan":23,"accumulatedTime":414,"signals":105,"distinctShips":6},{"fromTime":1374248972125,"toTime":1374249122001,"spanLength":2,"timeSinceLastSpan":55,"accumulatedTime":472,"signals":58,"distinctShips":7},{"fromTime":1374250432644,"toTime":1374251214883,"spanLength":13,"timeSinceLastSpan":21,"accumulatedTime":507,"signals":120,"distinctShips":7},{"fromTime":1374251846000,"toTime":1374251977000,"spanLength":2,"timeSinceLastSpan":10,"accumulatedTime":520,"signals":5,"distinctShips":4},{"fromTime":1374254798441,"toTime":1374255211329,"spanLength":6,"timeSinceLastSpan":47,"accumulatedTime":573,"signals":33,"distinctShips":6},{"fromTime":1374257362000,"toTime":1374257820000,"spanLength":7,"timeSinceLastSpan":35,"accumulatedTime":617,"signals":16,"distinctShips":7},{"fromTime":1374261058127,"toTime":1374261077807,"spanLength":1,"timeSinceLastSpan":53,"accumulatedTime":671,"signals":2,"distinctShips":1},{"fromTime":1374263221000,"toTime":1374263407000,"spanLength":3,"timeSinceLastSpan":35,"accumulatedTime":710,"signals":16,"distinctShips":5},{"fromTime":1374269013000,"toTime":1374269185000,"spanLength":2,"timeSinceLastSpan":93,"accumulatedTime":806,"signals":6,"distinctShips":2},{"fromTime":1374274821000,"toTime":1374274933000,"spanLength":1,"timeSinceLastSpan":93,"accumulatedTime":902,"signals":4,"distinctShips":4},{"fromTime":1374278293629,"toTime":1374278393470,"spanLength":1,"timeSinceLastSpan":56,"accumulatedTime":960,"signals":4,"distinctShips":3},{"fromTime":1374220785115,"toTime":1374220774501,"spanLength":1,"timeSinceLastSpan":33,"accumulatedTime":0,"signals":3,"distinctShips":1},{"fromTime":1374226711355,"toTime":1374227266676,"spanLength":9,"timeSinceLastSpan":98,"accumulatedTime":108,"signals":163,"distinctShips":7},{"fromTime":1374228081000,"toTime":1374228352000,"spanLength":4,"timeSinceLastSpan":13,"accumulatedTime":126,"signals":38,"distinctShips":6},{"fromTime":1374232884293,"toTime":1374233250297,"spanLength":6,"timeSinceLastSpan":75,"accumulatedTime":207,"signals":71,"distinctShips":6},{"fromTime":1374233909000,"toTime":1374234299000,"spanLength":6,"timeSinceLastSpan":10,"accumulatedTime":225,"signals":33,"distinctShips":7},{"fromTime":1374238552052,"toTime":1374239226409,"spanLength":11,"timeSinceLastSpan":70,"accumulatedTime":307,"signals":46,"distinctShips":6},{"fromTime":1374239872000,"toTime":1374239943000,"spanLength":1,"timeSinceLastSpan":10,"accumulatedTime":319,"signals":8,"distinctShips":5},{"fromTime":1374243070626,"toTime":1374243403490,"spanLength":5,"timeSinceLastSpan":52,"accumulatedTime":377,"signals":42,"distinctShips":6},{"fromTime":1374244812816,"toTime":1374245655000,"spanLength":14,"timeSinceLastSpan":23,"accumulatedTime":414,"signals":105,"distinctShips":6},{"fromTime":1374248972125,"toTime":1374249122001,"spanLength":2,"timeSinceLastSpan":55,"accumulatedTime":472,"signals":58,"distinctShips":7},{"fromTime":1374250432644,"toTime":1374251214883,"spanLength":13,"timeSinceLastSpan":21,"accumulatedTime":507,"signals":120,"distinctShips":7},{"fromTime":1374251846000,"toTime":1374251977000,"spanLength":2,"timeSinceLastSpan":10,"accumulatedTime":520,"signals":5,"distinctShips":4},{"fromTime":1374254798441,"toTime":1374255211329,"spanLength":6,"timeSinceLastSpan":47,"accumulatedTime":573,"signals":33,"distinctShips":6},{"fromTime":1374257362000,"toTime":1374257820000,"spanLength":7,"timeSinceLastSpan":35,"accumulatedTime":617,"signals":16,"distinctShips":7},{"fromTime":1374261058127,"toTime":1374261077807,"spanLength":1,"timeSinceLastSpan":53,"accumulatedTime":671,"signals":2,"distinctShips":1},{"fromTime":1374263221000,"toTime":1374263407000,"spanLength":3,"timeSinceLastSpan":35,"accumulatedTime":710,"signals":16,"distinctShips":5},{"fromTime":1374269013000,"toTime":1374269185000,"spanLength":2,"timeSinceLastSpan":93,"accumulatedTime":806,"signals":6,"distinctShips":2},{"fromTime":1374274821000,"toTime":1374274933000,"spanLength":1,"timeSinceLastSpan":93,"accumulatedTime":902,"signals":4,"distinctShips":4},{"fromTime":1374278293629,"toTime":1374379393470,"spanLength":1,"timeSinceLastSpan":56,"accumulatedTime":960,"signals":4,"distinctShips":3}]; @@ -741,223 +746,222 @@ function CoverageUI () { //// alert(topleft); // } - this.drawCoverage = function(){ - - //get the multiplication factor for corresponding zoom level - var multifactor = self.getMultiplicationFactor(); + this.drawCoverage = function () { + + //get the multiplication factor for corresponding zoom level + var multifactor = self.getMultiplicationFactor(); // multifactor = 5; - $('#multiplicationFactor').html(multifactor); - - // activate loading panel - $("#loadingPanel").css('visibility', 'visible'); - - // get lat lon points for each screen corner - var topleftpixel = new OpenLayers.Pixel(0, 0); - var bottomrightpixel = new OpenLayers.Pixel($("#map").width(), $("#map").height()); - var topleftlonlat = map.getLonLatFromPixel(topleftpixel).transform( - map.getProjectionObject(), // from Spherical Mercator Projection - new OpenLayers.Projection("EPSG:4326") // to WGS 1984 - ); - var bottomrightlonlat = map.getLonLatFromPixel(bottomrightpixel).transform( - map.getProjectionObject(), // from Spherical Mercator Projection - new OpenLayers.Projection("EPSG:4326") // to WGS 1984 - ); - var screenarea = topleftlonlat.lat.toFixed(4)+","+topleftlonlat.lon.toFixed(4)+","+bottomrightlonlat.lat.toFixed(4)+","+bottomrightlonlat.lon.toFixed(4); - - //convert enabled sources to string to be sent - var dataToBeSent = self.enabledSourcesToString(); - - - - //use json client to fetch data from web service - aisJsonClient.getCoverage(dataToBeSent, screenarea, multifactor,selectedStartDate.getTime(), selectedEndDate.getTime(), function(data){ - - //remove existing cells - polygonLayer.removeAllFeatures(); - - $('#latSize').html(data.latSize.toFixed(4) + " degrees"); - $('#lonSize').html(data.lonSize.toFixed(4) + " degrees"); - - - var minExpectedMessages = - $.each(data.cells, function(key, val) { - var points = [ - new OpenLayers.Geometry.Point(val.lon, val.lat), - new OpenLayers.Geometry.Point(val.lon, val.lat+data.latSize), - new OpenLayers.Geometry.Point(val.lon+data.lonSize, val.lat+data.latSize), - new OpenLayers.Geometry.Point(val.lon+data.lonSize, val.lat) - ]; - var totalMessages = 0; + $('#multiplicationFactor').html(multifactor); + + // activate loading panel + $("#loadingPanel").css('visibility', 'visible'); + + // get lat lon points for each screen corner + var topleftpixel = new OpenLayers.Pixel(0, 0); + var bottomrightpixel = new OpenLayers.Pixel($("#map").width(), $("#map").height()); + var topleftlonlat = map.getLonLatFromPixel(topleftpixel).transform( + map.getProjectionObject(), // from Spherical Mercator Projection + new OpenLayers.Projection("EPSG:4326") // to WGS 1984 + ); + var bottomrightlonlat = map.getLonLatFromPixel(bottomrightpixel).transform( + map.getProjectionObject(), // from Spherical Mercator Projection + new OpenLayers.Projection("EPSG:4326") // to WGS 1984 + ); + var screenarea = topleftlonlat.lat.toFixed(4) + "," + topleftlonlat.lon.toFixed(4) + "," + bottomrightlonlat.lat.toFixed(4) + "," + bottomrightlonlat.lon.toFixed(4); + + //convert enabled sources to string to be sent + var dataToBeSent = self.enabledSourcesToString(); + + + //use json client to fetch data from web service + aisJsonClient.getCoverage(dataToBeSent, screenarea, multifactor, selectedStartDate.getTime(), selectedEndDate.getTime(), function (data) { + + //remove existing cells + polygonLayer.removeAllFeatures(); + + $('#latSize').html(data.latSize.toFixed(4) + " degrees"); + $('#lonSize').html(data.lonSize.toFixed(4) + " degrees"); + + + var minExpectedMessages = + $.each(data.cells, function (key, val) { + var points = [ + new OpenLayers.Geometry.Point(val.lon, val.lat), + new OpenLayers.Geometry.Point(val.lon, val.lat + data.latSize), + new OpenLayers.Geometry.Point(val.lon + data.lonSize, val.lat + data.latSize), + new OpenLayers.Geometry.Point(val.lon + data.lonSize, val.lat) + ]; + var totalMessages = 0; var coverageValue = 0; - if(self.isVDMCategory()) { - totalMessages = (val.nrOfMisMes+val.nrOfRecMes); - coverageValue = val.nrOfRecMes/totalMessages; + if (self.isVDMCategory()) { + totalMessages = (val.nrOfMisMes + val.nrOfRecMes); + coverageValue = val.nrOfRecMes / totalMessages; } else { totalMessages = val.numberOfVsiMessages; coverageValue = val.averageSignalStrength; } computeThresholdValue = function (threshold) { - if (self.isVDMCategory()) { - return threshold / 100; - } else { - return threshold; - } - }; - - var color; - if(totalMessages >= self.minExpectedMessages){ - if(coverageValue >= computeThresholdValue(self.maxThreshold)){ - color ='green'; - }else if(coverageValue >= computeThresholdValue(self.minThreshold)){ - color ='yellow'; - }else{ - color ='red'; - } - - self.drawPolygon({ - lat: val.lat, - lon: val.lon, - points: points, - fillcolor: color, - totalMessages: totalMessages, - receivedMessages: val.nrOfRecMes, - averageSignalStrength: val.averageSignalStrength - }); - } - }); - + if (self.isVDMCategory()) { + return threshold / 100; + } else { + return threshold; + } + }; + + var color; + if (totalMessages >= self.minExpectedMessages) { + if (coverageValue >= computeThresholdValue(self.maxThreshold)) { + color = 'green'; + } else if (coverageValue >= computeThresholdValue(self.minThreshold)) { + color = 'yellow'; + } else { + color = 'red'; + } + + self.drawPolygon({ + lat: val.lat, + lon: val.lon, + points: points, + fillcolor: color, + totalMessages: totalMessages, + receivedMessages: val.nrOfRecMes, + averageSignalStrength: val.averageSignalStrength + }); + } + }); + // self.loading = false; - $("#loadingPanel").css('visibility', 'hidden'); - }); + $("#loadingPanel").css('visibility', 'hidden'); + }); } - - this.drawPolygon = function(options){ - var site_points = []; - for (i in options.points) { - options.points[i].transform( - new OpenLayers.Projection("EPSG:4326"), // transform from WGS 1984 - map.getProjectionObject() // to Spherical Mercator Projection - ); - site_points.push(options.points[i]); - } - var linear_ring = new OpenLayers.Geometry.LinearRing(site_points); - polygonFeature = new OpenLayers.Feature.Vector( - new OpenLayers.Geometry.Polygon([linear_ring]),null); - polygonFeature.attributes.fillcolor = options.fillcolor; - polygonFeature.type = "cell"; - polygonFeature.lon = options.lon; - polygonFeature.lat = options.lat; - polygonFeature.totalMessages = options.totalMessages; - polygonFeature.receivedMessages = options.receivedMessages; + + this.drawPolygon = function (options) { + var site_points = []; + for (i in options.points) { + options.points[i].transform( + new OpenLayers.Projection("EPSG:4326"), // transform from WGS 1984 + map.getProjectionObject() // to Spherical Mercator Projection + ); + site_points.push(options.points[i]); + } + var linear_ring = new OpenLayers.Geometry.LinearRing(site_points); + polygonFeature = new OpenLayers.Feature.Vector( + new OpenLayers.Geometry.Polygon([linear_ring]), null); + polygonFeature.attributes.fillcolor = options.fillcolor; + polygonFeature.type = "cell"; + polygonFeature.lon = options.lon; + polygonFeature.lat = options.lat; + polygonFeature.totalMessages = options.totalMessages; + polygonFeature.receivedMessages = options.receivedMessages; polygonFeature.averageSignalStrength = options.averageSignalStrength; - polygonLayer.addFeatures([polygonFeature]); - + polygonLayer.addFeatures([polygonFeature]); + } - - this.enabledSourcesToString = function(){ - var string=""; - $.each(this.sources, function(key, source) { - if(source.enabled){ - string += source.mmsi+","; - } - }); - return string; + + this.enabledSourcesToString = function () { + var string = ""; + $.each(this.sources, function (key, source) { + if (source.enabled) { + string += source.mmsi + ","; + } + }); + return string; } - - this.getMultiplicationFactor = function(){ - var zLevel = map.getZoom(); - var multifactor; - if(zLevel > 10){ - multifactor = 1; - }else if(zLevel == 10){ - multifactor = 1; - }else if(zLevel == 9){ - multifactor = 2; - }else if(zLevel == 8){ - multifactor = 3; - }else if(zLevel == 7){ - multifactor = 4; - }else if(zLevel == 6){ - multifactor = 8; - }else if(zLevel == 5){ - multifactor = 20; - }else if(zLevel == 4){ - multifactor = 40; - }else if(zLevel == 3){ - multifactor = 60; - }else if(zLevel == 2){ - multifactor = 60; - }else if(zLevel == 1){ - multifactor = 80; + + this.getMultiplicationFactor = function () { + var zLevel = map.getZoom(); + var multifactor; + if (zLevel > 10) { + multifactor = 1; + } else if (zLevel == 10) { + multifactor = 1; + } else if (zLevel == 9) { + multifactor = 2; + } else if (zLevel == 8) { + multifactor = 3; + } else if (zLevel == 7) { + multifactor = 4; + } else if (zLevel == 6) { + multifactor = 8; + } else if (zLevel == 5) { + multifactor = 20; + } else if (zLevel == 4) { + multifactor = 40; + } else if (zLevel == 3) { + multifactor = 60; + } else if (zLevel == 2) { + multifactor = 60; + } else if (zLevel == 1) { + multifactor = 80; } // alert(multifactor) - return multifactor; + return multifactor; } - - this.onFeatureSelect = function(evt) { - + + this.onFeatureSelect = function (evt) { + feature = evt.feature; - + //determine if feature is a cell or a source - if(feature.type == "cell"){ - - //remove potential source - self.selectedSource = null; - self.drawSources(); - - //Setting up cell details panel - $("#featureDetailsPanel > .panelHeader").html("Cell Details"); - $("#featureDetailsPanel > .panelContainer").html(self.drawCellDetails(feature)); - $("#featureDetailsPanel").slideDown(); - - - }else{ - self.selectedSource = self.sources[feature.mmsi]; + if (feature.type == "cell") { + + //remove potential source + self.selectedSource = null; + self.drawSources(); + + //Setting up cell details panel + $("#featureDetailsPanel > .panelHeader").html("Cell Details"); + $("#featureDetailsPanel > .panelContainer").html(self.drawCellDetails(feature)); + $("#featureDetailsPanel").slideDown(); + + + } else { + self.selectedSource = self.sources[feature.mmsi]; self.refreshSourceDetails(); $("#featureDetailsPanel").slideDown(); $("#featureDetailsPanel > .panelHeader").html("Source Details"); self.drawSources(); } - + } - this.drawCellDetails = function(feature) { + this.drawCellDetails = function (feature) { var cellDetail = - '
Source
'+ - '
'+feature.mmsi+'
'+ - '
Cell Latitude
'+ - '
'+feature.lat.toFixed(4)+'
'+ - '
Cell Longitude
'+ - '
'+feature.lon.toFixed(4)+'
'; - - if (self.isVDMCategory()) { - cellDetail = cellDetail.concat( - '
Received Messages
'+ - '
'+feature.receivedMessages+'
'+ - '
Expected Messages
'+ - '
'+feature.totalMessages+'
'+ - '
Coverage
'+ - '
'+((feature.receivedMessages/feature.totalMessages)*100).toFixed(2) + ' ' + self.thresholdUnit +'
'); - } else { - cellDetail = cellDetail.concat( - '
Number of Messages
'+ - '
'+feature.totalMessages+'
'+ - '
Average Signal Strength
'+ - '
'+feature.averageSignalStrength + ' ' + self.thresholdUnit + '
'); - } + '
Source
' + + '
' + feature.mmsi + '
' + + '
Cell Latitude
' + + '
' + feature.lat.toFixed(4) + '
' + + '
Cell Longitude
' + + '
' + feature.lon.toFixed(4) + '
'; + + if (self.isVDMCategory()) { + cellDetail = cellDetail.concat( + '
Received Messages
' + + '
' + feature.receivedMessages + '
' + + '
Expected Messages
' + + '
' + feature.totalMessages + '
' + + '
Coverage
' + + '
' + ((feature.receivedMessages / feature.totalMessages) * 100).toFixed(2) + ' ' + self.thresholdUnit + '
'); + } else { + cellDetail = cellDetail.concat( + '
Number of Messages
' + + '
' + feature.totalMessages + '
' + + '
Average Signal Strength
' + + '
' + feature.averageSignalStrength + ' ' + self.thresholdUnit + '
'); + } - return cellDetail; + return cellDetail; } - this.cleanupCellDetails = function() { + this.cleanupCellDetails = function () { $("#featureDetailsPanel > .panelHeader").html("Cell Details"); $("#featureDetailsPanel > .panelContainer").html(""); $("#featureDetailsPanel").slideUp(); } - - this.onFeatureUnselect = function(evt) { + + this.onFeatureUnselect = function (evt) { // feature = evt.feature; // @@ -970,7 +974,7 @@ function CoverageUI () { //// $("#featureDetailsPanel").slideUp(); // self.drawSources(); //// } - + } - + } From 696806e72eeec750f8a68b3a83dad26cd83ff6d9 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 26 Mar 2017 14:28:15 -0400 Subject: [PATCH 35/59] Add signal strength data to exports --- .../ais/coverage/export/ExportDataType.java | 28 +++++++ .../export/generators/CSVGenerator.java | 44 +++++------ .../export/generators/KMLGenerator.java | 78 +++++++++---------- .../export/generators/XMLGenerator.java | 44 ++++------- .../coverage/rest/CoverageRestService.java | 61 +++++++++------ .../coverage/export/ExportDataTypeTest.java | 34 ++++++++ web/index.html | 6 +- web/js/CoverageUI.js | 9 +++ 8 files changed, 185 insertions(+), 119 deletions(-) create mode 100644 src/main/java/dk/dma/ais/coverage/export/ExportDataType.java create mode 100644 src/test/java/dk/dma/ais/coverage/export/ExportDataTypeTest.java diff --git a/src/main/java/dk/dma/ais/coverage/export/ExportDataType.java b/src/main/java/dk/dma/ais/coverage/export/ExportDataType.java new file mode 100644 index 0000000..7166e16 --- /dev/null +++ b/src/main/java/dk/dma/ais/coverage/export/ExportDataType.java @@ -0,0 +1,28 @@ +package dk.dma.ais.coverage.export; + +/** + * Supported data types for export. + */ +public enum ExportDataType { + RECEIVED_MESSAGES(0.8, 0.5), SIGNAL_STRENGTH(-75, -110); + + private final double greenThreshold; + private final double redThreshold; + + ExportDataType(double greenThreshold, double redThreshold) { + this.greenThreshold = greenThreshold; + this.redThreshold = redThreshold; + } + + public static ExportDataType forType(String type) { + return ExportDataType.valueOf(type.toUpperCase()); + } + + public double greenThreshold() { + return greenThreshold; + } + + public double redThreshold() { + return redThreshold; + } +} diff --git a/src/main/java/dk/dma/ais/coverage/export/generators/CSVGenerator.java b/src/main/java/dk/dma/ais/coverage/export/generators/CSVGenerator.java index 5e6449b..01e31ad 100644 --- a/src/main/java/dk/dma/ais/coverage/export/generators/CSVGenerator.java +++ b/src/main/java/dk/dma/ais/coverage/export/generators/CSVGenerator.java @@ -14,20 +14,18 @@ */ package dk.dma.ais.coverage.export.generators; +import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.data.Source; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Collection; import java.util.Date; -import javax.servlet.http.HttpServletResponse; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import dk.dma.ais.coverage.data.Cell; -import dk.dma.ais.coverage.data.Source; - public class CSVGenerator { private static final Logger LOG = LoggerFactory.getLogger(KMLGenerator.class); @@ -35,7 +33,7 @@ public class CSVGenerator { public static void generateCSV(Collection grids, double latSize, double lonSize, int multiplicity, HttpServletResponse response) { - LOG.info("startet csv generation"); + LOG.info("Started CSV generation"); HttpServletResponse out = response; @@ -61,7 +59,7 @@ public static void generateCSV(Collection grids, double latSize, double LOG.error(e.getMessage()); e.printStackTrace(); } - LOG.info("Finished csv generation"); + LOG.info("Finished CSV generation"); } private static void writeLine(String line, HttpServletResponse out) { @@ -74,21 +72,19 @@ private static void writeLine(String line, HttpServletResponse out) { } private static void generateGrid(String bsMmsi, Collection cells, HttpServletResponse out, double latSize, double lonSize) { - for (Cell cell : cells) { - - // We ignore cells, where average number of messages, is below 10 per ship - // Maybe there is a bug in AISMessage system, that assign some messages to wrong Base Stations - // Bug found and fixed - // if (cell.NOofReceivedSignals / cell.ships.size() > 10) { - - writeLine( - cell.getLatitude() + "," + cell.getLongitude() + "," + (cell.getLatitude() + latSize) + "," - + (cell.getLongitude() + lonSize) + "," + cell.getNOofReceivedSignals() + "," - + cell.getNOofMissingSignals() + "," + (cell.getCoverage() * 100), out); - - // } - + StringBuilder lineBuilder = new StringBuilder(); + lineBuilder.append(cell.getLatitude()).append(",") + .append(cell.getLongitude()).append(",") + .append(cell.getLatitude() + latSize).append(",") + .append(cell.getLongitude() + lonSize).append(",") + .append(cell.getNOofReceivedSignals()).append(",") + .append(cell.getNOofMissingSignals()).append(",") + .append(cell.getCoverage() * 100).append(",") + .append(cell.getNumberOfVsiMessages()).append(",") + .append(cell.getAverageSignalStrength()); + + writeLine(lineBuilder.toString(), out); } } diff --git a/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java b/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java index 53b91f6..9e1d5d3 100644 --- a/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java +++ b/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java @@ -14,20 +14,19 @@ */ package dk.dma.ais.coverage.export.generators; +import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.data.Source; +import dk.dma.ais.coverage.export.ExportDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Collection; import java.util.Date; -import javax.servlet.http.HttpServletResponse; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import dk.dma.ais.coverage.data.Cell; -import dk.dma.ais.coverage.data.Source; - //TODO retrieve sources with larger cells. //TODO retrieve cell data from both super and individual source @@ -37,9 +36,9 @@ public class KMLGenerator { // public static void generateKML(CoverageCalculator calc, String path) { public static void generateKML(Collection grids, double latSize, double lonSize, int multiplicity, - HttpServletResponse response) { + ExportDataType exportDataType, HttpServletResponse response) { - LOG.info("startet kml generation"); + LOG.info("Started KML generation"); HttpServletResponse out = response; @@ -103,7 +102,7 @@ public static void generateKML(Collection grids, double latSize, double writeLine("", out); for (Source grid : grids) { - generateGrid(grid.getIdentifier(), grid.getGrid().values(), out, latSize * multiplicity, lonSize * multiplicity); + generateGrid(grid.getIdentifier(), grid.getGrid().values(), out, latSize * multiplicity, lonSize * multiplicity, exportDataType); } writeLine("", out); @@ -116,7 +115,7 @@ public static void generateKML(Collection grids, double latSize, double LOG.error(e.getMessage()); e.printStackTrace(); } - LOG.info("Finished kml generation"); + LOG.info("Finished KML generation"); } private static void writeLine(String line, HttpServletResponse out) { @@ -128,45 +127,41 @@ private static void writeLine(String line, HttpServletResponse out) { } } - private static void generateGrid(String bsMmsi, Collection cells, HttpServletResponse out, double latSize, double lonSize) { - + private static void generateGrid(String bsMmsi, Collection cells, HttpServletResponse out, double latSize, double lonSize, ExportDataType exportDataType) { writeLine("", out); - writeLine("" + bsMmsi + "", out); - writeLine("0", out); + writeLine(" " + bsMmsi + "", out); + writeLine(" 0", out); for (Cell cell : cells) { + double dataToExport; + if (exportDataType == ExportDataType.RECEIVED_MESSAGES) { + dataToExport = cell.getCoverage(); + } else { + dataToExport = cell.getAverageSignalStrength(); + } - // We ignore cells, where average number of messages, is below 10 per ship - // Maybe there is a bug in AISMessage system, that assign some messages to wrong Base Stations - // Bug found and fixed - // if (cell.NOofReceivedSignals / cell.ships.size() > 10) { - - if (cell.getCoverage() > 0.8) { // green + if (dataToExport > exportDataType.greenThreshold()) { // green generatePlacemark("#greenStyle", cell, 300, out, latSize, lonSize); - } else if (cell.getCoverage() > 0.5) { // orange + } else if (dataToExport > exportDataType.redThreshold()) { // orange generatePlacemark("#orangeStyle", cell, 200, out, latSize, lonSize); } else { // red generatePlacemark("#redStyle", cell, 100, out, latSize, lonSize); } - - // } - } writeLine("", out); - } private static void generatePlacemark(String style, Cell cell, int z, HttpServletResponse out, double latSize, double lonSize) { - writeLine("", out); - writeLine("" + cell.getId() + "", out); - writeLine("" + style + "", out); - writeLine("", out); - writeLine("relativeToGround", out); - writeLine("1", out); - writeLine("", out); - writeLine("", out); - writeLine("", out); + writeLine(" ", out); + writeLine(" " + cell.getId() + "", out); + writeLine(" " + style + "", out); + writeLine(" ", out); + writeLine(" relativeToGround", out); + writeLine(" 1", out); + writeLine(" ", out); + writeLine(" ", out); + writeLine(" ", out); writeLine( cell.getLongitude() + "," + cell.getLatitude() + "," + z + " " + (cell.getLongitude() + lonSize) + "," @@ -174,12 +169,11 @@ private static void generatePlacemark(String style, Cell cell, int z, HttpServle + (cell.getLatitude() + latSize) + "," + z + " " + cell.getLongitude() + "," + (cell.getLatitude() + latSize) + "," + z, out); - writeLine("", out); - writeLine("", out); - writeLine("", out); - writeLine("", out); - writeLine("", out); - + writeLine(" ", out); + writeLine(" ", out); + writeLine(" ", out); + writeLine(" ", out); + writeLine(" ", out); } } diff --git a/src/main/java/dk/dma/ais/coverage/export/generators/XMLGenerator.java b/src/main/java/dk/dma/ais/coverage/export/generators/XMLGenerator.java index 2c3e843..5f77940 100644 --- a/src/main/java/dk/dma/ais/coverage/export/generators/XMLGenerator.java +++ b/src/main/java/dk/dma/ais/coverage/export/generators/XMLGenerator.java @@ -14,28 +14,25 @@ */ package dk.dma.ais.coverage.export.generators; +import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.data.Source; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Collection; import java.util.Date; -import javax.servlet.http.HttpServletResponse; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import dk.dma.ais.coverage.data.Cell; -import dk.dma.ais.coverage.data.Source; - public class XMLGenerator { - private static final Logger LOG = LoggerFactory.getLogger(XMLGenerator.class); public static void generateXML(Collection grids, double latSize, double lonSize, int multiplicity, HttpServletResponse response) { - LOG.info("startet csv generation"); + LOG.info("Started XML generation"); HttpServletResponse out = response; @@ -63,7 +60,7 @@ public static void generateXML(Collection grids, double latSize, double LOG.error(e.getMessage()); e.printStackTrace(); } - LOG.info("Finished csv generation"); + LOG.info("Finished XML generation"); } private static void writeLine(String line, HttpServletResponse out) { @@ -76,25 +73,18 @@ private static void writeLine(String line, HttpServletResponse out) { } private static void generateGrid(String bsMmsi, Collection cells, HttpServletResponse out, double latSize, double lonSize) { - for (Cell cell : cells) { - - // We ignore cells, where average number of messages, is below 10 per ship - // Maybe there is a bug in AISMessage system, that assign some messages to wrong Base Stations - // Bug found and fixed - // if (cell.NOofReceivedSignals / cell.ships.size() > 10) { - writeLine("", out); - writeLine("" + cell.getLatitude() + "", out); - writeLine("" + cell.getLongitude() + "", out); - writeLine("" + (cell.getLatitude() + latSize) + "", out); - writeLine("" + (cell.getLongitude() + lonSize) + "", out); - writeLine("" + cell.getNOofReceivedSignals() + "", out); - writeLine("" + cell.getNOofMissingSignals() + "", out); - writeLine("" + (cell.getCoverage() * 100) + "", out); + writeLine(" " + cell.getLatitude() + "", out); + writeLine(" " + cell.getLongitude() + "", out); + writeLine(" " + (cell.getLatitude() + latSize) + "", out); + writeLine(" " + (cell.getLongitude() + lonSize) + "", out); + writeLine(" " + cell.getNOofReceivedSignals() + "", out); + writeLine(" " + cell.getNOofMissingSignals() + "", out); + writeLine(" " + (cell.getCoverage() * 100) + "", out); + writeLine(" " + cell.getNumberOfVsiMessages() + "", out); + writeLine(" " + cell.getAverageSignalStrength() + "", out); writeLine("", out); - - // } } } } diff --git a/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java b/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java index 9ea1d81..34be56c 100644 --- a/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java +++ b/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java @@ -18,8 +18,17 @@ import dk.dma.ais.coverage.CoverageHandler; import dk.dma.ais.coverage.Helper; import dk.dma.ais.coverage.calculator.TerrestrialCalculator; -import dk.dma.ais.coverage.data.*; -import dk.dma.ais.coverage.export.data.*; +import dk.dma.ais.coverage.data.Cell; +import dk.dma.ais.coverage.data.ICoverageData; +import dk.dma.ais.coverage.data.OnlyMemoryData; +import dk.dma.ais.coverage.data.Source; +import dk.dma.ais.coverage.data.TimeSpan; +import dk.dma.ais.coverage.export.ExportDataType; +import dk.dma.ais.coverage.export.data.ExportShipTimeSpan; +import dk.dma.ais.coverage.export.data.JSonCoverageMap; +import dk.dma.ais.coverage.export.data.JsonConverter; +import dk.dma.ais.coverage.export.data.JsonSource; +import dk.dma.ais.coverage.export.data.Status; import dk.dma.ais.coverage.export.generators.ChartGenerator; import dk.dma.ais.coverage.export.generators.KMLGenerator; import dk.dma.ais.coverage.export.generators.XMLGenerator; @@ -29,14 +38,25 @@ import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.*; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.UriInfo; import java.io.IOException; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; //import dk.dma.ais.coverage.export.CSVGenerator; @@ -104,8 +124,6 @@ public JSonCoverageMap coverage(@Context HttpServletRequest request) { String area = request.getParameter("area"); long starttime = Long.parseLong(request.getParameter("starttime")); long endtime = Long.parseLong(request.getParameter("endtime")); -// System.out.println(starttime); -// System.out.println(endtime); String[] areaArray = area.split(","); @@ -135,13 +153,13 @@ public JSonCoverageMap coverage(@Context HttpServletRequest request) { @Path("export") @Produces(MediaType.APPLICATION_JSON) public Object export(@QueryParam("exportType") String exportType, @QueryParam("exportMultiFactor") String exportMultiFactor, - @Context HttpServletResponse response, @QueryParam("startTime") String startTime, @QueryParam("endTime") String endTime) { - // return null; + @Context HttpServletResponse response, @QueryParam("startTime") String startTime, @QueryParam("endTime") String endTime, + @QueryParam("exportDataType") String exportDataType) { + int multiplicity = Integer.parseInt(exportMultiFactor); - long starttime = Long.parseLong(startTime); - long endtime = Long.parseLong(endTime); -// System.out.println(starttime); -// System.out.println(endtime); + Date starttime = new Date(Long.parseLong(startTime)); + Date endtime = new Date(Long.parseLong(endTime)); + ExportDataType dataType = ExportDataType.forType(exportDataType); // // // BaseStationHandler gh = new BaseStationHandler(); @@ -179,26 +197,23 @@ public Object export(@QueryParam("exportType") String exportType, @QueryParam("e Cell activesbscell = superbs.getGrid().get(cell.getId()); if (activesbscell != null) { - int receivedsignals = cell.getNOofReceivedSignals(new Date(starttime), new Date(endtime)); + int receivedsignals = cell.getNOofReceivedSignals(starttime, endtime); dhCell.addReceivedSignals(receivedsignals); - int sbstotalmessages = activesbscell.getNOofReceivedSignals(new Date(starttime), new Date(endtime)) - + activesbscell.getNOofMissingSignals(new Date(starttime), new Date(endtime)); + + int sbstotalmessages = activesbscell.getNOofReceivedSignals(starttime, endtime) + + activesbscell.getNOofMissingSignals(starttime, endtime); dhCell.addNOofMissingSignals(sbstotalmessages - receivedsignals); - } - // LOG.debug("cell for export created: " + summedbs.getCell(cell.getLatitude(), - // cell.getLongitude()).getNOofReceivedSignals() + "-" + summedbs.getCell(cell.getLatitude(), - // cell.getLongitude()).getNOofMissingSignals()); + dhCell.addVsiMessages(activesbscell.getNumberOfVsiMessages(starttime, endtime), + activesbscell.getAverageSignalStrength(starttime, endtime)); + } } } if (exportType.equals("KML")) { - // System.out.println(expotype); - KMLGenerator.generateKML(dh.getSources(), AisCoverage.get().getConf().getLatSize(), AisCoverage.get().getConf().getLonSize(), multiplicity, response); + KMLGenerator.generateKML(dh.getSources(), AisCoverage.get().getConf().getLatSize(), AisCoverage.get().getConf().getLonSize(), multiplicity, dataType, response); } else if (exportType.equals("CSV")) { - // System.out.println(expotype); // CSVGenerator.generateCSV(dh.getSources(), Helper.latSize, Helper.lonSize, multiplicity, response); } else if (exportType.equals("XML")) { - // System.out.println(expotype); XMLGenerator.generateXML(dh.getSources(), AisCoverage.get().getConf().getLatSize(), AisCoverage.get().getConf().getLonSize(), multiplicity, response); } else { System.out.println("wrong exporttype"); diff --git a/src/test/java/dk/dma/ais/coverage/export/ExportDataTypeTest.java b/src/test/java/dk/dma/ais/coverage/export/ExportDataTypeTest.java new file mode 100644 index 0000000..5909a14 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/export/ExportDataTypeTest.java @@ -0,0 +1,34 @@ +package dk.dma.ais.coverage.export; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.closeTo; +import static org.junit.Assert.assertThat; + +public class ExportDataTypeTest { + + @Test + public void givenReceivedMessages_whenValueOf_thenReceivedMessagesIsReturned() { + ExportDataType exportDataType = ExportDataType.forType("received_messages"); + + assertThat(exportDataType, is(ExportDataType.RECEIVED_MESSAGES)); + } + + @Test + public void givenSignalStrength_whenValueOf_thensignalStrengthIsReturned() { + ExportDataType exportDataType = ExportDataType.forType("signal_strength"); + + assertThat(exportDataType, is(ExportDataType.SIGNAL_STRENGTH)); + } + + @Test + public void givenReceivedMessages_whenGreenThreshold_thenValueIs80Percent() { + assertThat(ExportDataType.RECEIVED_MESSAGES.greenThreshold(), is(closeTo(0.8, 0.001))); + } + + @Test + public void givenReceivedMessages_whenRedThreshold_thenValueIs50Percent() { + assertThat(ExportDataType.RECEIVED_MESSAGES.redThreshold(), is(closeTo(0.5, 0.001))); + } +} diff --git a/web/index.html b/web/index.html index 9e8cba7..8cdb2fa 100644 --- a/web/index.html +++ b/web/index.html @@ -123,10 +123,10 @@

-

+
+

+
diff --git a/web/js/CoverageUI.js b/web/js/CoverageUI.js index 077d9b0..269c497 100644 --- a/web/js/CoverageUI.js +++ b/web/js/CoverageUI.js @@ -16,6 +16,9 @@ function CoverageUI() { this.endDate; //Is the end time of the analysis or the point where the analysis is now (used for sliding window) this.selectedStartDate; this.selectedEndDate; + this.exportDataType = 'received_messages'; + this.savedExportDataType = 'signal_strength'; + var topleftPoint;//points which define sat box var bottomRightPoint var shiptrackactive = false; @@ -90,17 +93,23 @@ function CoverageUI() { var maxThresholdToSave = self.maxThreshold; var minThresholdToSave = self.minThreshold; var thresholdUnitToSave = self.thresholdUnit; + var exportDataTypeToSave = self.exportDataType; self.maxThreshold = self.savedMaxThreshold; self.minThreshold = self.savedMinThreshold; self.thresholdUnit = self.savedThresholdUnit; + self.exportDataType = self.savedExportDataType; self.savedMaxThreshold = maxThresholdToSave; self.savedMinThreshold = minThresholdToSave; self.savedThresholdUnit = thresholdUnitToSave; + self.savedExportDataType = exportDataTypeToSave; self.drawSlider(); self.cleanupCellDetails(); + + $('#exportDataType').val(self.exportDataType); + self.changed = true; }); From 3d11b32de80cf61cb9b5dbb6baac0afb2f40f4bd Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 26 Mar 2017 19:51:11 -0400 Subject: [PATCH 36/59] Do not filter out any message --- .../java/dk/dma/ais/coverage/calculator/SatCalculator.java | 7 ------- .../dma/ais/coverage/calculator/TerrestrialCalculator.java | 5 +++++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/calculator/SatCalculator.java b/src/main/java/dk/dma/ais/coverage/calculator/SatCalculator.java index e53160b..3b03afc 100644 --- a/src/main/java/dk/dma/ais/coverage/calculator/SatCalculator.java +++ b/src/main/java/dk/dma/ais/coverage/calculator/SatCalculator.java @@ -397,13 +397,6 @@ private TimeSpan mergeTimeSpans(TimeSpan span1, TimeSpan span2) { */ @Override public boolean filterMessage(CustomMessage customMessage) { - - // if(customMessage.getSog() < 3 || customMessage.getSog() > 50) - // return true; - if (customMessage.getCog() == 360) { - return true; - } - return false; } diff --git a/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java b/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java index c8b3c60..69d0bfb 100644 --- a/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java +++ b/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java @@ -291,6 +291,11 @@ public JSonCoverageMap getTerrestrialCoverage(double latStart, double lonStart, return map; } + @Override + public boolean filterMessage(CustomMessage customMessage) { + return false; + } + // Getters and setters private double getY(double seconds, Long p1Time, Long p2Time, double p1y, double p2y) { double distanceInMeters = p2y - p1y; From ae521a78c7b35c19f68e033ff2c317a8c82b5410 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 26 Mar 2017 19:51:49 -0400 Subject: [PATCH 37/59] Add missing signals to specific source --- .../dk/dma/ais/coverage/calculator/TerrestrialCalculator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java b/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java index 69d0bfb..35bdf05 100644 --- a/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java +++ b/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java @@ -200,6 +200,10 @@ private void calculateMissingPoints(CustomMessage m1, CustomMessage m2, boolean Date stamp = new Date((long) (m1.getTimestamp().getTime() + (i * expectedTransmittingFrequency * 1000))); dataHandler.incrementMissingSignals(AbstractCalculator.SUPERSOURCE_MMSI, projection.y2Lat(xMissing, yMissing), projection.x2Lon(xMissing, yMissing), stamp); + for (String source : m1.getSourceList()) { + dataHandler.incrementMissingSignals(source, projection.y2Lat(xMissing, yMissing), + projection.x2Lon(xMissing, yMissing), stamp); + } } } } From 227f4482a23c566264ecb9f20583ee7a0e7f4867 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 26 Mar 2017 19:52:28 -0400 Subject: [PATCH 38/59] Ensure up-to-date cell is used for coverage calculation --- .../ais/coverage/calculator/TerrestrialCalculator.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java b/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java index 35bdf05..1cfff96 100644 --- a/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java +++ b/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java @@ -248,7 +248,7 @@ public JSonCoverageMap getTerrestrialCoverage(double latStart, double lonStart, map.latSize = conf.getLatSize() * multiplicationFactor; map.lonSize = conf.getLonSize() * multiplicationFactor; - HashMap JsonCells = new HashMap(); + HashMap JsonCells = new HashMap<>(); QueryParams params = new QueryParams(); params.latStart = latStart; @@ -281,12 +281,8 @@ public JSonCoverageMap getTerrestrialCoverage(double latStart, double lonStart, if (superCell == null) { } else { - ExportCell existing = JsonCells.get(cell.getId()); - ExportCell theCell = JsonConverter.toJsonCell(cell, superCell, starttime, endtime); - if (existing == null || theCell.getCoverage() > existing.getCoverage()) { - JsonCells.put(cell.getId(), theCell); - } - + ExportCell theCell = JsonConverter.toJsonCell(cell, superCell, starttime, endtime); + JsonCells.put(cell.getId(), theCell); } } From 3ba9fac6f6cdf97a8381a0bb5812a684a1ea2c88 Mon Sep 17 00:00:00 2001 From: John Martel Date: Tue, 28 Mar 2017 07:59:19 -0400 Subject: [PATCH 39/59] Use persisted cell id --- src/main/java/dk/dma/ais/coverage/data/Cell.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/dk/dma/ais/coverage/data/Cell.java b/src/main/java/dk/dma/ais/coverage/data/Cell.java index 890d044..a240f2d 100644 --- a/src/main/java/dk/dma/ais/coverage/data/Cell.java +++ b/src/main/java/dk/dma/ais/coverage/data/Cell.java @@ -27,6 +27,7 @@ public class Cell { private int averageSignalStrength; private double latitude; private double longitude; + private final String id; private List timeSpans; private Map fixedWidthSpans = new HashMap(); @@ -49,11 +50,13 @@ public void setTimeSpans(List timeSpans) { public Cell(Source grid, double lat, double lon, String id) { this.latitude = lat; this.longitude = lon; + this.id = id; } public Cell(double lat, double lon, String id) { this.latitude = lat; this.longitude = lon; + this.id = id; } public synchronized void incrementNOofReceivedSignals() { @@ -103,7 +106,7 @@ public void setLongitude(double longitude) { } public String getId() { - return this.latitude + "_" + this.longitude; + return this.id; } public synchronized int getNOofReceivedSignals(Date starttime, Date endTime) { From a8b59ed4ea2e065e02489fdeaf6682fb322d986e Mon Sep 17 00:00:00 2001 From: John Martel Date: Tue, 28 Mar 2017 07:59:56 -0400 Subject: [PATCH 40/59] Display cells that have only missing signals --- .../dk/dma/ais/coverage/calculator/TerrestrialCalculator.java | 4 +--- src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java b/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java index 1cfff96..5d1bd00 100644 --- a/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java +++ b/src/main/java/dk/dma/ais/coverage/calculator/TerrestrialCalculator.java @@ -267,9 +267,7 @@ public JSonCoverageMap getTerrestrialCoverage(double latStart, double lonStart, List celllistSuper = dataHandler.getCells(params); Map superMap = new HashMap(); for (Cell cell : celllistSuper) { - if (cell.getNOofReceivedSignals() > 0) { - superMap.put(cell.getId(), cell); - } + superMap.put(cell.getId(), cell); } if (!celllist.isEmpty()) { diff --git a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java index a6adba4..71e76da 100644 --- a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java +++ b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java @@ -199,9 +199,7 @@ private List getCells(double latStart, double lonStart, // add cells for particular source to cell-list. for (Cell cell : cellMultiplicationSource.getGrid().values()) { - if (cell.getNOofReceivedSignals() > 0) { - cells.add(cell); - } + cells.add(cell); } } } From 3f21599a4feac38b4dc2bc3d317acebb047978cf Mon Sep 17 00:00:00 2001 From: John Martel Date: Tue, 28 Mar 2017 08:00:40 -0400 Subject: [PATCH 41/59] Bump version number --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b0958d3..c6ddfaa 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ dk.dma.ais ais-coverage - 0.3.0-CCG + 0.3.1-CCG AIS coverage analyzer AIS coverage analyzer From d417c9ee4d6c73c734addafec0726ca13ee6afbc Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 14 May 2017 20:37:32 -0400 Subject: [PATCH 42/59] Bump version number for new bugfixes --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c6ddfaa..1b423a5 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ dk.dma.ais ais-coverage - 0.3.1-CCG + 0.3.2-CCG AIS coverage analyzer AIS coverage analyzer From 088dda3621d444e77acfa675d1e55e014861e502 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 14 May 2017 21:07:42 -0400 Subject: [PATCH 43/59] Change default coverage thresholds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Canadian Coast Guard prefers to have Red < 20% < Yellow <= 50% for number of messages received and Red < -107dBm < Yellow <= -101dBm for VSI coverage. --- .../java/dk/dma/ais/coverage/export/ExportDataType.java | 2 +- .../dk/dma/ais/coverage/export/ExportDataTypeTest.java | 8 ++++---- web/js/CoverageUI.js | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/export/ExportDataType.java b/src/main/java/dk/dma/ais/coverage/export/ExportDataType.java index 7166e16..9251d3c 100644 --- a/src/main/java/dk/dma/ais/coverage/export/ExportDataType.java +++ b/src/main/java/dk/dma/ais/coverage/export/ExportDataType.java @@ -4,7 +4,7 @@ * Supported data types for export. */ public enum ExportDataType { - RECEIVED_MESSAGES(0.8, 0.5), SIGNAL_STRENGTH(-75, -110); + RECEIVED_MESSAGES(0.5, 0.2), SIGNAL_STRENGTH(-101, -107); private final double greenThreshold; private final double redThreshold; diff --git a/src/test/java/dk/dma/ais/coverage/export/ExportDataTypeTest.java b/src/test/java/dk/dma/ais/coverage/export/ExportDataTypeTest.java index 5909a14..7f1d141 100644 --- a/src/test/java/dk/dma/ais/coverage/export/ExportDataTypeTest.java +++ b/src/test/java/dk/dma/ais/coverage/export/ExportDataTypeTest.java @@ -23,12 +23,12 @@ public void givenSignalStrength_whenValueOf_thensignalStrengthIsReturned() { } @Test - public void givenReceivedMessages_whenGreenThreshold_thenValueIs80Percent() { - assertThat(ExportDataType.RECEIVED_MESSAGES.greenThreshold(), is(closeTo(0.8, 0.001))); + public void givenReceivedMessages_whenGreenThreshold_thenValueIs50Percent() { + assertThat(ExportDataType.RECEIVED_MESSAGES.greenThreshold(), is(closeTo(0.5, 0.001))); } @Test - public void givenReceivedMessages_whenRedThreshold_thenValueIs50Percent() { - assertThat(ExportDataType.RECEIVED_MESSAGES.redThreshold(), is(closeTo(0.5, 0.001))); + public void givenReceivedMessages_whenRedThreshold_thenValueIs20Percent() { + assertThat(ExportDataType.RECEIVED_MESSAGES.redThreshold(), is(closeTo(0.2, 0.001))); } } diff --git a/web/js/CoverageUI.js b/web/js/CoverageUI.js index 269c497..d6d59ae 100644 --- a/web/js/CoverageUI.js +++ b/web/js/CoverageUI.js @@ -4,10 +4,10 @@ function CoverageUI() { this.sources = []; this.selectedSource = null; this.changed = true; - this.minThreshold = 50; - this.maxThreshold = 80; - this.savedMinThreshold = -110; - this.savedMaxThreshold = -75; + this.minThreshold = 20; + this.maxThreshold = 50; + this.savedMinThreshold = -107; + this.savedMaxThreshold = -101; this.thresholdUnit = "%" this.savedThresholdUnit = 'dB'; this.minExpectedMessages = 100; From 2ae4b91af8d596c305b9b2a1c43c0423f3dc336d Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 14 May 2017 21:13:29 -0400 Subject: [PATCH 44/59] Fix VSI coverage unit Also increase lightly side panel's size to show VSI thresholds on single line. --- web/css/style.css | 4 ++-- web/js/CoverageUI.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/css/style.css b/web/css/style.css index 68ebe99..1a0bf71 100644 --- a/web/css/style.css +++ b/web/css/style.css @@ -77,7 +77,7 @@ body { } #leftSide { - width: 230px; + width: 250px; padding: 0px; vertical-align: middle; position: absolute; @@ -287,4 +287,4 @@ hr { .grayPointer { background-image: url('../img/graypointer.png'); background-repeat: no-repeat; -} \ No newline at end of file +} diff --git a/web/js/CoverageUI.js b/web/js/CoverageUI.js index d6d59ae..638b595 100644 --- a/web/js/CoverageUI.js +++ b/web/js/CoverageUI.js @@ -9,7 +9,7 @@ function CoverageUI() { this.savedMinThreshold = -107; this.savedMaxThreshold = -101; this.thresholdUnit = "%" - this.savedThresholdUnit = 'dB'; + this.savedThresholdUnit = 'dBm'; this.minExpectedMessages = 100; this.exportMultiplicationFactor = 4; this.startDate; //is the time when the analysis started (used for sliding window) From 7f8b588348ec406e4831d2747074f1a695fbe311 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 14 May 2017 21:17:27 -0400 Subject: [PATCH 45/59] Change VSI thresholds slider's range The previous range was too broad for actually plausible values, making slight corrections using the slider difficult to do. --- web/js/CoverageUI.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/js/CoverageUI.js b/web/js/CoverageUI.js index 638b595..0dfb88c 100644 --- a/web/js/CoverageUI.js +++ b/web/js/CoverageUI.js @@ -222,8 +222,8 @@ function CoverageUI() { this.drawSlider = function () { $("#slider-range").dragslider({ range: true, - min: self.isVDMCategory() ? 0 : -200, - max: self.isVDMCategory() ? 100 : 0, + min: self.isVDMCategory() ? 0 : -120, + max: self.isVDMCategory() ? 100 : -85, rangeDrag: true, values: [self.minThreshold, self.maxThreshold], slide: function (event, ui) { From 2933612a2cfa11213d276142c75d8ba49b2bbb42 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 14 May 2017 21:48:02 -0400 Subject: [PATCH 46/59] Fill KML polygons with some transparency --- .../ais/coverage/export/generators/KMLGenerator.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java b/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java index 9e1d5d3..e539c50 100644 --- a/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java +++ b/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java @@ -67,7 +67,8 @@ public static void generateKML(Collection grids, double latSize, double writeLine(" ff0000ff", out); writeLine(" ", out); writeLine(" ", out); - writeLine(" ff0000ff", out); + writeLine(" 550000ff", out); + writeLine(" 1", out); writeLine(" ", out); writeLine("", out); writeLine("", out); writeLine("", out); for (Source grid : grids) { From 7edc19de92c930c1973f811cf2174ca18c3d1c9b Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 14 May 2017 22:04:24 -0400 Subject: [PATCH 47/59] Show configuration of AisBus queue size in sample The AisBus can overflow quite fast with the default configuration, so it may prove useful to configure the queue size. --- src/main/resources/coverage-mongodb-sample.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/coverage-mongodb-sample.xml b/src/main/resources/coverage-mongodb-sample.xml index ba5790b..017d8dc 100644 --- a/src/main/resources/coverage-mongodb-sample.xml +++ b/src/main/resources/coverage-mongodb-sample.xml @@ -18,6 +18,7 @@ xsi:type="distributerConsumerConfiguration"> UNFILTERED + 100000 From 11f4456ad091b140d84d0eee150ef8e159bd72fb Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 11 Jun 2017 20:39:01 -0400 Subject: [PATCH 48/59] Document how to run the program --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 444a333..85d5087 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,10 @@ AisCoverage is a tool for calculating how well AIS receivers (sources) cover a g M2 Eclipse plugin or ./mvnw eclipse:eclipse + +## Running + + target/ais-coverage-[version]-dist/ais-coverage-[version]/coverage.sh ## Rest API ## From 4eb6b6a136786beca0dfd6fe0ab359cc637a5667 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 11 Jun 2017 22:04:53 -0400 Subject: [PATCH 49/59] Enable exporting to CSV --- .../dk/dma/ais/coverage/export/generators/CSVGenerator.java | 3 +-- .../java/dk/dma/ais/coverage/rest/CoverageRestService.java | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/export/generators/CSVGenerator.java b/src/main/java/dk/dma/ais/coverage/export/generators/CSVGenerator.java index 01e31ad..c16c950 100644 --- a/src/main/java/dk/dma/ais/coverage/export/generators/CSVGenerator.java +++ b/src/main/java/dk/dma/ais/coverage/export/generators/CSVGenerator.java @@ -42,11 +42,10 @@ public static void generateCSV(Collection grids, double latSize, double String fileName = "aiscoverage-" + dateFormat.format(date) + "_latSize " + latSize + "_lonSize " + lonSize + "multiplicationfactor" + multiplicity + ".csv"; - // out.setContentType("application/vnd.google-earth.kml+xml"); out.setContentType("text/csv"); out.setHeader("Content-Disposition", "attachment; filename=" + fileName); - writeLine("latstart, longstart, latend, longend, received, missing, coverage percentage", out); + writeLine("latstart, longstart, latend, longend, received, missing, coverage percentage, receivedvsimessages, averagesignalstrength", out); for (Source grid : grids) { generateGrid(grid.getIdentifier(), grid.getGrid().values(), out, latSize * multiplicity, lonSize * multiplicity); diff --git a/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java b/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java index 34be56c..68c72e9 100644 --- a/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java +++ b/src/main/java/dk/dma/ais/coverage/rest/CoverageRestService.java @@ -29,6 +29,7 @@ import dk.dma.ais.coverage.export.data.JsonConverter; import dk.dma.ais.coverage.export.data.JsonSource; import dk.dma.ais.coverage.export.data.Status; +import dk.dma.ais.coverage.export.generators.CSVGenerator; import dk.dma.ais.coverage.export.generators.ChartGenerator; import dk.dma.ais.coverage.export.generators.KMLGenerator; import dk.dma.ais.coverage.export.generators.XMLGenerator; @@ -212,7 +213,7 @@ public Object export(@QueryParam("exportType") String exportType, @QueryParam("e if (exportType.equals("KML")) { KMLGenerator.generateKML(dh.getSources(), AisCoverage.get().getConf().getLatSize(), AisCoverage.get().getConf().getLonSize(), multiplicity, dataType, response); } else if (exportType.equals("CSV")) { - // CSVGenerator.generateCSV(dh.getSources(), Helper.latSize, Helper.lonSize, multiplicity, response); + CSVGenerator.generateCSV(dh.getSources(), AisCoverage.get().getConf().getLatSize(), AisCoverage.get().getConf().getLonSize(), multiplicity, response); } else if (exportType.equals("XML")) { XMLGenerator.generateXML(dh.getSources(), AisCoverage.get().getConf().getLatSize(), AisCoverage.get().getConf().getLonSize(), multiplicity, response); } else { From 1422097cb54b1ee3f9b65dcce84a3e062e36ac84 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 11 Jun 2017 22:05:32 -0400 Subject: [PATCH 50/59] Bump version number --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1b423a5..793b02f 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ dk.dma.ais ais-coverage - 0.3.2-CCG + 0.4.0-CCG AIS coverage analyzer AIS coverage analyzer From d64743de883cfb207aea9d4784188cfe0f4699e1 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 18 Jun 2017 20:52:16 -0400 Subject: [PATCH 51/59] Fix KML placemarks fill color in Google Earth --- .../export/generators/KMLGenerator.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java b/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java index e539c50..a8a24ac 100644 --- a/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java +++ b/src/main/java/dk/dma/ais/coverage/export/generators/KMLGenerator.java @@ -99,7 +99,7 @@ public static void generateKML(Collection grids, double latSize, double writeLine(" ff00ff00", out); writeLine(" ", out); writeLine(" ", out); - writeLine(" 5500ff55", out); + writeLine(" 5500ff00", out); writeLine(" 1", out); writeLine(" ", out); writeLine("", out); @@ -143,11 +143,11 @@ private static void generateGrid(String bsMmsi, Collection cells, HttpServ } if (dataToExport > exportDataType.greenThreshold()) { // green - generatePlacemark("#greenStyle", cell, 300, out, latSize, lonSize); + generatePlacemark("#greenStyle", cell, 0, out, latSize, lonSize); } else if (dataToExport > exportDataType.redThreshold()) { // orange - generatePlacemark("#orangeStyle", cell, 200, out, latSize, lonSize); + generatePlacemark("#orangeStyle", cell, 0, out, latSize, lonSize); } else { // red - generatePlacemark("#redStyle", cell, 100, out, latSize, lonSize); + generatePlacemark("#redStyle", cell, 0, out, latSize, lonSize); } } @@ -160,17 +160,18 @@ private static void generatePlacemark(String style, Cell cell, int z, HttpServle writeLine(" " + cell.getId() + "", out); writeLine(" " + style + "", out); writeLine(" ", out); - writeLine(" relativeToGround", out); + writeLine(" clampedToGround", out); writeLine(" 1", out); writeLine(" ", out); writeLine(" ", out); writeLine(" ", out); writeLine( - cell.getLongitude() + "," + cell.getLatitude() + "," + z + " " + (cell.getLongitude() + lonSize) + "," - + cell.getLatitude() + "," + z + " " + (cell.getLongitude() + lonSize) + "," - + (cell.getLatitude() + latSize) + "," + z + " " + cell.getLongitude() + "," - + (cell.getLatitude() + latSize) + "," + z, out); + cell.getLongitude() + "," + cell.getLatitude() + "," + z + " " + + (cell.getLongitude() + lonSize) + "," + cell.getLatitude() + "," + z + " " + + (cell.getLongitude() + lonSize) + "," + (cell.getLatitude() + latSize) + "," + z + " " + + cell.getLongitude() + "," + (cell.getLatitude() + latSize) + "," + z + " " + + cell.getLongitude() + "," + cell.getLatitude() + "," + z, out); writeLine(" ", out); writeLine(" ", out); From 3c12b431c8e3ddd40630b7eb2d71838716da0859 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 18 Jun 2017 22:05:50 -0400 Subject: [PATCH 52/59] Log errors to a separate file Some data-related errors can be clogging logs if they happen for every packet received, making logs impossible to read. By separating errors from other log messages, it is easier to go through the logs. --- src/main/resources/log4j.xml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/main/resources/log4j.xml b/src/main/resources/log4j.xml index 75ed97a..2d51908 100644 --- a/src/main/resources/log4j.xml +++ b/src/main/resources/log4j.xml @@ -19,7 +19,21 @@ - + + + + + + + + + + + + + + + @@ -27,6 +41,7 @@ + @@ -35,6 +50,7 @@ + From bf8e9173b3801b10091c5bfba836ac7cd4611e02 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 18 Jun 2017 23:13:43 -0400 Subject: [PATCH 53/59] Allow configuration of message queue size --- .../dk/dma/ais/coverage/CoverageHandler.java | 11 +++++-- .../AisCoverageConfiguration.java | 29 ++++++++++++------- .../AisCoverageConfigurationTest.java | 17 +++++++++++ 3 files changed, 44 insertions(+), 13 deletions(-) create mode 100644 src/test/java/dk/dma/ais/coverage/configuration/AisCoverageConfigurationTest.java diff --git a/src/main/java/dk/dma/ais/coverage/CoverageHandler.java b/src/main/java/dk/dma/ais/coverage/CoverageHandler.java index 2aad68a..49e1cb5 100644 --- a/src/main/java/dk/dma/ais/coverage/CoverageHandler.java +++ b/src/main/java/dk/dma/ais/coverage/CoverageHandler.java @@ -37,7 +37,6 @@ */ public class CoverageHandler { private static final Logger LOG = LoggerFactory.getLogger(CoverageHandler.class); - public static final int MESSAGE_BUFFER_SIZE = 10000; private List calculators = new ArrayList(); private ICoverageData dataHandler; @@ -49,13 +48,17 @@ public class CoverageHandler { @Override protected boolean removeEldestEntry(Map.Entry eldest) { - if (this.size() > MESSAGE_BUFFER_SIZE) { + if (this.size() > getMessageBufferSize()) { process(eldest.getValue()); } - return this.size() > MESSAGE_BUFFER_SIZE; + return this.size() > getMessageBufferSize(); } }; + private int getMessageBufferSize() { + return this.conf.getMessageBufferSize(); + } + //Fields used for debugging purposes private int unfiltCount; private long biggestDelay; @@ -67,6 +70,8 @@ public CoverageHandler(AisCoverageConfiguration conf) { this.conf=conf; Helper.conf=conf; + LOG.info("Message buffer size initialized with value [{}]", getMessageBufferSize()); + //Creating up data handler dataHandler = new OnlyMemoryData(); LOG.info("coverage calculators set up with memory only data handling"); diff --git a/src/main/java/dk/dma/ais/coverage/configuration/AisCoverageConfiguration.java b/src/main/java/dk/dma/ais/coverage/configuration/AisCoverageConfiguration.java index 2a7eb70..5132c21 100644 --- a/src/main/java/dk/dma/ais/coverage/configuration/AisCoverageConfiguration.java +++ b/src/main/java/dk/dma/ais/coverage/configuration/AisCoverageConfiguration.java @@ -14,12 +14,9 @@ */ package dk.dma.ais.coverage.configuration; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.util.HashMap; -import java.util.Map; +import dk.dma.ais.configuration.bus.AisBusConfiguration; +import dk.dma.ais.coverage.data.Source_UserProvided; +import dk.dma.ais.coverage.web.WebServerConfiguration; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; @@ -27,10 +24,12 @@ import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; - -import dk.dma.ais.configuration.bus.AisBusConfiguration; -import dk.dma.ais.coverage.data.Source_UserProvided; -import dk.dma.ais.coverage.web.WebServerConfiguration; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.util.HashMap; +import java.util.Map; /** * Class to represent AIS coverage configuration. To be marshalled and @@ -43,6 +42,7 @@ public class AisCoverageConfiguration { private WebServerConfiguration serverConfiguration; private double latSize = 0.0225225225; private double lonSize = 0.0386812541; + private int messageBufferSize = 10000; private int verbosityLevel; private DatabaseConfiguration dbConf = new DatabaseConfiguration(); private Map sourcenames = new HashMap(); @@ -77,6 +77,7 @@ public void setAisbusConfiguration(AisBusConfiguration aisbusConfiguration) { this.aisbusConfiguration = aisbusConfiguration; } + public WebServerConfiguration getServerConfiguration() { return serverConfiguration; } @@ -102,6 +103,14 @@ public double getLonSize() { return this.lonSize; } + public int getMessageBufferSize() { + return messageBufferSize; + } + + public void setMessageBufferSize(int messageBufferSize) { + this.messageBufferSize = messageBufferSize; + } + public static void save(String filename, AisCoverageConfiguration conf) throws JAXBException, FileNotFoundException { JAXBContext context = JAXBContext diff --git a/src/test/java/dk/dma/ais/coverage/configuration/AisCoverageConfigurationTest.java b/src/test/java/dk/dma/ais/coverage/configuration/AisCoverageConfigurationTest.java new file mode 100644 index 0000000..5c452c0 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/configuration/AisCoverageConfigurationTest.java @@ -0,0 +1,17 @@ +package dk.dma.ais.coverage.configuration; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class AisCoverageConfigurationTest { + + @Test + public void whenNewInstance_thenMessageBufferSizeDefaultsTo10000() { + AisCoverageConfiguration configuration = new AisCoverageConfiguration(); + + assertThat(configuration.getMessageBufferSize(), is(equalTo(10000))); + } +} From 6d37acd40a539b4b6e0d7d795eb76c6e50800b23 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 18 Jun 2017 23:14:44 -0400 Subject: [PATCH 54/59] Handle messages of types 18 and 19 --- .../java/dk/dma/ais/coverage/data/OnlyMemoryData.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java index 71e76da..8d934f1 100644 --- a/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java +++ b/src/main/java/dk/dma/ais/coverage/data/OnlyMemoryData.java @@ -22,7 +22,7 @@ import dk.dma.ais.coverage.data.Source.ReceiverType; import dk.dma.ais.message.AisMessage; import dk.dma.ais.message.AisMessage4; -import dk.dma.ais.message.AisPositionMessage; +import dk.dma.ais.message.IVesselPositionMessage; import dk.dma.ais.packet.AisPacket; import dk.dma.ais.packet.AisPacketTags; import dk.dma.ais.packet.AisPacketTags.SourceType; @@ -275,7 +275,7 @@ public CustomMessage packetToCustomMessage(AisPacket packet) { ReceiverType receiverType = ReceiverType.NOTDEFINED; Date timestamp = null; ShipClass shipClass = null; - AisPositionMessage posMessage; + IVesselPositionMessage posMessage; SourceType sourceType = SourceType.TERRESTRIAL; // Get source tag properties @@ -333,8 +333,8 @@ public CustomMessage packetToCustomMessage(AisPacket packet) { // Handle position messages. If it's not a position message // the calculators can't use them - if (aisMessage instanceof AisPositionMessage) { - posMessage = (AisPositionMessage) aisMessage; + if (aisMessage instanceof IVesselPositionMessage) { + posMessage = (IVesselPositionMessage) aisMessage; } else { return null; } From fe1052f6f131b708f1de3e12858ff6767e4f30f4 Mon Sep 17 00:00:00 2001 From: John Martel Date: Sun, 18 Jun 2017 23:18:52 -0400 Subject: [PATCH 55/59] Bump version number --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 793b02f..7463944 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ dk.dma.ais ais-coverage - 0.4.0-CCG + 0.5.0-CCG AIS coverage analyzer AIS coverage analyzer From 118bc8f850108a1e6a272467dcea2de234f545de Mon Sep 17 00:00:00 2001 From: John Martel Date: Mon, 28 Aug 2017 01:19:51 -0400 Subject: [PATCH 56/59] Log first message timestamp at startup This gives an idea when reading the logs about the state of the loaded data at startup. --- src/main/java/dk/dma/ais/coverage/AisCoverage.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/dk/dma/ais/coverage/AisCoverage.java b/src/main/java/dk/dma/ais/coverage/AisCoverage.java index f993b95..fb8c385 100644 --- a/src/main/java/dk/dma/ais/coverage/AisCoverage.java +++ b/src/main/java/dk/dma/ais/coverage/AisCoverage.java @@ -25,9 +25,11 @@ import dk.dma.ais.coverage.web.WebServer; import dk.dma.ais.packet.AisPacket; import dk.dma.ais.reader.AisReader; +import dk.dma.commons.util.DateTimeUtil; import net.jcip.annotations.GuardedBy; import net.jcip.annotations.ThreadSafe; import org.apache.commons.collections4.IterableUtils; +import org.joda.time.PeriodType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,6 +108,9 @@ private void loadCoverageDataFromDatabase() { } Instant end = Instant.now(); LOG.info("Loading coverage data and converting to memory structure took [{}] ms", Duration.between(start, end).toMillis()); + if (Helper.firstMessage != null) { + LOG.info("First message timestamp: {}. Window size should be: {}", Helper.firstMessage, DateTimeUtil.toInterval(Helper.firstMessage, new Date()).toPeriod(PeriodType.hours()).getHours()); + } } private void adjustSystemEarliestMessageFromCell(Cell cell) { From 782139e17a8c12b78e25e6b8ffd98ae9e39cfeaa Mon Sep 17 00:00:00 2001 From: John Martel Date: Mon, 28 Aug 2017 01:02:28 -0400 Subject: [PATCH 57/59] Fix `Purger` handling of large window sizes Large window sizes such as `1000` would trigger an incorrect trimmed window size due to int arithmetic overflow. --- src/main/java/dk/dma/ais/coverage/Purger.java | 33 ++++++++++--------- .../java/dk/dma/ais/coverage/PurgerTest.java | 27 +++++++++++++++ 2 files changed, 44 insertions(+), 16 deletions(-) create mode 100644 src/test/java/dk/dma/ais/coverage/PurgerTest.java diff --git a/src/main/java/dk/dma/ais/coverage/Purger.java b/src/main/java/dk/dma/ais/coverage/Purger.java index ab74981..be77200 100644 --- a/src/main/java/dk/dma/ais/coverage/Purger.java +++ b/src/main/java/dk/dma/ais/coverage/Purger.java @@ -14,12 +14,11 @@ */ package dk.dma.ais.coverage; -import java.util.Date; - +import dk.dma.ais.coverage.data.ICoverageData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import dk.dma.ais.coverage.data.ICoverageData; +import java.util.Date; public class Purger extends Thread { private static final Logger LOG = LoggerFactory.getLogger(Purger.class); @@ -28,8 +27,7 @@ public class Purger extends Thread { private final ICoverageData dataHandler; private final int pollTimeInSeconds; - public Purger(int maxWindowSize, ICoverageData dataHandler, - int pollTimeInSeconds) { + public Purger(int maxWindowSize, ICoverageData dataHandler, int pollTimeInSeconds) { this.maxWindowSize = maxWindowSize; this.dataHandler = dataHandler; this.pollTimeInSeconds = pollTimeInSeconds; @@ -39,20 +37,13 @@ public Purger(int maxWindowSize, ICoverageData dataHandler, public void run() { while (true) { if (Helper.latestMessage != null && Helper.firstMessage != null) { - int windowSize = (int) ((Helper.getCeilDate( - Helper.latestMessage).getTime() - Helper.getFloorDate( - Helper.firstMessage).getTime()) / 1000 / 60 / 60); + int currentWindowSize = getCurrentWindowSize(); - if (windowSize > maxWindowSize) { - Date trimPoint = new Date(Helper.getCeilDate( - Helper.latestMessage).getTime() - - (1000 * 60 * 60 * maxWindowSize)); - LOG.info("Window size: " + windowSize - + ". Max window size: " + maxWindowSize - + ". Lets purge data until " + trimPoint); + if (currentWindowSize > maxWindowSize) { + Date trimPoint = getTrimPoint(); + LOG.info("Window size: {}. Max window size: {}. Lets purge data until {}", currentWindowSize, maxWindowSize, trimPoint); dataHandler.trimWindow(trimPoint); - } } @@ -64,4 +55,14 @@ public void run() { } } + private int getCurrentWindowSize() { + long latestMessageCeilingDate = Helper.getCeilDate(Helper.latestMessage).getTime(); + long firstMessageFloorDate = Helper.getFloorDate(Helper.firstMessage).getTime(); + return (int) ((latestMessageCeilingDate - firstMessageFloorDate) / 1000 / 60 / 60); + } + + Date getTrimPoint() { + return new Date(Helper.getCeilDate(Helper.latestMessage).getTime() - (1000L * 60 * 60 * maxWindowSize)); + } + } diff --git a/src/test/java/dk/dma/ais/coverage/PurgerTest.java b/src/test/java/dk/dma/ais/coverage/PurgerTest.java new file mode 100644 index 0000000..dba3de0 --- /dev/null +++ b/src/test/java/dk/dma/ais/coverage/PurgerTest.java @@ -0,0 +1,27 @@ +package dk.dma.ais.coverage; + +import dk.dma.commons.util.DateTimeUtil; +import org.apache.commons.lang3.time.DateUtils; +import org.joda.time.PeriodType; +import org.junit.Test; + +import java.util.Date; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class PurgerTest { + + @Test + public void givenLargeWindowSize_whenGetTrimPoint_windowSizeCorrectlyConvertedToHours() { + Purger purger = new Purger(1000, null, 1); + Helper.firstMessage = new Date(); + final int maxWindowSize = 1000; + Helper.latestMessage = DateUtils.addHours(new Date(), 2 * maxWindowSize); + + Date trimPoint = purger.getTrimPoint(); + + assertThat(DateTimeUtil.toInterval(Helper.firstMessage, trimPoint).toPeriod(PeriodType.hours()).getHours(), is(equalTo(maxWindowSize))); + } +} From 59cc8397eeda6863fa604a3596899afd310f458c Mon Sep 17 00:00:00 2001 From: John Martel Date: Mon, 28 Aug 2017 01:10:32 -0400 Subject: [PATCH 58/59] Multithread handling of incoming AIS packets Handling AIS packets takes time, as an AIS custom message is extracted from the packet and is added to a buffer that records the source of the message. Over time, this can lead to overflow on the AisBus side as packets are not consumed as fast as they are being received. This introduces a `received but yet unhandled` buffer on the receiver side and uses a fixed thread pool to process and remove packets stored in that buffer. A configuration option is also introduced to restrain this buffer in size and allow overflowing. --- README.md | 23 ++++- .../java/dk/dma/ais/coverage/AisCoverage.java | 1 + .../dk/dma/ais/coverage/CoverageHandler.java | 90 +++++++++++++++---- .../AisCoverageConfiguration.java | 8 ++ .../AisCoverageConfigurationTest.java | 7 ++ 5 files changed, 109 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 85d5087..19881d4 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,28 @@ M2 Eclipse plugin or ## Storing Coverage Data ## Only MongoDB is supported at the moment. Mongo v3.4.2 or above needs to be installed and authentication is not yet supported. -Data (atthe highest detailed level) is persisted by a background thread at regular (configurable) intervals. The default is 60 minutes. +Data (at the highest detailed level) is persisted by a background thread at regular (configurable) intervals. The default is 60 minutes. + +## Handling incoming AIS packets + +To avoid saturating the system with incoming packets, an overflow mechanism is implemented both at the `AisBus` and `CoverageHandler` +levels. The `AisBus` drops packets if lower layers of the application cannot handle more incoming packets. The `CoverageHandler` consumes the +packets provided by the bus through its consumers, but since it requires time to process every single packet, another buffer level is introduced +to let the bus provide as much packets as possible and give handling threads a chance to process messages with dropping as few as possible. + +The size of the `AisBus` can be configured with the `` configuration element, while the `CoverageHandler` can be configured with the `` element: + +```xml + + + 10000 + + + 10000 + +``` + +Past these limits, the system will start overflowing and dropping packets. ## Distribution ## diff --git a/src/main/java/dk/dma/ais/coverage/AisCoverage.java b/src/main/java/dk/dma/ais/coverage/AisCoverage.java index fb8c385..150be1d 100644 --- a/src/main/java/dk/dma/ais/coverage/AisCoverage.java +++ b/src/main/java/dk/dma/ais/coverage/AisCoverage.java @@ -164,6 +164,7 @@ public void stop() { } aisBus.cancel(); + handler.stop(); LOG.info("aisbus stopped"); persisterService.stop(); diff --git a/src/main/java/dk/dma/ais/coverage/CoverageHandler.java b/src/main/java/dk/dma/ais/coverage/CoverageHandler.java index 49e1cb5..8c319b6 100644 --- a/src/main/java/dk/dma/ais/coverage/CoverageHandler.java +++ b/src/main/java/dk/dma/ais/coverage/CoverageHandler.java @@ -14,6 +14,7 @@ */ package dk.dma.ais.coverage; +import dk.dma.ais.bus.OverflowLogger; import dk.dma.ais.coverage.calculator.AbstractCalculator; import dk.dma.ais.coverage.calculator.SatCalculator; import dk.dma.ais.coverage.calculator.TerrestrialCalculator; @@ -27,23 +28,37 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Queue; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; /** * Handler for received AisPackets */ public class CoverageHandler { private static final Logger LOG = LoggerFactory.getLogger(CoverageHandler.class); + private final OverflowLogger overflowLogger = new OverflowLogger(LOG); + + private final Queue unhandledPackets = new ConcurrentLinkedQueue<>(); + private final ExecutorService packetHandlingThreadPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() / 2); + private final int maximumUnhandledPacketsKept; private List calculators = new ArrayList(); + private ICoverageData dataHandler; + private AisCoverageConfiguration conf; //A doublet filtered message buffer, where a custom message will include a list of all sources - private LinkedHashMap doubletBuffer = new LinkedHashMap() { + private Map doubletBuffer = Collections.synchronizedMap(new LinkedHashMap() { private static final long serialVersionUID = 1L; @Override @@ -53,12 +68,11 @@ protected boolean removeEldestEntry(Map.Entry eldest) { } return this.size() > getMessageBufferSize(); } - }; + }); private int getMessageBufferSize() { return this.conf.getMessageBufferSize(); } - //Fields used for debugging purposes private int unfiltCount; private long biggestDelay; @@ -70,6 +84,9 @@ public CoverageHandler(AisCoverageConfiguration conf) { this.conf=conf; Helper.conf=conf; + maximumUnhandledPacketsKept = conf.getReceivedPacketsBufferSize(); + + LOG.info("Using {} thread(s) to handle incoming AIS packets", Runtime.getRuntime().availableProcessors() / 2); LOG.info("Message buffer size initialized with value [{}]", getMessageBufferSize()); //Creating up data handler @@ -118,25 +135,16 @@ public void setDataHandler(ICoverageData dataHandler) { public void receiveUnfiltered(AisPacket packet) { unfiltCount++; - - //extract relevant information from packet - CustomMessage message = dataHandler.packetToCustomMessage(packet); - if (message == null) { + + final int numberOfUnhandledPackets = unhandledPackets.size(); + if (numberOfUnhandledPackets >= maximumUnhandledPacketsKept) { + overflowLogger.log("Received AIS packets buffer overflow: " + numberOfUnhandledPackets + " currently unhandled packets"); return; } - - String key = message.getKey(); - - //Add to doublet buffer. - CustomMessage existing = doubletBuffer.get(key); - if (existing == null) { - doubletBuffer.put(key, message); - }else{ - if (message.getSourceList().iterator().hasNext()) { - existing.addSourceMMSI(message.getSourceList().iterator().next()); - } - } + unhandledPackets.add(packet); + + packetHandlingThreadPool.submit(new AisPacketHandler()); } void process(CustomMessage m){ @@ -195,4 +203,48 @@ public void run() { public SatCalculator getSatCalc() { return (SatCalculator) calculators.get(1); } + + public void stop() { + packetHandlingThreadPool.shutdown(); + + try { + if (!packetHandlingThreadPool.awaitTermination(60, TimeUnit.SECONDS)) { + packetHandlingThreadPool.shutdownNow(); + if (!packetHandlingThreadPool.awaitTermination(60, TimeUnit.SECONDS)) { + LOG.warn("Pool did not terminate"); + } + } + } catch (InterruptedException ie) { + packetHandlingThreadPool.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + + private class AisPacketHandler implements Callable { + + @Override + public Void call() throws Exception { + AisPacket packet = unhandledPackets.poll(); + + //extract relevant information from packet + CustomMessage message = dataHandler.packetToCustomMessage(packet); + if (message == null) { + return null; + } + + String key = message.getKey(); + + //Add to doublet buffer. + CustomMessage existing = doubletBuffer.get(key); + if (existing == null) { + doubletBuffer.put(key, message); + } else { + if (message.getSourceList().iterator().hasNext()) { + existing.addSourceMMSI(message.getSourceList().iterator().next()); + } + } + + return null; + } + } } diff --git a/src/main/java/dk/dma/ais/coverage/configuration/AisCoverageConfiguration.java b/src/main/java/dk/dma/ais/coverage/configuration/AisCoverageConfiguration.java index 5132c21..983170b 100644 --- a/src/main/java/dk/dma/ais/coverage/configuration/AisCoverageConfiguration.java +++ b/src/main/java/dk/dma/ais/coverage/configuration/AisCoverageConfiguration.java @@ -47,6 +47,7 @@ public class AisCoverageConfiguration { private DatabaseConfiguration dbConf = new DatabaseConfiguration(); private Map sourcenames = new HashMap(); private int windowSize = 5; + private int receivedPacketsBufferSize = 10000; public Map getSourceNameMap() { return sourcenames; @@ -146,4 +147,11 @@ public void setWindowSize(int windowSize) { this.windowSize = windowSize; } + public int getReceivedPacketsBufferSize() { + return receivedPacketsBufferSize; + } + + public void setReceivedPacketsBufferSize(int receivedPacketsBufferSize) { + this.receivedPacketsBufferSize = receivedPacketsBufferSize; + } } diff --git a/src/test/java/dk/dma/ais/coverage/configuration/AisCoverageConfigurationTest.java b/src/test/java/dk/dma/ais/coverage/configuration/AisCoverageConfigurationTest.java index 5c452c0..ec3006a 100644 --- a/src/test/java/dk/dma/ais/coverage/configuration/AisCoverageConfigurationTest.java +++ b/src/test/java/dk/dma/ais/coverage/configuration/AisCoverageConfigurationTest.java @@ -14,4 +14,11 @@ public void whenNewInstance_thenMessageBufferSizeDefaultsTo10000() { assertThat(configuration.getMessageBufferSize(), is(equalTo(10000))); } + + @Test + public void whenNewInstance_thenReceivedPacketsBufferSizeDefaultsTo10000() { + AisCoverageConfiguration configuration = new AisCoverageConfiguration(); + + assertThat(configuration.getReceivedPacketsBufferSize(), is(equalTo(10000))); + } } From 50802fae4b19160a70973c4974e5d7446a046424 Mon Sep 17 00:00:00 2001 From: John Martel Date: Mon, 28 Aug 2017 01:16:02 -0400 Subject: [PATCH 59/59] Bump version number --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7463944..faea29d 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ dk.dma.ais ais-coverage - 0.5.0-CCG + 0.6.0-CCG AIS coverage analyzer AIS coverage analyzer