From 69db56ebce7567396af9f345dd762be6a42ad249 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 10 Jul 2019 02:37:30 +0200 Subject: [PATCH 1/4] NAB: add new naive detector --- nab/detectors/naive/__init__.py | 0 nab/detectors/naive/naive_detector.py | 36 +++++++++++++++++++++++++++ run.py | 4 ++- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 nab/detectors/naive/__init__.py create mode 100644 nab/detectors/naive/naive_detector.py diff --git a/nab/detectors/naive/__init__.py b/nab/detectors/naive/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/nab/detectors/naive/naive_detector.py b/nab/detectors/naive/naive_detector.py new file mode 100644 index 000000000..9489698c4 --- /dev/null +++ b/nab/detectors/naive/naive_detector.py @@ -0,0 +1,36 @@ +# ---------------------------------------------------------------------- +# Copyright (C) 2015, Numenta, Inc. Unless you have an agreement +# with Numenta, Inc., for a separate license for this software code, the +# following terms and conditions apply: +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero Public License for more details. +# +# You should have received a copy of the GNU Affero Public License +# along with this program. If not, see http://www.gnu.org/licenses. +# +# http://numenta.org/licenses/ +# ---------------------------------------------------------------------- + +""" +This is implementation of the "naive forecast" is a baseline algorithm for time-series +forecasting. It repeats the last seen value. So `P(t+1) = P(t)`. +""" + +from nab.detectors.base import AnomalyDetector + + + +class NaiveDetector(AnomalyDetector): + + def handleRecord(self, inputData): + """The anomaly score is simply the last seen value""" + print(inputData) + anomalyScore = 0.5 + return (anomalyScore, ) diff --git a/run.py b/run.py index f4d34e1b6..fdff4baea 100755 --- a/run.py +++ b/run.py @@ -134,7 +134,7 @@ def main(args): type=str, default=["null", "numenta", "random", "bayesChangePt", "windowedGaussian", "expose", "relativeEntropy", - "earthgeckoSkyline"], + "earthgeckoSkyline", "naive"], help="Comma separated list of detector(s) to use, e.g. " "null,numenta") @@ -209,6 +209,8 @@ def main(args): if "earthgeckoSkyline" in args.detectors: from nab.detectors.earthgecko_skyline.earthgecko_skyline_detector import EarthgeckoSkylineDetector + if "naive" in args.detectors: + from nab.detectors.naive.naive_detector import NaiveDetector if args.skipConfirmation or checkInputs(args): main(args) From 9c0fc4f5aafba655e4a7d5d0e7d7c598c306b43d Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 10 Jul 2019 14:29:53 +0200 Subject: [PATCH 2/4] WIP implement new naive detector does random-walk forecasting, ie returns last seen value: P(t+1) = P(t) --- nab/detectors/naive/naive_detector.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/nab/detectors/naive/naive_detector.py b/nab/detectors/naive/naive_detector.py index 9489698c4..daca63faf 100644 --- a/nab/detectors/naive/naive_detector.py +++ b/nab/detectors/naive/naive_detector.py @@ -25,12 +25,22 @@ from nab.detectors.base import AnomalyDetector +class NaiveDetector(AnomalyDetector): -class NaiveDetector(AnomalyDetector): + def initialize(self): + super().initialize() + self.predicted = 0.0 #previous value, last seen def handleRecord(self, inputData): """The anomaly score is simply the last seen value""" - print(inputData) - anomalyScore = 0.5 - return (anomalyScore, ) + current = float(inputData["value"]) + inputData['predicted'] = self.predicted + anomalyScore = 1 #FIXME how compute anomaly score from predicted, current? + + ret = [anomalyScore, self.predicted] + self.predicted=current + return (ret) + + def getAdditionalHeaders(self): + return ['predicted'] From a4297c14a3afc875ce432a77cd686c0663ad6917 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 10 Jul 2019 16:57:29 +0200 Subject: [PATCH 3/4] Review: anomaly score fn for Naive detector thanks for the help @japillow ! Improved doc, --- config/thresholds.json | 16 +++++++- nab/detectors/naive/naive_detector.py | 59 +++++++++++++++++++++++---- results/final_results.json | 9 +++- 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/config/thresholds.json b/config/thresholds.json index 69127a41c..1f5d3307d 100644 --- a/config/thresholds.json +++ b/config/thresholds.json @@ -83,6 +83,20 @@ "threshold": 1.0 } }, + "naive": { + "reward_low_FN_rate": { + "score": -232.0, + "threshold": 1.1 + }, + "reward_low_FP_rate": { + "score": -116.0, + "threshold": 1.1 + }, + "standard": { + "score": -116.0, + "threshold": 1.1 + } + }, "null": { "reward_low_FN_rate": { "score": -232.0, @@ -209,4 +223,4 @@ "threshold": 1.0 } } -} +} \ No newline at end of file diff --git a/nab/detectors/naive/naive_detector.py b/nab/detectors/naive/naive_detector.py index daca63faf..0e235470c 100644 --- a/nab/detectors/naive/naive_detector.py +++ b/nab/detectors/naive/naive_detector.py @@ -18,29 +18,74 @@ # http://numenta.org/licenses/ # ---------------------------------------------------------------------- -""" -This is implementation of the "naive forecast" is a baseline algorithm for time-series -forecasting. It repeats the last seen value. So `P(t+1) = P(t)`. -""" +import math #exp from nab.detectors.base import AnomalyDetector +EPSILON = 0.00000001 + class NaiveDetector(AnomalyDetector): + """ + This is implementation of the "naive forecast", aka "random walk forecasting", + which is a baseline algorithm for time-series forecasting. + It predicts the last seen value. So `Prediction(t+1) = Input(t)`. + + Hyperparameter to optimize is @param coef in `initialize`. + """ - def initialize(self): + def initialize(self, coef=10.0): + """ + @param `coef` for the activation function that scales anomaly score to [0, 1.0] + The function is: `anomalyScore = 1-exp(-coef*x)`, where + `x=abs(current - predicted)/predicted`. + """ super().initialize() self.predicted = 0.0 #previous value, last seen + self.coef = coef + def handleRecord(self, inputData): - """The anomaly score is simply the last seen value""" + """The predicted value is simply the last seen value, + Anomaly score is computed as a function of current,predicted. + + See @ref `initialize` param `coef`. + """ current = float(inputData["value"]) inputData['predicted'] = self.predicted - anomalyScore = 1 #FIXME how compute anomaly score from predicted, current? + try: + anomalyScore = self.anomalyFn_(current, self.predicted) + except: + #on any math error (overflow,...), we mark this as anomaly. tough love + anomalyScore = 1.0 ret = [anomalyScore, self.predicted] self.predicted=current return (ret) + def getAdditionalHeaders(self): return ['predicted'] + + + def anomalyFn_(self, current, predicted): + """ + compute anomaly score from 2 scalars + """ + if predicted == 0.0: + predicted = EPSILON #avoid division by zero + + # the computation + x = abs(current - predicted)/predicted + score = 1-math.exp(-self.coef * x) + + # bound to anomaly range (should not happen, but some are over) + if(score > 1): + score = 1.0 + elif(score < 0): + score = 0.0 + + print(score) + assert(score >= 0 and score <= 1) + return score + diff --git a/results/final_results.json b/results/final_results.json index 8161ade25..4af27f94c 100644 --- a/results/final_results.json +++ b/results/final_results.json @@ -24,6 +24,11 @@ "reward_low_FP_rate": 43.40851703291233, "standard": 57.99434335898808 }, + "naive": { + "reward_low_FN_rate": 0.0, + "reward_low_FP_rate": 0.0, + "standard": 0.0 + }, "null": { "reward_low_FN_rate": 0.0, "reward_low_FP_rate": 0.0, @@ -35,9 +40,9 @@ "standard": 70.10105611943965 }, "numentaTM": { - "reward_low_FN_rate": 69.185068229060349, + "reward_low_FN_rate": 69.18506822906035, "reward_low_FP_rate": 56.665308225043105, - "standard": 64.553464412543107 + "standard": 64.55346441254311 }, "random": { "reward_low_FN_rate": 25.876351314080456, From ee7e5802b56e36468e450790c77d39d617c8ec9d Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Wed, 10 Jul 2019 17:59:47 +0200 Subject: [PATCH 4/4] Naive: debugging out of bounds scores --- nab/detectors/naive/naive_detector.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nab/detectors/naive/naive_detector.py b/nab/detectors/naive/naive_detector.py index 0e235470c..36fa2fbea 100644 --- a/nab/detectors/naive/naive_detector.py +++ b/nab/detectors/naive/naive_detector.py @@ -80,12 +80,11 @@ def anomalyFn_(self, current, predicted): score = 1-math.exp(-self.coef * x) # bound to anomaly range (should not happen, but some are over) - if(score > 1): - score = 1.0 - elif(score < 0): - score = 0.0 +# if(score > 1): +# score = 1.0 +# elif(score < 0): +# score = 0.0 - print(score) - assert(score >= 0 and score <= 1) + assert(score >= 0 and score <= 1), print("ERR: score: "+str(score)+ "\t curr: "+str(current)+"\t pred: "+str(predicted)) return score