Feature: alarm when over data plan

alarm/Alarm.js

@@ -603,6 +603,27 @@ class AbnormalBandwidthUsageAlarm extends Alarm {
     return fc.getTimingConfig("alarm.abnormal_bandwidth_usage.cooldown") || 60 * 60 * 4
   }
 }
+class OverDataPlanUsageAlarm extends Alarm{
+  constructor(timestamp, device, info) {
+    super("ALARM_OVER_DATA_PLAN_USAGE", timestamp, device, info);
+  }
+  isDup(alarm) {
+    let alarm2 = this;
+    if(alarm.type !== alarm2.type) {
+      return false;
+    }
+    if(alarm['p.monthly.endts'] != alarm2['p.monthly.endts'] || alarm['p.alarm.level'] != alarm2['p.alarm.level']){
+      return false;
+    }
+    return true;
+  }
+  getExpirationTime() {
+    return fc.getTimingConfig("alarm.data_plan.cooldown") || 60 * 60 * 24 * 30
+  }
+  requiredKeys(){
+    return [];
+  }
+}
 
 class LargeTransferAlarm extends OutboundAlarm {
   constructor(timestamp, device, destID, info) {
@@ -773,6 +794,7 @@ let classMapping = {
   ALARM_GAME: GameAlarm.prototype,
   ALARM_LARGE_UPLOAD: LargeTransferAlarm.prototype,
   ALARM_ABNORMAL_BANDWIDTH_USAGE: AbnormalBandwidthUsageAlarm.prototype,
+  ALARM_OVER_DATA_PLAN_USAGE: OverDataPlanUsageAlarm.prototype,
   ALARM_NEW_DEVICE: NewDeviceAlarm.prototype,
   ALARM_DEVICE_BACK_ONLINE: DeviceBackOnlineAlarm.prototype,
   ALARM_DEVICE_OFFLINE: DeviceOfflineAlarm.prototype,
@@ -794,6 +816,7 @@ module.exports = {
   PornAlarm: PornAlarm,
   LargeTransferAlarm: LargeTransferAlarm,
   AbnormalBandwidthUsageAlarm: AbnormalBandwidthUsageAlarm,
+  OverDataPlanUsageAlarm: OverDataPlanUsageAlarm,
   NewDeviceAlarm: NewDeviceAlarm,
   DeviceBackOnlineAlarm: DeviceBackOnlineAlarm,
   DeviceOfflineAlarm: DeviceOfflineAlarm,

controllers/netbot.js

@@ -1180,6 +1180,22 @@ class netBot extends ControllerBot {
           this.simpleTxData(msg, {}, err, callback);
         });
         break;
+      case "dataPlan":
+        (async () => {
+          const { total, date, enable } = value;
+          const featureName = 'data_plan';
+          if (enable) {
+            await fc.enableDynamicFeature(featureName)
+            await rclient.setAsync("sys:data:plan", JSON.stringify({ total: total, date: date }));
+          } else {
+            await fc.disableDynamicFeature(featureName);
+            await rclient.delAsync("sys:data:plan");
+          }
+          this.simpleTxData(msg, {}, null, callback);
+        })().catch((err) => {
+          this.simpleTxData(msg, {}, err, callback);
+        });
+        break;
       default:
         this.simpleTxData(msg, null, new Error("Unsupported set action"), callback);
         break;

intel/DestInfoIntel.js

@@ -43,6 +43,12 @@ class DestInfoIntel extends Intel {
     if (alarm["p.transfer.inbound.size"]) {
       alarm["p.transfer.inbound.humansize"] = formatBytes(alarm["p.transfer.inbound.size"]);
     }
+    if (alarm["p.totalUsage"]) {
+      alarm["p.totalUsage.humansize"] = formatBytes(alarm["p.totalUsage"]);
+    }
+    if (alarm["p.planUsage"]) {
+      alarm["p.planUsage.humansize"] = formatBytes(alarm["p.planUsage"]);
+    }
 
     let destIP = alarm["p.dest.ip"];
 

locales/en.json

@@ -67,23 +67,23 @@
   "NOTIF_ALARM_INTEL_REPORT": "Firewalla blocked {{{p.attempts}}} attack attempts to your network yesterday",
   "NOTIF_ALARM_INTEL_REPORT_N": "In last 24 hours, auto prevented {{{p.firstDomain}}} and {{{num_of_domains}}} sites from attacking {{{p.device.name}}}",
   "NOTIF_TITLE_ALARM_BRO_NOTICE": "Security Alarm",
-  "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_AUTOBLOCK":     "Auto blocked a TLS Heartbleed attack on device {{{p.device.name}}}",
-  "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack":                   "Detected a TLS Heartbleed attack on device {{{p.device.name}}}",
+  "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_AUTOBLOCK": "Auto blocked a TLS Heartbleed attack on device {{{p.device.name}}}",
+  "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack": "Detected a TLS Heartbleed attack on device {{{p.device.name}}}",
   "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success_AUTOBLOCK": "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}, auto blocked for further damage",
-  "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success":           "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}",
+  "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success": "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}",
   "NOTIF_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match_AUTOBLOCK": "Auto blocked a malicious file transfer on device {{{p.device.name}}}",
-  "NOTIF_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match":               "Detected a malicious file transfer on device {{{p.device.name}}}",
+  "NOTIF_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match": "Detected a malicious file transfer on device {{{p.device.name}}}",
   "NOTIF_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim_AUTOBLOCK": "Auto blocked a SQL injection attack on device {{{p.device.name}}}",
-  "NOTIF_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim":               "Detected a SQL injection attack on device {{{p.device.name}}}",
+  "NOTIF_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim": "Detected a SQL injection attack on device {{{p.device.name}}}",
   "NOTIF_ALARM_BRO_NOTICE": "{{{p.message}}}",
-  "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_AUTOBLOCK":     "Auto blocked a TLS Heartbleed attack on device {{{p.device.name}}}",
-  "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack":                   "Detected a TLS Heartbleed attack on device {{{p.device.name}}}",
+  "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_AUTOBLOCK": "Auto blocked a TLS Heartbleed attack on device {{{p.device.name}}}",
+  "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack": "Detected a TLS Heartbleed attack on device {{{p.device.name}}}",
   "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success_AUTOBLOCK": "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}, auto blocked for further damage",
-  "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success":           "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}",
+  "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success": "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}",
   "INFO_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match_AUTOBLOCK": "Auto blocked a malicious file transfer on device {{{p.device.name}}}",
-  "INFO_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match":               "Detected a malicious file transfer on device {{{p.device.name}}}",
+  "INFO_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match": "Detected a malicious file transfer on device {{{p.device.name}}}",
   "INFO_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim_AUTOBLOCK": "Auto blocked a SQL injection attack to device {{{p.device.name}}}",
-  "INFO_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim":               "Detected a SQL injection attack to device {{{p.device.name}}}",
+  "INFO_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim": "Detected a SQL injection attack to device {{{p.device.name}}}",
   "INFO_ALARM_BRO_NOTICE": "{{{p.message}}}",
   "NOTIF_TITLE_ALARM_SUBNET": "Router Configuration Warning",
   "NOTIF_ALARM_SUBNET": "An configuration problem of your router is found",
@@ -100,5 +100,7 @@
   "NOTIF_VPN_CLIENT_LINK_BROKEN_TITLE": "VPN Disconnected",
   "NOTIF_VPN_CLIENT_LINK_BROKEN_BODY": "VPN {{{profileId}}} is disconnected unexpectedly.",
   "NOTIF_TITLE_ALARM_ABNORMAL_BANDWIDTH_USAGE": "Abnormal Bandwidth Usage",
-  "NOTIF_ALARM_ABNORMAL_BANDWIDTH_USAGE": "Device {{{p.device.name}}} is transfering too much data from sites {{{p.dest.names}}} and etc."
-}
+  "NOTIF_ALARM_ABNORMAL_BANDWIDTH_USAGE": "Device {{{p.device.name}}} is using unusual amount of bandwidth: {{{p.totalUsage.humansize}}} in the last {{{p.duration}}} hours ({{{p.percentage}}} of total bandwidth usage).",
+  "NOTIF_TITLE_ALARM_OVER_DATA_PLAN_USAGE": "Bandwidth usage alarm",
+  "NOTIF_ALARM_OVER_DATA_PLAN_USAGE": "Your network has used up {{{p.percentage}}} of monthly planned bandwidth. Used {{{p.totalUsage.humansize}}}, Planned {{{p.planUsage.humansize}}}."
+}

net2/HostManager.js

@@ -315,9 +315,29 @@ module.exports = class HostManager {
     let uploadStats = await getHitsAsync("upload", "1minute", 60)
     return { downloadStats, uploadStats }
   }
-  async monthlyDataStats(mac) {
+  async monthlyDataStats(mac, date) {
     //default calender month
-    const days = new Date().getDate();
+    const now = new Date();
+    let days = now.getDate();
+    const month = now.getMonth(),
+      year = now.getFullYear(),
+      lastMonthDays = new Date(year, month, 0).getDate(),
+      currentMonthDays = new Date(year, month + 1, 0).getDate();
+    let monthlyBeginTs, monthlyEndTs;
+    if (date && date != 1) {
+      if (days < date) {
+        days = lastMonthDays - date + 1 + days;
+        monthlyBeginTs = new Date(year, month - 1, date);
+        monthlyEndTs = new Date(year, month, date);
+      } else {
+        days = days - date + 1;
+        monthlyBeginTs = new Date(year, month, date);
+        monthlyEndTs = new Date(year, month + 1, date);
+      }
+    } else {
+      monthlyBeginTs = new Date(year, month, 1);
+      monthlyEndTs = new Date(year, month + 1, 1);
+    }
     const downloadKey = `download${mac ? ':' + mac : ''}`;
     const uploadKey = `upload${mac ? ':' + mac : ''}`;
     const downloadStats = await getHitsAsync(downloadKey, '1day', days) || [];
@@ -333,7 +353,9 @@ module.exports = class HostManager {
       downloadStats: downloadStats,
       uploadStats: uploadStats,
       totalDownload: totalDownload,
-      totalUpload: totalUpload
+      totalUpload: totalUpload,
+      monthlyBeginTs: monthlyBeginTs / 1000,
+      monthlyEndTs: monthlyEndTs / 1000
     }
   }
 

net2/config.json

@@ -31,6 +31,7 @@
     "alarm.cooldown": 900,
     "alarm.large_upload.cooldown": 14400,
     "alarm.abnormal_bandwidth_usage.cooldown": 14400,
+    "alarm.data_plan.cooldown": 2592000,
     "alarm.vpn_client_connection.cooldown": 14400,
     "alarm.upnp.cooldown": 604800,
     "alarm.subnet.cooldown": 2592000
@@ -499,13 +500,14 @@
     "WirelessInterfaceSensor": {},
     "DataUsageSensor": {
       "refreshInterval": 15,
-      "ratio": 2,
+      "ratio": 1.2,
       "percentage": 0.8,
       "analytics_hours": 8,
       "smWindow": 2,
       "mdWindow": 8,
-      "topXflows": 2,
-      "minsize": 100000000
+      "topXflows": 10,
+      "minsize": 150000000,
+      "dataPlanMinPercentage": 0.8
     }
   },
   "hooks": {
@@ -592,6 +594,7 @@
     "game": true,
     "large_upload": true,
     "abnormal_bandwidth_usage": true,
+    "data_plan": true,
     "new_device": true,
     "new_device_block": false,
     "device_online": true,

sensor/DataUsageSensor.js

@@ -28,27 +28,30 @@ const flowTool = new FlowTool();
 const Alarm = require('../alarm/Alarm.js');
 const AlarmManager2 = require('../alarm/AlarmManager2.js');
 const alarmManager2 = new AlarmManager2();
-const featureName = 'abnormal_bandwidth_usage';
+const abnormalBandwidthUsageFeatureName = 'abnormal_bandwidth_usage';
+const dataPlanFeatureName = 'data_plan';
+const rclient = require('../util/redis_manager.js').getRedisClient();
+const fc = require('../net2/config.js');
 class DataUsageSensor extends Sensor {
     constructor() {
         super();
     }
     run() {
-        //todo add policy for per device data usage monitor or system
         this.refreshInterval = (this.config.refreshInterval || 15) * 60 * 1000;
-        this.ratio = this.config.ratio || 2;
+        this.ratio = this.config.ratio || 1.2;
         this.analytics_hours = this.config.analytics_hours || 8;
         this.percentage = this.config.percentage || 0.8;
-        this.topXflows = this.config.topXflows || 2;
-        this.minsize = this.config.minsize || 100 * 1000 * 1000;
+        this.topXflows = this.config.topXflows || 10;
+        this.minsize = this.config.minsize || 150 * 1000 * 1000;
         this.smWindow = this.config.smWindow || 2;
         this.mdWindow = this.config.mdWindow || 8;
+        this.dataPlanMinPercentage = this.config.dataPlanMinPercentage || 0.8;
         this.slot = 4// 1hour 4 slots
-        this.hookFeature(featureName);
+        this.hookFeature();
     }
     job() {
-        this.checkDataUsage()
-        this.checkMonthlyDataUsage()
+        fc.isFeatureOn(abnormalBandwidthUsageFeatureName) && this.checkDataUsage()
+        fc.isFeatureOn(dataPlanFeatureName) && this.checkMonthlyDataUsage()
     }
     globalOn() {
     }
@@ -68,12 +71,18 @@ class DataUsageSensor extends Sensor {
             const dataUsageMdHourWindow = await this.getTimewindowDataUsage(this.mdWindow, mac);
             const hostRecentlyTotalUsage = this.getRecentlyDataUsage(dataUsage, this.smWindow * this.slot)
             const hostDataUsagePercentage = hostRecentlyTotalUsage / systemRecentlyTotalUsage || 0;
-            const begin = dataUsage[0].ts, end = dataUsage[dataUsage.length - 1].ts;
-            for (let i = 0; i < dataUsageSmHourWindow.length; i++) {
-                if (dataUsageSmHourWindow[i].count > this.minsize && dataUsageMdHourWindow[i].count > this.minsize) {
-                    const ratio = dataUsageSmHourWindow[i].count / dataUsageMdHourWindow[i].count;
-                    if (ratio > this.ratio && hostDataUsagePercentage > this.percentage) {
-                        this.genAbnormalBandwidthUsageAlarm(host, begin, end, hostRecentlyTotalUsage, dataUsage);
+            const end = dataUsage[dataUsage.length - 1].ts;
+            const begin = end - this.smWindow * 60 * 60;
+            const steps = this.smWindow * this.slot;
+            const length = dataUsageSmHourWindow.length;
+            if (hostRecentlyTotalUsage < steps * this.minsize || hostDataUsagePercentage < this.percentage) continue;
+            for (let i = 1; i <= steps; i++) {
+                const smUsage = dataUsageSmHourWindow[length - i].count,
+                    mdUsage = dataUsageMdHourWindow[length - i].count;
+                if (smUsage > this.minsize && mdUsage > this.minsize && smUsage > mdUsage) {
+                    const ratio = smUsage / mdUsage;
+                    if (ratio > this.ratio) {
+                        this.genAbnormalBandwidthUsageAlarm(host, begin, end, hostRecentlyTotalUsage, dataUsage, hostDataUsagePercentage);
                         break;
                     }
                 }
@@ -92,7 +101,7 @@ class DataUsageSensor extends Sensor {
         const uploadStats = await getHitsAsync(uploadKey, "15minutes", analytics_slots);
         let dataUsageTimeWindow = [];
         if (downloadStats.length < slots) return;
-        for (let i = slots - 1; i < downloadStats.length; i++) {
+        for (let i = slots; i < downloadStats.length; i++) {
             let temp = {
                 count: 0,
                 ts: downloadStats[i][0]
@@ -115,13 +124,14 @@ class DataUsageSensor extends Sensor {
         }
         return total;
     }
-    async genAbnormalBandwidthUsageAlarm(host, begin, end, totalUsage, dataUsage) {
+    async genAbnormalBandwidthUsageAlarm(host, begin, end, totalUsage, dataUsage, percentage) {
         log.info("genAbnormalBandwidthUsageAlarm", host.o.mac, begin, end)
         //get top flows from begin to end
         const mac = host.o.mac;
         const name = host.o.name || host.o.bname;
         const flows = await this.getSumFlows(mac, begin, end);
         const destNames = flows.map((flow) => flow.aggregationHost).join(',')
+        percentage = percentage * 100;
         let alarm = new Alarm.AbnormalBandwidthUsageAlarm(new Date() / 1000, name, {
             "p.device.mac": mac,
             "p.device.id": name,
@@ -132,7 +142,9 @@ class DataUsageSensor extends Sensor {
             "p.end.ts": end,
             "e.transfers": dataUsage,
             "p.flows": JSON.stringify(flows),
-            "p.dest.names": destNames
+            "p.dest.names": destNames,
+            "p.duration": this.smWindow,
+            "p.percentage": percentage.toFixed(2) + '%'
         });
         await alarmManager2.enqueueAlarm(alarm);
     }
@@ -170,12 +182,25 @@ class DataUsageSensor extends Sensor {
         })
     }
     async checkMonthlyDataUsage() {
-        //data plan 1TB,10TB, etc..
-        //monthly? 11.01-11.30 or 11.05 - 12.05
-        const dataPlan = '';
-        const { totalDownload, totalUpload } = await hostManager.monthlyDataStats();
-        if (totalDownload + totalUpload > dataPlan) {
+        log.info("Start check monthly data usage")
+        let dataPlan = await rclient.getAsync('sys:data:plan');
+        if (!dataPlan) return;
+        dataPlan = JSON.parse(dataPlan);
+        const { date, total } = dataPlan;
+        const { totalDownload, totalUpload, monthlyBeginTs, monthlyEndTs } = await hostManager.monthlyDataStats(null, date);
+        let percentage = ((totalDownload + totalUpload) / total)
+        if (percentage >= this.dataPlanMinPercentage) {
             //gen over data plan alarm
+            const level = Math.floor(percentage * 10);
+            percentage = percentage * 100;
+            let alarm = new Alarm.OverDataPlanUsageAlarm(new Date() / 1000, null, {
+                "p.monthly.endts": monthlyEndTs,
+                "p.percentage": percentage.toFixed(2) + '%',
+                "p.totalUsage": totalDownload + totalUpload,
+                "p.planUsage": total,
+                "p.alarm.level": level >= 10 ? 'over' : level
+            });
+            await alarmManager2.enqueueAlarm(alarm);
         }
     }
 }