diff --git a/alarm/Alarm.js b/alarm/Alarm.js index 6c3b4938c3..eb3db70275 100644 --- a/alarm/Alarm.js +++ b/alarm/Alarm.js @@ -603,6 +603,27 @@ class AbnormalBandwidthUsageAlarm extends Alarm { return fc.getTimingConfig("alarm.abnormal_bandwidth_usage.cooldown") || 60 * 60 * 4 } } +class OverDataPlanUsageAlarm extends Alarm{ + constructor(timestamp, device, info) { + super("ALARM_OVER_DATA_PLAN_USAGE", timestamp, device, info); + } + isDup(alarm) { + let alarm2 = this; + if(alarm.type !== alarm2.type) { + return false; + } + if(alarm['p.monthly.endts'] != alarm2['p.monthly.endts'] || alarm['p.alarm.level'] != alarm2['p.alarm.level']){ + return false; + } + return true; + } + getExpirationTime() { + return fc.getTimingConfig("alarm.data_plan.cooldown") || 60 * 60 * 24 * 30 + } + requiredKeys(){ + return []; + } +} class LargeTransferAlarm extends OutboundAlarm { constructor(timestamp, device, destID, info) { @@ -773,6 +794,7 @@ let classMapping = { ALARM_GAME: GameAlarm.prototype, ALARM_LARGE_UPLOAD: LargeTransferAlarm.prototype, ALARM_ABNORMAL_BANDWIDTH_USAGE: AbnormalBandwidthUsageAlarm.prototype, + ALARM_OVER_DATA_PLAN_USAGE: OverDataPlanUsageAlarm.prototype, ALARM_NEW_DEVICE: NewDeviceAlarm.prototype, ALARM_DEVICE_BACK_ONLINE: DeviceBackOnlineAlarm.prototype, ALARM_DEVICE_OFFLINE: DeviceOfflineAlarm.prototype, @@ -794,6 +816,7 @@ module.exports = { PornAlarm: PornAlarm, LargeTransferAlarm: LargeTransferAlarm, AbnormalBandwidthUsageAlarm: AbnormalBandwidthUsageAlarm, + OverDataPlanUsageAlarm: OverDataPlanUsageAlarm, NewDeviceAlarm: NewDeviceAlarm, DeviceBackOnlineAlarm: DeviceBackOnlineAlarm, DeviceOfflineAlarm: DeviceOfflineAlarm, diff --git a/controllers/netbot.js b/controllers/netbot.js index 25e335787f..35f787f029 100644 --- a/controllers/netbot.js +++ b/controllers/netbot.js @@ -1180,6 +1180,22 @@ class netBot extends ControllerBot { this.simpleTxData(msg, {}, err, callback); }); break; + case "dataPlan": + (async () => { + const { total, date, enable } = value; + const featureName = 'data_plan'; + if (enable) { + await fc.enableDynamicFeature(featureName) + await rclient.setAsync("sys:data:plan", JSON.stringify({ total: total, date: date })); + } else { + await fc.disableDynamicFeature(featureName); + await rclient.delAsync("sys:data:plan"); + } + this.simpleTxData(msg, {}, null, callback); + })().catch((err) => { + this.simpleTxData(msg, {}, err, callback); + }); + break; default: this.simpleTxData(msg, null, new Error("Unsupported set action"), callback); break; diff --git a/intel/DestInfoIntel.js b/intel/DestInfoIntel.js index 12b46e7804..d4f5809832 100644 --- a/intel/DestInfoIntel.js +++ b/intel/DestInfoIntel.js @@ -43,6 +43,12 @@ class DestInfoIntel extends Intel { if (alarm["p.transfer.inbound.size"]) { alarm["p.transfer.inbound.humansize"] = formatBytes(alarm["p.transfer.inbound.size"]); } + if (alarm["p.totalUsage"]) { + alarm["p.totalUsage.humansize"] = formatBytes(alarm["p.totalUsage"]); + } + if (alarm["p.planUsage"]) { + alarm["p.planUsage.humansize"] = formatBytes(alarm["p.planUsage"]); + } let destIP = alarm["p.dest.ip"]; diff --git a/locales/en.json b/locales/en.json index fbafe9533b..b46ec28fd3 100644 --- a/locales/en.json +++ b/locales/en.json @@ -67,23 +67,23 @@ "NOTIF_ALARM_INTEL_REPORT": "Firewalla blocked {{{p.attempts}}} attack attempts to your network yesterday", "NOTIF_ALARM_INTEL_REPORT_N": "In last 24 hours, auto prevented {{{p.firstDomain}}} and {{{num_of_domains}}} sites from attacking {{{p.device.name}}}", "NOTIF_TITLE_ALARM_BRO_NOTICE": "Security Alarm", - "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_AUTOBLOCK": "Auto blocked a TLS Heartbleed attack on device {{{p.device.name}}}", - "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack": "Detected a TLS Heartbleed attack on device {{{p.device.name}}}", + "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_AUTOBLOCK": "Auto blocked a TLS Heartbleed attack on device {{{p.device.name}}}", + "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack": "Detected a TLS Heartbleed attack on device {{{p.device.name}}}", "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success_AUTOBLOCK": "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}, auto blocked for further damage", - "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success": "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}", + "NOTIF_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success": "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}", "NOTIF_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match_AUTOBLOCK": "Auto blocked a malicious file transfer on device {{{p.device.name}}}", - "NOTIF_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match": "Detected a malicious file transfer on device {{{p.device.name}}}", + "NOTIF_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match": "Detected a malicious file transfer on device {{{p.device.name}}}", "NOTIF_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim_AUTOBLOCK": "Auto blocked a SQL injection attack on device {{{p.device.name}}}", - "NOTIF_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim": "Detected a SQL injection attack on device {{{p.device.name}}}", + "NOTIF_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim": "Detected a SQL injection attack on device {{{p.device.name}}}", "NOTIF_ALARM_BRO_NOTICE": "{{{p.message}}}", - "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_AUTOBLOCK": "Auto blocked a TLS Heartbleed attack on device {{{p.device.name}}}", - "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack": "Detected a TLS Heartbleed attack on device {{{p.device.name}}}", + "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_AUTOBLOCK": "Auto blocked a TLS Heartbleed attack on device {{{p.device.name}}}", + "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack": "Detected a TLS Heartbleed attack on device {{{p.device.name}}}", "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success_AUTOBLOCK": "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}, auto blocked for further damage", - "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success": "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}", + "INFO_ALARM_BRO_NOTICE_Heartbleed::SSL_Heartbeat_Attack_Success": "Detected a successful TLS Heartbleed attack on device {{{p.device.name}}}", "INFO_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match_AUTOBLOCK": "Auto blocked a malicious file transfer on device {{{p.device.name}}}", - "INFO_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match": "Detected a malicious file transfer on device {{{p.device.name}}}", + "INFO_ALARM_BRO_NOTICE_TeamCymruMalwareHashRegistry::Match": "Detected a malicious file transfer on device {{{p.device.name}}}", "INFO_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim_AUTOBLOCK": "Auto blocked a SQL injection attack to device {{{p.device.name}}}", - "INFO_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim": "Detected a SQL injection attack to device {{{p.device.name}}}", + "INFO_ALARM_BRO_NOTICE_HTTP::SQL_Injection_Victim": "Detected a SQL injection attack to device {{{p.device.name}}}", "INFO_ALARM_BRO_NOTICE": "{{{p.message}}}", "NOTIF_TITLE_ALARM_SUBNET": "Router Configuration Warning", "NOTIF_ALARM_SUBNET": "An configuration problem of your router is found", @@ -100,5 +100,7 @@ "NOTIF_VPN_CLIENT_LINK_BROKEN_TITLE": "VPN Disconnected", "NOTIF_VPN_CLIENT_LINK_BROKEN_BODY": "VPN {{{profileId}}} is disconnected unexpectedly.", "NOTIF_TITLE_ALARM_ABNORMAL_BANDWIDTH_USAGE": "Abnormal Bandwidth Usage", - "NOTIF_ALARM_ABNORMAL_BANDWIDTH_USAGE": "Device {{{p.device.name}}} is transfering too much data from sites {{{p.dest.names}}} and etc." -} + "NOTIF_ALARM_ABNORMAL_BANDWIDTH_USAGE": "Device {{{p.device.name}}} is using unusual amount of bandwidth: {{{p.totalUsage.humansize}}} in the last {{{p.duration}}} hours ({{{p.percentage}}} of total bandwidth usage).", + "NOTIF_TITLE_ALARM_OVER_DATA_PLAN_USAGE": "Bandwidth usage alarm", + "NOTIF_ALARM_OVER_DATA_PLAN_USAGE": "Your network has used up {{{p.percentage}}} of monthly planned bandwidth. Used {{{p.totalUsage.humansize}}}, Planned {{{p.planUsage.humansize}}}." +} \ No newline at end of file diff --git a/net2/HostManager.js b/net2/HostManager.js index 9280091541..dbbf1a7eaf 100644 --- a/net2/HostManager.js +++ b/net2/HostManager.js @@ -315,9 +315,29 @@ module.exports = class HostManager { let uploadStats = await getHitsAsync("upload", "1minute", 60) return { downloadStats, uploadStats } } - async monthlyDataStats(mac) { + async monthlyDataStats(mac, date) { //default calender month - const days = new Date().getDate(); + const now = new Date(); + let days = now.getDate(); + const month = now.getMonth(), + year = now.getFullYear(), + lastMonthDays = new Date(year, month, 0).getDate(), + currentMonthDays = new Date(year, month + 1, 0).getDate(); + let monthlyBeginTs, monthlyEndTs; + if (date && date != 1) { + if (days < date) { + days = lastMonthDays - date + 1 + days; + monthlyBeginTs = new Date(year, month - 1, date); + monthlyEndTs = new Date(year, month, date); + } else { + days = days - date + 1; + monthlyBeginTs = new Date(year, month, date); + monthlyEndTs = new Date(year, month + 1, date); + } + } else { + monthlyBeginTs = new Date(year, month, 1); + monthlyEndTs = new Date(year, month + 1, 1); + } const downloadKey = `download${mac ? ':' + mac : ''}`; const uploadKey = `upload${mac ? ':' + mac : ''}`; const downloadStats = await getHitsAsync(downloadKey, '1day', days) || []; @@ -333,7 +353,9 @@ module.exports = class HostManager { downloadStats: downloadStats, uploadStats: uploadStats, totalDownload: totalDownload, - totalUpload: totalUpload + totalUpload: totalUpload, + monthlyBeginTs: monthlyBeginTs / 1000, + monthlyEndTs: monthlyEndTs / 1000 } } diff --git a/net2/config.json b/net2/config.json index fdf18aa531..7f3f63d87b 100644 --- a/net2/config.json +++ b/net2/config.json @@ -31,6 +31,7 @@ "alarm.cooldown": 900, "alarm.large_upload.cooldown": 14400, "alarm.abnormal_bandwidth_usage.cooldown": 14400, + "alarm.data_plan.cooldown": 2592000, "alarm.vpn_client_connection.cooldown": 14400, "alarm.upnp.cooldown": 604800, "alarm.subnet.cooldown": 2592000 @@ -499,13 +500,14 @@ "WirelessInterfaceSensor": {}, "DataUsageSensor": { "refreshInterval": 15, - "ratio": 2, + "ratio": 1.2, "percentage": 0.8, "analytics_hours": 8, "smWindow": 2, "mdWindow": 8, - "topXflows": 2, - "minsize": 100000000 + "topXflows": 10, + "minsize": 150000000, + "dataPlanMinPercentage": 0.8 } }, "hooks": { @@ -592,6 +594,7 @@ "game": true, "large_upload": true, "abnormal_bandwidth_usage": true, + "data_plan": true, "new_device": true, "new_device_block": false, "device_online": true, diff --git a/sensor/DataUsageSensor.js b/sensor/DataUsageSensor.js index 8de14ad764..c21a072441 100644 --- a/sensor/DataUsageSensor.js +++ b/sensor/DataUsageSensor.js @@ -28,27 +28,30 @@ const flowTool = new FlowTool(); const Alarm = require('../alarm/Alarm.js'); const AlarmManager2 = require('../alarm/AlarmManager2.js'); const alarmManager2 = new AlarmManager2(); -const featureName = 'abnormal_bandwidth_usage'; +const abnormalBandwidthUsageFeatureName = 'abnormal_bandwidth_usage'; +const dataPlanFeatureName = 'data_plan'; +const rclient = require('../util/redis_manager.js').getRedisClient(); +const fc = require('../net2/config.js'); class DataUsageSensor extends Sensor { constructor() { super(); } run() { - //todo add policy for per device data usage monitor or system this.refreshInterval = (this.config.refreshInterval || 15) * 60 * 1000; - this.ratio = this.config.ratio || 2; + this.ratio = this.config.ratio || 1.2; this.analytics_hours = this.config.analytics_hours || 8; this.percentage = this.config.percentage || 0.8; - this.topXflows = this.config.topXflows || 2; - this.minsize = this.config.minsize || 100 * 1000 * 1000; + this.topXflows = this.config.topXflows || 10; + this.minsize = this.config.minsize || 150 * 1000 * 1000; this.smWindow = this.config.smWindow || 2; this.mdWindow = this.config.mdWindow || 8; + this.dataPlanMinPercentage = this.config.dataPlanMinPercentage || 0.8; this.slot = 4// 1hour 4 slots - this.hookFeature(featureName); + this.hookFeature(); } job() { - this.checkDataUsage() - this.checkMonthlyDataUsage() + fc.isFeatureOn(abnormalBandwidthUsageFeatureName) && this.checkDataUsage() + fc.isFeatureOn(dataPlanFeatureName) && this.checkMonthlyDataUsage() } globalOn() { } @@ -68,12 +71,18 @@ class DataUsageSensor extends Sensor { const dataUsageMdHourWindow = await this.getTimewindowDataUsage(this.mdWindow, mac); const hostRecentlyTotalUsage = this.getRecentlyDataUsage(dataUsage, this.smWindow * this.slot) const hostDataUsagePercentage = hostRecentlyTotalUsage / systemRecentlyTotalUsage || 0; - const begin = dataUsage[0].ts, end = dataUsage[dataUsage.length - 1].ts; - for (let i = 0; i < dataUsageSmHourWindow.length; i++) { - if (dataUsageSmHourWindow[i].count > this.minsize && dataUsageMdHourWindow[i].count > this.minsize) { - const ratio = dataUsageSmHourWindow[i].count / dataUsageMdHourWindow[i].count; - if (ratio > this.ratio && hostDataUsagePercentage > this.percentage) { - this.genAbnormalBandwidthUsageAlarm(host, begin, end, hostRecentlyTotalUsage, dataUsage); + const end = dataUsage[dataUsage.length - 1].ts; + const begin = end - this.smWindow * 60 * 60; + const steps = this.smWindow * this.slot; + const length = dataUsageSmHourWindow.length; + if (hostRecentlyTotalUsage < steps * this.minsize || hostDataUsagePercentage < this.percentage) continue; + for (let i = 1; i <= steps; i++) { + const smUsage = dataUsageSmHourWindow[length - i].count, + mdUsage = dataUsageMdHourWindow[length - i].count; + if (smUsage > this.minsize && mdUsage > this.minsize && smUsage > mdUsage) { + const ratio = smUsage / mdUsage; + if (ratio > this.ratio) { + this.genAbnormalBandwidthUsageAlarm(host, begin, end, hostRecentlyTotalUsage, dataUsage, hostDataUsagePercentage); break; } } @@ -92,7 +101,7 @@ class DataUsageSensor extends Sensor { const uploadStats = await getHitsAsync(uploadKey, "15minutes", analytics_slots); let dataUsageTimeWindow = []; if (downloadStats.length < slots) return; - for (let i = slots - 1; i < downloadStats.length; i++) { + for (let i = slots; i < downloadStats.length; i++) { let temp = { count: 0, ts: downloadStats[i][0] @@ -115,13 +124,14 @@ class DataUsageSensor extends Sensor { } return total; } - async genAbnormalBandwidthUsageAlarm(host, begin, end, totalUsage, dataUsage) { + async genAbnormalBandwidthUsageAlarm(host, begin, end, totalUsage, dataUsage, percentage) { log.info("genAbnormalBandwidthUsageAlarm", host.o.mac, begin, end) //get top flows from begin to end const mac = host.o.mac; const name = host.o.name || host.o.bname; const flows = await this.getSumFlows(mac, begin, end); const destNames = flows.map((flow) => flow.aggregationHost).join(',') + percentage = percentage * 100; let alarm = new Alarm.AbnormalBandwidthUsageAlarm(new Date() / 1000, name, { "p.device.mac": mac, "p.device.id": name, @@ -132,7 +142,9 @@ class DataUsageSensor extends Sensor { "p.end.ts": end, "e.transfers": dataUsage, "p.flows": JSON.stringify(flows), - "p.dest.names": destNames + "p.dest.names": destNames, + "p.duration": this.smWindow, + "p.percentage": percentage.toFixed(2) + '%' }); await alarmManager2.enqueueAlarm(alarm); } @@ -170,12 +182,25 @@ class DataUsageSensor extends Sensor { }) } async checkMonthlyDataUsage() { - //data plan 1TB,10TB, etc.. - //monthly? 11.01-11.30 or 11.05 - 12.05 - const dataPlan = ''; - const { totalDownload, totalUpload } = await hostManager.monthlyDataStats(); - if (totalDownload + totalUpload > dataPlan) { + log.info("Start check monthly data usage") + let dataPlan = await rclient.getAsync('sys:data:plan'); + if (!dataPlan) return; + dataPlan = JSON.parse(dataPlan); + const { date, total } = dataPlan; + const { totalDownload, totalUpload, monthlyBeginTs, monthlyEndTs } = await hostManager.monthlyDataStats(null, date); + let percentage = ((totalDownload + totalUpload) / total) + if (percentage >= this.dataPlanMinPercentage) { //gen over data plan alarm + const level = Math.floor(percentage * 10); + percentage = percentage * 100; + let alarm = new Alarm.OverDataPlanUsageAlarm(new Date() / 1000, null, { + "p.monthly.endts": monthlyEndTs, + "p.percentage": percentage.toFixed(2) + '%', + "p.totalUsage": totalDownload + totalUpload, + "p.planUsage": total, + "p.alarm.level": level >= 10 ? 'over' : level + }); + await alarmManager2.enqueueAlarm(alarm); } } }