noursaidi · noursaidi · Jun 24, 2020 · Jun 25, 2020 · Jun 25, 2020 · Jun 25, 2020
diff --git a/.checkstyle.xml b/.checkstyle.xml
@@ -42,7 +42,7 @@
 
     <module name="LineLength">
         <property name="fileExtensions" value="java"/>
-        <property name="max" value="100"/>
+        <property name="max" value="120"/>
         <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
     </module>
 

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -0,0 +1,75 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: DAQ test suite
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 */2 * * *'
+
+jobs:
+  integration_tests:
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [base, many, aux, topo, modules, dhcp]
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Set up JDK 1.11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.11
+    - name: Installing dependencies
+      run: |
+        bin/setup_daq
+    - name: Running ${{ matrix.test }} test
+      env:
+        DOCKER_STARTUP_TIMEOUT_MS: 60000
+        GCP_BASE64_CRED: ${{ secrets.GCP_BASE64_CRED }}
+      run: |
+        bin/test_daq ${{ matrix.test }}
+    - name: Generated test report
+      if: ${{ always() }}
+      run: |
+        echo '************* Use sed to filter out timestamp prefix *************'
+        echo 'Download log archive, find the right job, and use:'
+        echo "  sed -e 's/^[-:.0-9TZ]\+ //' 7_Generated\ test\ report.txt"
+        cat inst/test_${{ matrix.test }}.out
+
+  unit_tests:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        bin/setup_dev
+    - name: Check style
+      run: |
+        bin/check_style
+    - name: Unit test
+      run: |
+        testing/run_unit_tests.sh
+
+  usi_tests:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.11
+    - name: Build with Maven
+      run: mvn -B clean compile test assembly:single --file usi/pom.xml
diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,9 @@ protoc-gen-doc/
 firebase/public/firebase_config.js
 firebase/public/deploy_version.js
 
+# Kleudgy apparmor workaround
+binhack/tcpdump
+
 validations/
 
 \#*
@@ -21,16 +24,15 @@ validations/
 *.save
 
 # Runtime or sub-module files
-inst/
-faucet/
-forch/
-mininet/
-local/
-local_xxx
-local.old
-firebase/.firebaserc
-firebase/.firebase
-firebase/functions/package-lock.json
+/inst/
+/faucet/
+/forch/
+/udmi/
+/mininet/
+/local/
+/firebase/.firebaserc
+/firebase/.firebase
+/firebase/functions/package-lock.json
 nohup.out
 **/node_modules/
 .vscode/

diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/.pylintrc b/.pylintrc
@@ -1,5 +1,7 @@
 [MASTER]
 good-names=fd,e
 output-format=parseable
-disable=F0401,R0201,R0902,W0703,I0011,W0403,E1102,R0903,E0611,W0511,W0613
+disable=F0401,R0201,R0902,W0703,I0011,W0403,E1102,R0903,E0611,W0511,W0613,E1601
 max-line-length=100
+ignored-classes=
+    OrchestrationConfig, DevicesState, PortBehavior
diff --git a/.stickler.yml b/.stickler.yml
@@ -7,8 +7,10 @@ linters:
     python: 3
     max-line-length: 100
   py3k:
+    ignore: E1601
   checkstyle:
     config: ./.checkstyle.xml
 files:
   ignore:
     - 'libs/proto/*'
+    - '*__init__.py'
diff --git a/.travis.yml b/.travis.yml
diff --git a/README.md b/README.md
@@ -1,3 +1,5 @@
+![CI Tests](https://github.com/faucetsdn/daq/workflows/DAQ%20test%20suite/badge.svg?branch=master)
+
 # DAQ: <b>D</b>evice <b>A</b>utomated <b>Q</b>ualification for IoT Devices.
 
 DAQ is a framework designed to test and operate IoT devices in an enterprise IoT environment.
@@ -17,10 +19,6 @@ More details about the goals and objectives behind this can be found in the IEEE
 [SDN capabilities](https://queue.acm.org/detail.cfm?id=2560327), such as the
 [FAUCET OpenFlow controller](https://faucet.nz/), to orchestrate "microsegmentation" on
 the network for improved security.
-* [_Universal Device Management Interface (UDMI)_](schemas/udmi/README.md): An interface
-specification designed to normalize the management of IoT devices from different manufacturers.
-This is a simple standard that provides for many of the common features not present in
-existing protocols (e.g. BACnet).
 * _Device Management Tools_: A suite of tools, consoles, and dashboards that help operate
 a robust ecosystem of IoT devices. (Details forthcoming.)
 

diff --git a/bin/alt_faucet b/bin/alt_faucet
@@ -0,0 +1,24 @@
+#!/bin/bash -e
+
+ROOT=$(realpath $(dirname $0)/..)
+cd $ROOT
+source etc/config_base.sh
+
+if [ -z "$switch_setup_ext_br" ]; then
+   echo switch_setup.ext_br not defined for alternate faucet setup.
+   false
+fi
+
+if [ -z "$switch_setup_alt_port" ]; then
+   echo switch_setup.alt_port not defined for alternate faucet setup.
+   false
+fi
+
+inst_name=$switch_setup_ext_br
+
+inst_dir=inst/faucet/daq-faucet-$inst_name
+mkdir -p $inst_dir
+cp config/faucet/faucet_$inst_name.yaml $inst_dir/faucet.yaml
+echo Launching alternate faucet install $inst_name on $switch_setup_alt_port
+echo DAQ autoclean docker kill daq-faucet-$inst_name
+cmd/faucet $inst_name $switch_setup_alt_port
diff --git a/bin/build_hash b/bin/build_hash
@@ -10,7 +10,7 @@ build_built=.build_built
 faucet_version=$(cd faucet; git rev-list -n 1 HEAD)
 echo "$faucet_version  faucet/HEAD" > $build_files
 
-find docker/ subset/ -type f | sort | xargs sha1sum >> $build_files
+find docker/ subset/ usi/ -type f | sort | xargs sha1sum >> $build_files
 build_hash=`cat $build_files | sha256sum | awk '{print $1}'`
 
 if [ "$1" == check ]; then

diff --git a/bin/build_proto b/bin/build_proto
@@ -32,6 +32,11 @@ sha1sum $proto_files > $WEB_ROOT/protos.hash
 
 gen_path=$ROOT/protoc-gen-doc/bin/protoc-gen-doc
 
+if [ -d venv ]; then
+    echo Entering virtual python environment...
+    source venv/bin/activate
+fi
+
 mkdir -p build/daq/proto build/proto
 cp $proto_files build/daq/proto/
 proto_files2=
@@ -56,3 +61,5 @@ mkdir -p libs/proto/
 touch libs/proto/__init__.py
 cp build/daq/proto/*.py libs/proto/
 cp build/protos.html $WEB_ROOT/
+
+python3 -m grpc_tools.protoc -I usi/src/main/proto/ --python_out=libs/proto/ --grpc_python_out=libs/proto/ usi/src/main/proto/usi.proto
diff --git a/bin/build_release b/bin/build_release
@@ -0,0 +1,75 @@
+#!/bin/bash -e
+
+if [ $# != 1 ]; then
+    echo $0 RELEASE_VERSION
+    false
+fi
+
+VERSION=$1
+shift
+
+ROOT=$(realpath $(dirname $0)/..)
+cd $ROOT
+
+changes=`git status --porcelain`
+if [ -n "$changes" ]; then
+    echo Working tree not clean.
+    false
+fi
+
+git checkout release_stable
+git fetch faucet
+git merge faucet/release_stable
+git checkout master
+
+changed=`git diff --name-only release_stable docs/changelog.md`
+if [ -z "$changed" ]; then
+    git log release_stable..HEAD --pretty=oneline | sed -e 's/^[a-z0-9]\+ //g'
+    echo docs/changelog.md has not been updated since last release_stable
+    echo Use the log lines above for inspiration.
+    false
+fi
+
+tagged=`git rev-list -n 1 $VERSION 2>/dev/null` || true
+if [ -n "$tagged" ]; then
+    echo Tag $VERSION already exists. Try the next version.
+    false
+fi
+
+source etc/config_base.sh
+
+if [ "$host_tests" != config/modules/all.conf ]; then
+    echo Configure your system with host_tests=config/modules/all.conf
+    false
+fi
+
+cmd/build force $VERSION
+
+cmd/build push
+
+git commit -a -m "$VERSION release"
+git tag -a $VERSION -m "$VERSION release"
+firebase/deploy.sh bos-daq-testing
+
+git push
+git push --tags
+
+# Check to see if a remote 'faucet' is defined, and if so, also update that.
+faucetgit=`git config remote.faucet.url`
+if [ -n "$faucetgit" ]; then
+    git push faucet
+    git push faucet --tags
+fi
+
+git checkout release_testing && git reset --hard $VERSION
+git push
+if [ -n "$faucetgit" ]; then
+    git push faucet
+fi
+
+git log -n 1
+
+# QA pass to make sure everything is ok.
+# `firebase/deploy.sh daq-qualification-labs`
+# `git checkout release_stable && git reset --hard $VERSION`
+# `git push`
diff --git a/bin/combine_reports b/bin/combine_reports
@@ -5,11 +5,4 @@ cd $ROOT
 
 source etc/config_base.sh
 
-FAILED=
-PYTHONPATH=daq python3 bin/python/combine_reports_from_date_range.py $conf_file $@ || FAILED=true
-
-if [ -n "$FAILED" ]; then
-    echo
-    echo Usage: $0 [from_time] [to_time] [from_gcp]
-    false
-fi
+PYTHONPATH=daq python3 bin/python/combine_reports.py $conf_file $@
diff --git a/bin/external_ovs b/bin/external_ovs
@@ -6,14 +6,19 @@ source etc/config_base.sh
 
 ext_intf=$switch_setup_data_intf
 ext_dpid=$switch_setup_of_dpid
-ext_ofpt=$switch_setup_lo_port
 ext_brid=$switch_setup_ext_br
 ext_brpt=$switch_setup_uplink_port
 
 ext_pri=${ext_intf}
 ext_sec=${ext_intf%-pri}-sec
 
-echo ext_dpid is $ext_dpid
+if [ -z "$switch_setup_alt_port" ]; then
+    ext_ofpt=$switch_setup_lo_port
+else
+    ext_ofpt=$switch_setup_alt_port
+fi
+
+echo ext_dpid is $ext_dpid on port $ext_ofpt
 echo network_config is $network_config
 
 dpid=$(printf %016x $ext_dpid)

diff --git a/bin/physical_sec b/bin/physical_sec
@@ -116,9 +116,12 @@ if [ -z "$ext_ctrl" ]; then
 else
     echo Bridging $ext_ctrl to $ctrl_bridge
     sudo ovs-vsctl add-port $ctrl_bridge $ext_ctrl
-    sudo ip addr flush dev $ext_ctrl
+    sudo ip -l 200 addr flush dev $ext_ctrl
 fi
 
+echo Warmup ping for $ext_addr
+ping -n -c 2 $ext_addr || true
+
 echo Checking external connection to $ext_addr
 if ! ping -n -c 2 $ext_addr; then
     echo