From 756e77ed2fe1df9f394c22bbfbdcffde631d82f2 Mon Sep 17 00:00:00 2001 From: Jack LeCroy <3073035+jacklecroy@users.noreply.github.com> Date: Thu, 5 Sep 2019 19:32:04 -0400 Subject: [PATCH 1/7] Start of create database script /database_mgmt/create_database.py --- .vscode/settings.json | 5 +++-- Pipfile | 3 +++ Pipfile.lock | 21 +++++++++++++++++++-- database_mgmt/create_database.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 database_mgmt/create_database.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 3691827..762f418 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,7 @@ { "python.linting.pylintEnabled": false, "python.linting.pep8Enabled": true, - "python.linting.enabled": true, - "python.linting.pep8Path": "pycodestyle" + "python.linting.enabled": false, + "python.linting.pep8Path": "pycodestyle", + "python.pythonPath": "C:\\Users\\Jack LeCroy\\.conda\\envs\\pipenv-test\\python.exe" } \ No newline at end of file diff --git a/Pipfile b/Pipfile index 5903fbd..e33e09e 100644 --- a/Pipfile +++ b/Pipfile @@ -4,12 +4,15 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] +pep8 = "*" [packages] websockets = "*" pandas = "*" mysql-connector = "*" requests = "*" +pycodestyle = "*" +dotenv = "*" [requires] python_version = "3.7.4" diff --git a/Pipfile.lock b/Pipfile.lock index 6e58f28..2d9c79e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7d453ed917cb5091c752cfe5caac404603167fc595826b3d9c4960aef46be38d" + "sha256": "3e20f96a02a06866dda33d82c680fdcd88371793da5f2a47c612ab0c7954993e" }, "pipfile-spec": 6, "requires": { @@ -86,6 +86,14 @@ "index": "pypi", "version": "==0.25.1" }, + "pycodestyle": { + "hashes": [ + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + ], + "index": "pypi", + "version": "==2.5.0" + }, "python-dateutil": { "hashes": [ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", @@ -140,5 +148,14 @@ "version": "==8.0.2" } }, - "develop": {} + "develop": { + "pep8": { + "hashes": [ + "sha256:b22cfae5db09833bb9bd7c8463b53e1a9c9b39f12e304a8d0bba729c501827ee", + "sha256:fe249b52e20498e59e0b5c5256aa52ee99fc295b26ec9eaa85776ffdb9fe6374" + ], + "index": "pypi", + "version": "==1.7.1" + } + } } diff --git a/database_mgmt/create_database.py b/database_mgmt/create_database.py new file mode 100644 index 0000000..2d1a9a1 --- /dev/null +++ b/database_mgmt/create_database.py @@ -0,0 +1,28 @@ +import os +import pandas as pd +from dotenv import load_dotenv +load_dotenv() + +sensor_dict = pd.read_csv("../can_mgmt/SensorDict.csv", header=0, dtype="str") + +db = Database() +cursor = db.get_cursor() +cursor.execute("CREATE DATABASE sensors") +cursor.execute("USE sensors") + +current_id = "#####" +for index, row in sensor_dict.iterrows(): + if row["id"][3] == current_id[3]: + add_query = "ADD COLUMN {} INT(16) NULL".format(row["name"]) + cursor.execute(add_query) + else: + current_id = row["id"] + table_name = current_id[0:3] + create_query = "CREATE TABLE {}".format(table_name) + alter_query = "ALTER TABLE {}".format(table_name) + timestamp_query = "ADD COLUMN timestamp DATETIME NOT NULL" + add_query = "ADD COLUMN {} INT(16) NULL".format(row["name"]) + cursor.execute(create_query) + cursor.execute(alter_query) + cursor.execute(timestamp_query) + cursor.execute(add_query) From aebb727b791ed145541e7e8e6ed5596c0bb88379 Mon Sep 17 00:00:00 2001 From: Jack LeCroy <3073035+jacklecroy@users.noreply.github.com> Date: Sat, 7 Sep 2019 11:49:18 -0400 Subject: [PATCH 2/7] Fix update script and run it There are some 'rogue' cells, we can worry about that later. --- .gitignore | 1 + .vscode/settings.json | 1 - can_mgmt/CANID.h | 159 +++++++++++++++++++--------------- can_mgmt/SensorDict.csv | 89 ++++++++++--------- can_mgmt/update-SensorDict.py | 18 ++-- 5 files changed, 152 insertions(+), 116 deletions(-) diff --git a/.gitignore b/.gitignore index 14b9c92..d522277 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ */.idea *.pyc /logs +.vs/* \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 762f418..4380941 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,5 +3,4 @@ "python.linting.pep8Enabled": true, "python.linting.enabled": false, "python.linting.pep8Path": "pycodestyle", - "python.pythonPath": "C:\\Users\\Jack LeCroy\\.conda\\envs\\pipenv-test\\python.exe" } \ No newline at end of file diff --git a/can_mgmt/CANID.h b/can_mgmt/CANID.h index 44f455d..145bf44 100644 --- a/can_mgmt/CANID.h +++ b/can_mgmt/CANID.h @@ -1,4 +1,4 @@ -/* Generated on 2019-03-02 12:35:46.467557 +/* Generated on 2019-09-07 11:43:50.028192 * This file contains all of the constants that are * being used on the CAN bus. * @@ -11,15 +11,15 @@ #define CANID_H #define WHEEL_SPEED_FRONT_CAN_ID 0x700 #define WHEEL_SPEED_FRONT_L_START_BIT 0 -#define WHEEL_SPEED_FRONT_L_END_BIT 15 -#define WHEEL_SPEED_FRONT_R_START_BIT 16 -#define WHEEL_SPEED_FRONT_R_END_BIT 31 +#define WHEEL_SPEED_FRONT_L_END_BIT 31 +#define WHEEL_SPEED_FRONT_R_START_BIT 32 +#define WHEEL_SPEED_FRONT_R_END_BIT 63 #define WHEEL_SPEED_REAR_CAN_ID 0x701 #define WHEEL_SPEED_REAR_L_START_BIT 0 -#define WHEEL_SPEED_REAR_L_END_BIT 15 -#define WHEEL_SPEED_REAR_R_START_BIT 16 -#define WHEEL_SPEED_REAR_R_END_BIT 31 +#define WHEEL_SPEED_REAR_L_END_BIT 31 +#define WHEEL_SPEED_REAR_R_START_BIT 32 +#define WHEEL_SPEED_REAR_R_END_BIT 63 #define TIRE_TEMP_FRONT_CAN_ID 0x710 #define TIRE_TEMP_FRONT_R_START_BIT 0 @@ -36,12 +36,14 @@ #define COOLANT_CAN_ID 0x720 #define COOLANT_MOTOR_START_BIT 0 #define COOLANT_MOTOR_END_BIT 15 -#define COOLANT_FLOW_START_BIT 16 +#define COOLANT_RAD_START_BIT 16 +#define COOLANT_RAD_END_BIT 31 +#define COOLANT_MC_START_BIT 32 +#define COOLANT_MC_END_BIT 47 + +#define COOLANT_CAN_ID 0x721 +#define COOLANT_FLOW_START_BIT 0 #define COOLANT_FLOW_END_BIT 31 -#define COOLANT_RAD_START_BIT 32 -#define COOLANT_RAD_END_BIT 47 -#define COOLANT_MC_START_BIT 48 -#define COOLANT_MC_END_BIT 63 #define SHOCK_POT_FRONT_CAN_ID 0x730 #define SHOCK_POT_FRONT_L_START_BIT 0 @@ -61,65 +63,65 @@ #define DROP_LINK_RIGHT_START_BIT 16 #define DROP_LINK_RIGHT_END_BIT 31 -#define STEER_ROD_FRONT_CAN_ID 0x750 -#define STEER_ROD_FRONT_L_START_BIT 0 -#define STEER_ROD_FRONT_L_END_BIT 15 -#define PUSHROD_FRONT_R_START_BIT 16 -#define PUSHROD_FRONT_R_END_BIT 31 +#define PUSHROD_FRONT_CAN_ID 0x750 +#define PUSHROD_FRONT_R_START_BIT 0 +#define PUSHROD_FRONT_R_END_BIT 15 +#define STEER_ROD_FRONT_R_START_BIT 16 +#define STEER_ROD_FRONT_R_END_BIT 31 #define PUSHROD_FRONT_L_START_BIT 32 #define PUSHROD_FRONT_L_END_BIT 47 -#define STEER_ROD_FRONT_R_START_BIT 48 -#define STEER_ROD_FRONT_R_END_BIT 63 +#define STEER_ROD_FRONT_L_START_BIT 48 +#define STEER_ROD_FRONT_L_END_BIT 63 #define PUSHROD_REAR_CAN_ID 0x760 #define PUSHROD_REAR_R_START_BIT 0 #define PUSHROD_REAR_R_END_BIT 15 -#define TIEROD_REAR_L_START_BIT 16 -#define TIEROD_REAR_L_END_BIT 31 -#define TIEROD_REAR_R_START_BIT 32 -#define TIEROD_REAR_R_END_BIT 47 -#define PUSHROD_REAR_L_START_BIT 48 -#define PUSHROD_REAR_L_END_BIT 63 +#define TIEROD_REAR_R_START_BIT 16 +#define TIEROD_REAR_R_END_BIT 31 +#define PUSHROD_REAR_L_START_BIT 32 +#define PUSHROD_REAR_L_END_BIT 47 +#define TIEROD_REAR_L_START_BIT 48 +#define TIEROD_REAR_L_END_BIT 63 #define LCA_FRONT_CAN_ID 0x770 -#define LCA_FRONT_LB_START_BIT 0 -#define LCA_FRONT_LB_END_BIT 15 -#define LCA_FRONT_RF_START_BIT 16 -#define LCA_FRONT_RF_END_BIT 31 -#define LCA_FRONT_LF_START_BIT 32 -#define LCA_FRONT_LF_END_BIT 47 +#define LCA_FRONT_LF_START_BIT 0 +#define LCA_FRONT_LF_END_BIT 15 +#define LCA_FRONT_LB_START_BIT 16 +#define LCA_FRONT_LB_END_BIT 31 +#define LCA_FRONT_RF_START_BIT 32 +#define LCA_FRONT_RF_END_BIT 47 #define LCA_FRONT_RB_START_BIT 48 #define LCA_FRONT_RB_END_BIT 63 #define LCA_REAR_CAN_ID 0x780 #define LCA_REAR_LF_START_BIT 0 #define LCA_REAR_LF_END_BIT 15 -#define LCA_REAR_RF_START_BIT 16 -#define LCA_REAR_RF_END_BIT 31 -#define LCA_REAR_LB_START_BIT 32 -#define LCA_REAR_LB_END_BIT 47 +#define LCA_REAR_LB_START_BIT 16 +#define LCA_REAR_LB_END_BIT 31 +#define LCA_REAR_RF_START_BIT 32 +#define LCA_REAR_RF_END_BIT 47 #define LCA_REAR_RB_START_BIT 48 #define LCA_REAR_RB_END_BIT 63 #define UCA_FRONT_CAN_ID 0x790 -#define UCA_FRONT_RB_START_BIT 0 -#define UCA_FRONT_RB_END_BIT 15 +#define UCA_FRONT_LF_START_BIT 0 +#define UCA_FRONT_LF_END_BIT 15 #define UCA_FRONT_LB_START_BIT 16 #define UCA_FRONT_LB_END_BIT 31 #define UCA_FRONT_RF_START_BIT 32 #define UCA_FRONT_RF_END_BIT 47 -#define UCA_FRONT_LF_START_BIT 48 -#define UCA_FRONT_LF_END_BIT 63 +#define UCA_FRONT_RB_START_BIT 48 +#define UCA_FRONT_RB_END_BIT 63 #define UCA_REAR_CAN_ID 0x7a0 #define UCA_REAR_LF_START_BIT 0 #define UCA_REAR_LF_END_BIT 15 -#define UCA_REAR_RB_START_BIT 16 -#define UCA_REAR_RB_END_BIT 31 -#define UCA_REAR_LB_START_BIT 32 -#define UCA_REAR_LB_END_BIT 47 -#define UCA_REAR_RF_START_BIT 48 -#define UCA_REAR_RF_END_BIT 63 +#define UCA_REAR_LB_START_BIT 16 +#define UCA_REAR_LB_END_BIT 31 +#define UCA_REAR_RF_START_BIT 32 +#define UCA_REAR_RF_END_BIT 47 +#define UCA_REAR_RB_START_BIT 48 +#define UCA_REAR_RB_END_BIT 63 #define FRONT_TORSIONAL_CAN_ID 0x7b0 #define FRONT_TORSIONAL_ARB_START_BIT 0 @@ -135,27 +137,48 @@ #define STEERING_ANGLE_START_BIT 16 #define STEERING_ANGLE_END_BIT 31 -#define IMU_CAN_ID 0x7d0 -#define IMU_TYPE_START_BIT 0 -#define IMU_TYPE_END_BIT 7 -#define IMU_DATA_START_BIT 8 -#define IMU_DATA_END_BIT 23 -#define IMU_TIME_START_BIT 24 -#define IMU_TIME_END_BIT 47 - -#define FLAG_WHEEL_SPEED_BIT 7 -#define FLAG_TIRE_TEMP_BIT 6 -#define FLAG_COOLANT_TEMP_BIT 5 -#define FLAG_COOLANT_FLOW_BIT 4 -#define FLAG_SHOCK_POTS_BIT 3 -#define FLAG_ARB_DROPLINKS_BIT 2 -#define FLAG_PUSH_RODS_BIT 1 -#define FLAG_TIRE_RODS_BIT 0 -#define FLAG_LCA_BIT 15 -#define FLAG_UCA_BIT 14 -#define FLAG_ARB_TORSIONAL_BIT 13 -#define FLAG_STEERING_ANGLE_BIT 12 -#define FLAG_STEERING_TORSIONAL_BIT 11 -#define FLAG_ACCELEROMETER_BIT 10 -#define FLAG_GYROSCOPE_BIT 9 +#define X_CAN_ID 0x7d0 +#define X_AXIS [15:8]_START_BIT 8 +#define X_AXIS [15:8]_END_BIT 15 +#define X_AXIS [7:0]_START_BIT 16 +#define X_AXIS [7:0]_END_BIT 23 +#define Y_AXIS [15:8] _START_BIT 24 +#define Y_AXIS [15:8] _END_BIT 31 +#define Y_AXIS [7:0]_START_BIT 32 +#define Y_AXIS [7:0]_END_BIT 39 +#define Z_AXIS [15:8] _START_BIT 40 +#define Z_AXIS [15:8] _END_BIT 47 +#define Z_AXIS [7:0] _START_BIT 48 +#define Z_AXIS [7:0] _END_BIT 55 + +#define L_WHEEL_SPD_CAN_ID 0x7f0 +#define L_WHEEL_SPD_ERROR_START_BIT 0 +#define L_WHEEL_SPD_ERROR_END_BIT 7 +#define R_WHEEL_SPEED_ERROR_START_BIT 8 +#define R_WHEEL_SPEED_ERROR_END_BIT 15 +#define MAX11614_ERROR_START_BIT 16 +#define MAX11614_ERROR_END_BIT 23 +#define MAX11616_ERROR_START_BIT 24 +#define MAX11616_ERROR_END_BIT 31 + +#define L_WHEEL_SPD_CAN_ID 0x7f1 +#define L_WHEEL_SPD_ERROR_START_BIT 0 +#define L_WHEEL_SPD_ERROR_END_BIT 7 +#define R_WHEEL_SPEED_ERROR_START_BIT 8 +#define R_WHEEL_SPEED_ERROR_END_BIT 15 +#define MAX11614_ERROR_START_BIT 16 +#define MAX11614_ERROR_END_BIT 23 +#define MAX11616_ERROR_START_BIT 24 +#define MAX11616_ERROR_END_BIT 31 +#define COOL_SPD_ERROR_START_BIT 32 +#define COOL_SPD_ERROR_END_BIT 39 + +#define FLAG_SPEED_CAN_MASK_BIT 7 +#define FLAG_UCA_CAN_MASK _BIT 6 +#define FLAG_LCA_CAN_MASK _BIT 5 +#define FLAG_SHOCK_CAN_MASK _BIT 4 +#define FLAG_ARB_TORS_CAN_MASK _BIT 3 +#define FLAG_PUSH_ROD_CAN_MASK _BIT 2 +#define FLAG_COOLANT_CAN_MASK _BIT 1 +#define FLAG_DROP_LINK_CAN_MASK_BIT 0 #endif /* CANID_H */ \ No newline at end of file diff --git a/can_mgmt/SensorDict.csv b/can_mgmt/SensorDict.csv index 8df926f..c1d2c73 100644 --- a/can_mgmt/SensorDict.csv +++ b/can_mgmt/SensorDict.csv @@ -1,65 +1,72 @@ id,MSB,LSB,name -0x700,15,0,WHEEL_SPEED_FRONT_L -0x700,31,16,WHEEL_SPEED_FRONT_R -0x701,15,0,WHEEL_SPEED_REAR_L -0x701,31,16,WHEEL_SPEED_REAR_R +0x700,31,0,WHEEL_SPEED_FRONT_L +0x700,63,32,WHEEL_SPEED_FRONT_R +0x701,31,0,WHEEL_SPEED_REAR_L +0x701,63,32,WHEEL_SPEED_REAR_R 0x710,15,0,TIRE_TEMP_FRONT_R 0x710,31,16,TIRE_TEMP_FRONT_L 0x711,15,0,TIRE_TEMP_REAR_R 0x711,31,16,TIRE_TEMP_REAR_L 0x720,15,0,COOLANT_MOTOR -0x720,31,16,COOLANT_FLOW -0x720,47,32,COOLANT_RAD -0x720,63,48,COOLANT_MC +0x720,31,16,COOLANT_RAD +0x720,47,32,COOLANT_MC +0x721,31,0,COOLANT_FLOW 0x730,15,0,SHOCK_POT_FRONT_L 0x730,31,16,SHOCK_POT_FRONT_R 0x731,15,0,SHOCK_POT_REAR_R 0x731,31,16,SHOCK_POT_REAR_L 0x740,15,0,DROP_LINK_LEFT 0x740,31,16,DROP_LINK_RIGHT -0x750,15,0,STEER_ROD_FRONT_L -0x750,31,16,PUSHROD_FRONT_R +0x750,15,0,PUSHROD_FRONT_R +0x750,31,16,STEER_ROD_FRONT_R 0x750,47,32,PUSHROD_FRONT_L -0x750,63,48,STEER_ROD_FRONT_R +0x750,63,48,STEER_ROD_FRONT_L 0x760,15,0,PUSHROD_REAR_R -0x760,31,16,TIEROD_REAR_L -0x760,47,32,TIEROD_REAR_R -0x760,63,48,PUSHROD_REAR_L -0x770,15,0,LCA_FRONT_LB -0x770,31,16,LCA_FRONT_RF -0x770,47,32,LCA_FRONT_LF +0x760,31,16,TIEROD_REAR_R +0x760,47,32,PUSHROD_REAR_L +0x760,63,48,TIEROD_REAR_L +0x770,15,0,LCA_FRONT_LF +0x770,31,16,LCA_FRONT_LB +0x770,47,32,LCA_FRONT_RF 0x770,63,48,LCA_FRONT_RB 0x780,15,0,LCA_REAR_LF -0x780,31,16,LCA_REAR_RF -0x780,47,32,LCA_REAR_LB +0x780,31,16,LCA_REAR_LB +0x780,47,32,LCA_REAR_RF 0x780,63,48,LCA_REAR_RB -0x790,15,0,UCA_FRONT_RB +0x790,15,0,UCA_FRONT_LF 0x790,31,16,UCA_FRONT_LB 0x790,47,32,UCA_FRONT_RF -0x790,63,48,UCA_FRONT_LF +0x790,63,48,UCA_FRONT_RB 0x7a0,15,0,UCA_REAR_LF -0x7a0,31,16,UCA_REAR_RB -0x7a0,47,32,UCA_REAR_LB -0x7a0,63,48,UCA_REAR_RF +0x7a0,31,16,UCA_REAR_LB +0x7a0,47,32,UCA_REAR_RF +0x7a0,63,48,UCA_REAR_RB 0x7b0,15,0,FRONT_TORSIONAL_ARB 0x7b1,15,0,REAR_TORSIONAL_ARB 0x7c0,15,0,STEERING_TORSION 0x7c0,31,16,STEERING_ANGLE -0x7d0,7,0,IMU_TYPE -0x7d0,23,8,IMU_DATA -0x7d0,47,24,IMU_TIME -0x7E0,7,7,FLAG_WHEEL_SPEED -0x7E0,6,6,FLAG_TIRE_TEMP -0x7E0,5,5,FLAG_COOLANT_TEMP -0x7E0,4,4,FLAG_COOLANT_FLOW -0x7E0,3,3,FLAG_SHOCK_POTS -0x7E0,2,2,FLAG_ARB_DROPLINKS -0x7E0,1,1,FLAG_PUSH_RODS -0x7E0,0,0,FLAG_TIRE_RODS -0x7E0,15,15,FLAG_LCA -0x7E0,14,14,FLAG_UCA -0x7E0,13,13,FLAG_ARB_TORSIONAL -0x7E0,12,12,FLAG_STEERING_ANGLE -0x7E0,11,11,FLAG_STEERING_TORSIONAL -0x7E0,10,10,FLAG_ACCELEROMETER -0x7E0,9,9,FLAG_GYROSCOPE +0x7d0,15,8,X_AXIS [15:8] +0x7d0,23,16,X_AXIS [7:0] +0x7d0,31,24,Y_AXIS [15:8] +0x7d0,39,32,Y_AXIS [7:0] +0x7d0,47,40,Z_AXIS [15:8] +0x7d0,55,48,Z_AXIS [7:0] +0x7f0,7,0,L_WHEEL_SPD_ERROR +0x7f0,15,8,R_WHEEL_SPEED_ERROR +0x7f0,23,16,MAX11614_ERROR +0x7f0,31,24,MAX11616_ERROR +0x7f1,7,0,L_WHEEL_SPD_ERROR +0x7f1,15,8,R_WHEEL_SPEED_ERROR +0x7f1,23,16,MAX11614_ERROR +0x7f1,31,24,MAX11616_ERROR +0x7f1,39,32,COOL_SPD_ERROR +0x7d0,3,3,0: Accel +0x7d0,3,3,1: Gyro +0x7e0,7,7,FLAG_SPEED_CAN_MASK +0x7e0,6,6,FLAG_UCA_CAN_MASK +0x7e0,5,5,FLAG_LCA_CAN_MASK +0x7e0,4,4,FLAG_SHOCK_CAN_MASK +0x7e0,3,3,FLAG_ARB_TORS_CAN_MASK +0x7e0,2,2,FLAG_PUSH_ROD_CAN_MASK +0x7e0,1,1,FLAG_COOLANT_CAN_MASK +0x7e0,0,0,FLAG_DROP_LINK_CAN_MASK diff --git a/can_mgmt/update-SensorDict.py b/can_mgmt/update-SensorDict.py index df3c807..6adcb8f 100644 --- a/can_mgmt/update-SensorDict.py +++ b/can_mgmt/update-SensorDict.py @@ -3,8 +3,8 @@ import datetime as dt SPREADSHEET_FILE_ID = "195Y2cf9C2mrA6QCLuLJv3NHsCqTLi-3Wd07q4T5rXuM" -OUTPUT_CSV_FILE = "SensorDict.csv" -OUTPUT_HEADER_FILE = "CANID.h" +OUTPUT_CSV_FILE = "can_mgmt/SensorDict.csv" +OUTPUT_HEADER_FILE = "can_mgmt/CANID.h" # Rows before the header are skipped. # The row after the header will also be skipped because it is an example row. HEADER_ROW = 5 @@ -52,10 +52,13 @@ flags_list = item.split("\n") for flag_i, flag in enumerate(flags_list): flag_paren_start = flag.find("(") - flag_bit = str(int(flag[flag_paren_start + 1]) + - 8 * int(column[4])) + try: + flag_bit = str(int(flag[flag_paren_start + 1]) + + 8 * int(column[4])) + except: + continue flag = flag[0:flag_paren_start - 1] - flags_df = flags_df.append({"id": index, "MSB": flag_bit, + flags_df = flags_df.append({"id": index.lower(), "MSB": flag_bit, "LSB": flag_bit, "name": flag}, ignore_index=True) continue @@ -108,7 +111,10 @@ row["LSB"])) last_ID = ID continue - last_underscore = sensor_name_and_type.rindex("_") + try: + last_underscore = sensor_name_and_type.rindex("_") + except: + continue sensor_name = sensor_name_and_type[0:last_underscore] if last_ID != ID: savefile_h.write("\n#define {}_CAN_ID {}\n".format(sensor_name, ID)) From 7524f0f824ab41d6c6c801aa4281d9aadb4aa209 Mon Sep 17 00:00:00 2001 From: Ryan Schwartz Date: Sat, 7 Sep 2019 12:36:40 -0400 Subject: [PATCH 3/7] Fix database queries to create the database Relates to #16 --- database_mgmt/create_database.py | 27 ++++++++++--------------- {database => database_mgmt}/database.py | 0 2 files changed, 11 insertions(+), 16 deletions(-) rename {database => database_mgmt}/database.py (100%) diff --git a/database_mgmt/create_database.py b/database_mgmt/create_database.py index 2d1a9a1..358c576 100644 --- a/database_mgmt/create_database.py +++ b/database_mgmt/create_database.py @@ -1,28 +1,23 @@ import os import pandas as pd -from dotenv import load_dotenv -load_dotenv() +from database import Database -sensor_dict = pd.read_csv("../can_mgmt/SensorDict.csv", header=0, dtype="str") +sensor_dict = pd.read_csv('can_mgmt/SensorDict.csv', header=0, dtype='str') db = Database() cursor = db.get_cursor() -cursor.execute("CREATE DATABASE sensors") -cursor.execute("USE sensors") +cursor.execute('CREATE DATABASE sensors') +cursor.execute('USE sensors') -current_id = "#####" +current_id = '#####' for index, row in sensor_dict.iterrows(): - if row["id"][3] == current_id[3]: - add_query = "ADD COLUMN {} INT(16) NULL".format(row["name"]) + if row['id'][3] == current_id[3]: + add_query = 'ALTER TABLE {} ADD COLUMN {} INT(16) NULL'.format(current_id[1:4], row['name']) cursor.execute(add_query) else: - current_id = row["id"] - table_name = current_id[0:3] - create_query = "CREATE TABLE {}".format(table_name) - alter_query = "ALTER TABLE {}".format(table_name) - timestamp_query = "ADD COLUMN timestamp DATETIME NOT NULL" - add_query = "ADD COLUMN {} INT(16) NULL".format(row["name"]) + current_id = row['id'] + table_name = current_id[1:4] # Tables can't start with a 0x (no starting integer), so start with x instead + create_query = 'CREATE TABLE {} (timestamp DATETIME(3) NOT NULL)'.format(table_name) + alter_query = 'ALTER TABLE {} ADD COLUMN {} INT(16) NULL'.format(table_name, row['name']) cursor.execute(create_query) cursor.execute(alter_query) - cursor.execute(timestamp_query) - cursor.execute(add_query) diff --git a/database/database.py b/database_mgmt/database.py similarity index 100% rename from database/database.py rename to database_mgmt/database.py From 83dbdd56aed3f6ec10d501386a497f08ffc0c4ae Mon Sep 17 00:00:00 2001 From: Ryan Schwartz Date: Sat, 7 Sep 2019 12:44:44 -0400 Subject: [PATCH 4/7] Add database tests --- tests/pathmagic.py | 2 +- tests/test_database.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/pathmagic.py b/tests/pathmagic.py index f109807..4f5683b 100644 --- a/tests/pathmagic.py +++ b/tests/pathmagic.py @@ -4,4 +4,4 @@ import sys path = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(path + '/../database/') +sys.path.append(path + '/../database_mgmt/') diff --git a/tests/test_database.py b/tests/test_database.py index f504ed8..b799cb1 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -17,3 +17,12 @@ def test_connection(self): def test_cursor(self): db.get_cursor() + + def test_creation(self): + import create_database + cursor = db.get_cursor() + cursor.execute('use sensors') + test = cursor.execute('describe x70') + print(test) + +TestDatabase().test_creation() \ No newline at end of file From ec820fbc54b6a114d9aad4a14e67fbb49b44b88f Mon Sep 17 00:00:00 2001 From: Ryan Schwartz Date: Sat, 7 Sep 2019 14:43:37 -0400 Subject: [PATCH 5/7] Add abstraction to create_database Create table and add column are now within the Database class --- database_mgmt/create_database.py | 12 +++++------ database_mgmt/database.py | 25 ++++++++++++++++++++--- database_mgmt/database_debug_functions.py | 10 +++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 database_mgmt/database_debug_functions.py diff --git a/database_mgmt/create_database.py b/database_mgmt/create_database.py index 358c576..02e6b6d 100644 --- a/database_mgmt/create_database.py +++ b/database_mgmt/create_database.py @@ -12,12 +12,10 @@ current_id = '#####' for index, row in sensor_dict.iterrows(): if row['id'][3] == current_id[3]: - add_query = 'ALTER TABLE {} ADD COLUMN {} INT(16) NULL'.format(current_id[1:4], row['name']) - cursor.execute(add_query) + db.add_column(current_id[1:4], row['name'], 'INT(16) NULL') else: current_id = row['id'] - table_name = current_id[1:4] # Tables can't start with a 0x (no starting integer), so start with x instead - create_query = 'CREATE TABLE {} (timestamp DATETIME(3) NOT NULL)'.format(table_name) - alter_query = 'ALTER TABLE {} ADD COLUMN {} INT(16) NULL'.format(table_name, row['name']) - cursor.execute(create_query) - cursor.execute(alter_query) + # Tables can't start with a 0x (no starting integer), so start with x + table_name = current_id[1:4] + db.create_table(table_name) + db.add_column(table_name, row['name'], 'INT(16) NULL') diff --git a/database_mgmt/database.py b/database_mgmt/database.py index 169e124..c540cf4 100644 --- a/database_mgmt/database.py +++ b/database_mgmt/database.py @@ -24,8 +24,27 @@ def connect(self): return db - def get_cursor(self): - if self._connection is None: - self.connect() + def require_connection(func): + def wrapper(*args): + self = args[0] + if self._connection is None: + self.connect() + return func(*args) + return wrapper + @require_connection + def get_cursor(self): return self._connection.cursor() + + @require_connection + def create_table(self, table): + cursor = self.get_cursor() + create_query = f'CREATE TABLE {table} (timestamp DATETIME(3) NOT NULL)' + cursor.execute(create_query) + + + @require_connection + def add_column(self, table, column, data_type): + cursor = self.get_cursor() + alter_query = f'ALTER TABLE {table} ADD COLUMN {column} {data_type}' + cursor.execute(alter_query) diff --git a/database_mgmt/database_debug_functions.py b/database_mgmt/database_debug_functions.py new file mode 100644 index 0000000..174ed4c --- /dev/null +++ b/database_mgmt/database_debug_functions.py @@ -0,0 +1,10 @@ +import os +import sys +from database import Database + +setting = sys.argv[1] + +if setting == 'wipe': + db = Database() + cursor = db.get_cursor() + cursor.execute("DROP DATABASE sensors") From 2be1b064e74aefe5b3108d87e5fb50ec6bf9e960 Mon Sep 17 00:00:00 2001 From: Ryan Schwartz Date: Sat, 7 Sep 2019 15:32:22 -0400 Subject: [PATCH 6/7] Move Database instantiation to static variables No need for database connections to be based on the instance, so move database connection to be static instead. --- database_mgmt/database.py | 15 +++++++-------- tests/test_database.py | 2 -- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/database_mgmt/database.py b/database_mgmt/database.py index c540cf4..1b8f063 100644 --- a/database_mgmt/database.py +++ b/database_mgmt/database.py @@ -5,17 +5,16 @@ class Database: - def __init__(self): - self.host = os.getenv('DATABASE_HOST') - self.user = os.getenv('DATABASE_USER') - self.password = os.getenv('DATABASE_PASSWORD') - self._connection = None + _host = os.getenv('DATABASE_HOST') + _user = os.getenv('DATABASE_USER') + _password = os.getenv('DATABASE_PASSWORD') + _connection = None def connect(self): db = mysql.connector.connect( - host=self.host, - user=self.user, - password=self.password + host=self._host, + user=self._user, + password=self._password ) # Set a connection so that a new connection doesn't diff --git a/tests/test_database.py b/tests/test_database.py index b799cb1..99cca59 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -24,5 +24,3 @@ def test_creation(self): cursor.execute('use sensors') test = cursor.execute('describe x70') print(test) - -TestDatabase().test_creation() \ No newline at end of file From 5551242e74b58198ebc660ed565a858707da0871 Mon Sep 17 00:00:00 2001 From: Jack LeCroy <3073035+jacklecroy@users.noreply.github.com> Date: Sat, 7 Sep 2019 15:45:25 -0400 Subject: [PATCH 7/7] Fix sensordict --- can_mgmt/CANID.h | 36 +++++++++++++++++++++-------------- can_mgmt/SensorData.csv | 1 - can_mgmt/SensorDict.csv | 17 +++++++++-------- can_mgmt/SensorDict.py | 31 ++---------------------------- can_mgmt/update-SensorDict.py | 12 +++--------- 5 files changed, 36 insertions(+), 61 deletions(-) delete mode 100644 can_mgmt/SensorData.csv diff --git a/can_mgmt/CANID.h b/can_mgmt/CANID.h index 145bf44..eabb40a 100644 --- a/can_mgmt/CANID.h +++ b/can_mgmt/CANID.h @@ -1,4 +1,4 @@ -/* Generated on 2019-09-07 11:43:50.028192 +/* Generated on 2019-09-07 14:44:25.169641 * This file contains all of the constants that are * being used on the CAN bus. * @@ -9,6 +9,16 @@ #ifndef CANID_H #define CANID_H +#define RAW_THROTTLE_CAN_ID 0x500 +#define RAW_THROTTLE_ONE_START_BIT 0 +#define RAW_THROTTLE_ONE_END_BIT 7 +#define RAW_THROTTLE_TWO_START_BIT 8 +#define RAW_THROTTLE_TWO_END_BIT 15 +#define BRAKE_VALUE_ONE_START_BIT 16 +#define BRAKE_VALUE_ONE_END_BIT 23 +#define BRAKE_VALUE_TWO_START_BIT 24 +#define BRAKE_VALUE_TWO_END_BIT 31 + #define WHEEL_SPEED_FRONT_CAN_ID 0x700 #define WHEEL_SPEED_FRONT_L_START_BIT 0 #define WHEEL_SPEED_FRONT_L_END_BIT 31 @@ -137,19 +147,17 @@ #define STEERING_ANGLE_START_BIT 16 #define STEERING_ANGLE_END_BIT 31 -#define X_CAN_ID 0x7d0 -#define X_AXIS [15:8]_START_BIT 8 -#define X_AXIS [15:8]_END_BIT 15 -#define X_AXIS [7:0]_START_BIT 16 -#define X_AXIS [7:0]_END_BIT 23 -#define Y_AXIS [15:8] _START_BIT 24 -#define Y_AXIS [15:8] _END_BIT 31 -#define Y_AXIS [7:0]_START_BIT 32 -#define Y_AXIS [7:0]_END_BIT 39 -#define Z_AXIS [15:8] _START_BIT 40 -#define Z_AXIS [15:8] _END_BIT 47 -#define Z_AXIS [7:0] _START_BIT 48 -#define Z_AXIS [7:0] _END_BIT 55 +#define IMU_TYPE_CAN_ID 0x7d0 +#define IMU_TYPE_AXIS_START_BIT 0 +#define IMU_TYPE_AXIS_END_BIT 7 +#define IMU_X_AXIS_START_BIT 8 +#define IMU_X_AXIS_END_BIT 23 +#define IMU_Y_AXIS_START_BIT 24 +#define IMU_Y_AXIS_END_BIT 39 +#define IMU_Z_AXIS_START_BIT 40 +#define IMU_Z_AXIS_END_BIT 55 +#define IMU_RESOLUTION_START_BIT 56 +#define IMU_RESOLUTION_END_BIT 63 #define L_WHEEL_SPD_CAN_ID 0x7f0 #define L_WHEEL_SPD_ERROR_START_BIT 0 diff --git a/can_mgmt/SensorData.csv b/can_mgmt/SensorData.csv deleted file mode 100644 index ae663f0..0000000 --- a/can_mgmt/SensorData.csv +++ /dev/null @@ -1 +0,0 @@ -timestamp,id,data diff --git a/can_mgmt/SensorDict.csv b/can_mgmt/SensorDict.csv index c1d2c73..71ddd67 100644 --- a/can_mgmt/SensorDict.csv +++ b/can_mgmt/SensorDict.csv @@ -1,4 +1,8 @@ id,MSB,LSB,name +0x500,7,0,RAW_THROTTLE_ONE +0x500,15,8,RAW_THROTTLE_TWO +0x500,23,16,BRAKE_VALUE_ONE +0x500,31,24,BRAKE_VALUE_TWO 0x700,31,0,WHEEL_SPEED_FRONT_L 0x700,63,32,WHEEL_SPEED_FRONT_R 0x701,31,0,WHEEL_SPEED_REAR_L @@ -45,12 +49,11 @@ id,MSB,LSB,name 0x7b1,15,0,REAR_TORSIONAL_ARB 0x7c0,15,0,STEERING_TORSION 0x7c0,31,16,STEERING_ANGLE -0x7d0,15,8,X_AXIS [15:8] -0x7d0,23,16,X_AXIS [7:0] -0x7d0,31,24,Y_AXIS [15:8] -0x7d0,39,32,Y_AXIS [7:0] -0x7d0,47,40,Z_AXIS [15:8] -0x7d0,55,48,Z_AXIS [7:0] +0x7d0,7,0,IMU_TYPE_AXIS +0x7d0,23,8,IMU_X_AXIS +0x7d0,39,24,IMU_Y_AXIS +0x7d0,55,40,IMU_Z_AXIS +0x7d0,63,56,IMU_RESOLUTION 0x7f0,7,0,L_WHEEL_SPD_ERROR 0x7f0,15,8,R_WHEEL_SPEED_ERROR 0x7f0,23,16,MAX11614_ERROR @@ -60,8 +63,6 @@ id,MSB,LSB,name 0x7f1,23,16,MAX11614_ERROR 0x7f1,31,24,MAX11616_ERROR 0x7f1,39,32,COOL_SPD_ERROR -0x7d0,3,3,0: Accel -0x7d0,3,3,1: Gyro 0x7e0,7,7,FLAG_SPEED_CAN_MASK 0x7e0,6,6,FLAG_UCA_CAN_MASK 0x7e0,5,5,FLAG_LCA_CAN_MASK diff --git a/can_mgmt/SensorDict.py b/can_mgmt/SensorDict.py index ff980fd..1417fa3 100644 --- a/can_mgmt/SensorDict.py +++ b/can_mgmt/SensorDict.py @@ -1,39 +1,12 @@ -import csv - +import time # Import sensor data from CSF file. This file contains information # about each individual sensor on the car as well as its can ID # and data offset used to parse the specific sensor's data class CANParser(): def __init__(self): - self.sensorLib = self.create_lib() - - self.saveFile = csv.DictWriter(open('SensorData.csv', "a"), - fieldnames=["timestamp", "id", "data"], - extrasaction="ignore") - - # Parses CSV file of individual sensors to a dict that is easily - # navigitable for parsing incoming data. - # list format: - # id : {'name1' : [msb, lsb], 'name2' : [msb, lsb], ...} - def create_lib(self): - sensorLibraryFile = csv.reader(open('SensorDict.csv')) - sensorIdList = {} - - for row in sensorLibraryFile: - if not row or row[0] == 'id': - continue - - canID = int(row[0], 16) - msb = int(row[1]) - lsb = int(row[2]) - friendlyName = row[3].strip() - - if canID not in sensorIdList: - sensorIdList[canID] = {} - sensorIdList[canID][friendlyName] = [msb, lsb] + database = Database() - return sensorIdList # Takes in a single data point with the format: # {"timestamp" : "XXX", "id" : "XXX", "data" : "XXXX"} diff --git a/can_mgmt/update-SensorDict.py b/can_mgmt/update-SensorDict.py index 6adcb8f..9e3e20e 100644 --- a/can_mgmt/update-SensorDict.py +++ b/can_mgmt/update-SensorDict.py @@ -52,13 +52,10 @@ flags_list = item.split("\n") for flag_i, flag in enumerate(flags_list): flag_paren_start = flag.find("(") - try: - flag_bit = str(int(flag[flag_paren_start + 1]) + + flag_bit = str(int(flag[flag_paren_start + 1]) + 8 * int(column[4])) - except: - continue flag = flag[0:flag_paren_start - 1] - flags_df = flags_df.append({"id": index.lower(), "MSB": flag_bit, + flags_df = flags_df.append({"id": index.upper(), "MSB": flag_bit, "LSB": flag_bit, "name": flag}, ignore_index=True) continue @@ -111,10 +108,7 @@ row["LSB"])) last_ID = ID continue - try: - last_underscore = sensor_name_and_type.rindex("_") - except: - continue + last_underscore = sensor_name_and_type.rindex("_") sensor_name = sensor_name_and_type[0:last_underscore] if last_ID != ID: savefile_h.write("\n#define {}_CAN_ID {}\n".format(sensor_name, ID))