Skip to content

Migrate solver execution logging in DataFileClass.run() from print() to logging #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
parthdagia05 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Migrate solver execution logging in DataFileClass.run() from print() to logging #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 14 additions & 9 deletions API/Classes/Case/DataFileClass.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,12 @@
import pandas as pd
import traceback
import json, shutil, os, time, subprocess
import logging
from collections import defaultdict
from itertools import product

logger = logging.getLogger(__name__)

from Classes.Base import Config
from Classes.Case.OsemosysClass import Osemosys
from Classes.Base.FileClass import File
Expand Down Expand Up @@ -2127,8 +2130,9 @@ def run( self, solver, caserun, lock=None ):
#PREPROCESS data.txt
#subprocess.run('preprocess_data.py' + datafile + dataFile_processed)

logger.debug("Starting preprocessing step for case %s", caserunname)
self.preprocessData(self.dataFile, self.dataFile_processed)
print("PREPROCESSING DONE! --- %s seconds --- %s" % (time.time() - start_time, caserunname))
logger.info("PREPROCESSING DONE! --- %s seconds --- %s", time.time() - start_time, caserunname)
txtOut = txtOut + ("Preprocessing time {:0.2f}s;{}".format(time.time() - start_time, '\n'))

#return output to variable preprocessed data file
Expand All @@ -2141,7 +2145,7 @@ def run( self, solver, caserun, lock=None ):
#original data file without preprocessing
#glpk_out = subprocess.run('glpsol --check -m ' + modelfile_original +' -d ' + datafile +' --wlp ' + lpfile, cwd=glpfolder, capture_output=True, text=True, shell=True)

print("CREATINON OF LP FILE DONE! --- %s seconds --- %s" % (time.time() - start_time, caserunname))
logger.info("CREATINON OF LP FILE DONE! --- %s seconds --- %s", time.time() - start_time, caserunname)
txtOut = txtOut + ("Creation of LP file time {:0.2f}s;{}".format(time.time() - start_time, '\n'))


Expand All @@ -2157,9 +2161,10 @@ def run( self, solver, caserun, lock=None ):

#cbc_out = subprocess.run('cbc ' + lpfile +' -presolve off -postsolve on -logLevel 3 solve -printing all -solu ' + resultfile, cwd=cbcfolder, capture_output=True, text=True, shell=True)
# prin
logger.debug("Running CBC solver in directory %s", cbcfolder)
cbc_out = subprocess.run('cbc ' + lpfile +' solve -printing all -solu ' + resultfile, cwd=cbcfolder, capture_output=True, text=True, shell=True)
# -printing all prints all constraints to result.txt
print("SOLUTION DONE! --- %s seconds --- %s" % (time.time() - start_time, caserunname))
logger.info("SOLUTION DONE! --- %s seconds --- %s", time.time() - start_time, caserunname)
txtOut = txtOut + ("Solution time {:0.2f}s;{}".format(time.time() - start_time, '\n'))
####output to lg file .log i .txt with errors
# out = subprocess.run('cbc ' + lpfile +' solve -solu ' + resultfile +'>'+ logfile, cwd=cbcfolder, capture_output=True, text=True, shell=True)
Expand Down Expand Up @@ -2204,14 +2209,14 @@ def run( self, solver, caserun, lock=None ):

if statusFlag == "success":
self.generateCSVfromCBC(self.dataFile, self.resFile, self.resPath)
print("CSV DONE! --- %s seconds --- %s" % (time.time() - start_time, caserunname))
logger.info("CSV DONE! --- %s seconds --- %s", time.time() - start_time, caserunname)
txtOut = txtOut + ("csv files extraction time {:0.2f} s;{}".format(time.time() - start_time, '\n'))
self.generateResultsViewer(caserunname)
print("PIVOT TABLE DONE! --- %s seconds --- %s" % (time.time() - start_time, caserunname))
logger.info("PIVOT TABLE DONE! --- %s seconds --- %s", time.time() - start_time, caserunname)
txtOut = txtOut + ("Pivot data preparation time {:0.2f}s;{}".format(time.time() - start_time, '\n'))


print("MESSAGES DONE! --- %s seconds --- %s" % (time.time() - start_time, caserunname))
logger.info("MESSAGES DONE! --- %s seconds --- %s", time.time() - start_time, caserunname)
txtOut = txtOut + ("Message preparation time {:0.2f}s;{}".format(time.time() - start_time, '\n'))

response = {
Expand All @@ -2231,9 +2236,9 @@ def run( self, solver, caserun, lock=None ):
return response
# urllib.request.urlretrieve(self.dataFile, dataFile)

except Exception as ex:
print(ex) # do whatever you want for debugging.
raise # re-raise exception.
except Exception:
logger.exception("Unhandled exception during solver execution")
raise
except(IOError, IndexError):
raise IndexError
except OSError:
Expand Down
7 changes: 7 additions & 0 deletions API/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,13 @@
from datetime import timedelta
# from pathlib import Path

import logging
if not logging.getLogger().handlers:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(name)s] %(levelname)s: %(message)s"
)

#import json
from Classes.Base import Config
# from API.Classes.Base.SyncS3 import SyncS3
Expand Down