Skip to content

Resume execution of workers #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Amphitryon0 wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Resume execution of workers #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
724fe4d
Convert calculation function to a generator
Amphitryon0 Jul 18, 2020
f6845dc
Share frame record between processes
Amphitryon0 Jul 18, 2020
2d5e821
Merge branch 'master' into resume-multithreading
Amphitryon0 Jul 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 80 additions & 45 deletions calculator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import copy
import random
from operator import itemgetter
from math import inf
from logger import log
from inventory import getAlphaSort, getTypeSort, remainingOutputsCanBeFulfilled
from moves import getInsertionIndex
Expand Down Expand Up @@ -27,25 +29,15 @@ def printResults(filename, writtenStep, framesTaken, totalFrames, inventory, out
log(5, "Calculator", "File", "Write", "Data for Step " + str(writtenStep[i]) + " written.")
file.close()

#Return the sorted version of the inventory, as dictated by the full sorted dictionary
#Also which direction to sort the inventory
def getSortedInventory(inventory, full_sorted_list, is_reversed):
sorted_inventory = []

if(is_reversed):
for i in range(len(full_sorted_list)-1,-1,-1):
for j in range(0,inventory.count(full_sorted_list[i])):
sorted_inventory.append(full_sorted_list[i])
else:
for i in range(0,len(full_sorted_list)):
for j in range(0,inventory.count(full_sorted_list[i])):
sorted_inventory.append(full_sorted_list[i])

#Remaining Spaces are "Blocked"
while(len(sorted_inventory) < 20):
sorted_inventory.append("BLOCKED")

return sorted_inventory
#Sort the inventory according to the ordering parameters given,
#transforming any blank spaces to unusable spaces at the end
def getSortedInventory(inventory, sort_positions, reversed):
return list(map(itemgetter(1),
sorted([(sort_positions[item], item)
if item != "NULL" and item != "BLOCKED"
else (-inf if reversed else inf, "BLOCKED")
for item in inventory],
key=itemgetter(0), reverse=reversed)))

def handleChapter5EarlySortEndItems(legal_moves, step_index, inventory, tempOutputsFulfilled, recipeList, itemNames, invFrames, SORT_FRAMES, sort_name, temp_frames_DB, temp_frames_CO, DB_place_index, CO_place_index):
TOSS_FRAMES = 23
Expand Down Expand Up @@ -158,7 +150,10 @@ def handleChapter5LateSortEndItems(legal_moves, step_index, inventory, tempOutpu

#Evaluates all possible placements of the Keel Mango and Courage Shell
#And all possible locations and types of sorting that can place the Coconut into a position where it can be duplicated
def handleChapter5Eval(legal_moves, step_index, temp_inventory, tempOutputsFulfilled, recipeList, itemNames, invFrames, temp_frames_DB, temp_frames_CO, full_alpha_list, full_type_list):
def handleChapter5Eval(legal_moves, step_index, temp_inventory,
tempOutputsFulfilled, recipeList, itemNames, invFrames,
temp_frames_DB, temp_frames_CO, alpha_sort_positions,
type_sort_positions):
#Various Frame Counts
TOSS_FRAMES = 23
ALPHA_SORT_FRAMES = 38
Expand All @@ -175,7 +170,8 @@ def handleChapter5Eval(legal_moves, step_index, temp_inventory, tempOutputsFulfi
#======================================

#Alphabetically Sorted Inventory
alpha_inventory = getSortedInventory(temp_inventory, full_alpha_list, False)
alpha_inventory = getSortedInventory(
temp_inventory, alpha_sort_positions, reversed=False)

#Only bother with further evaluation if the sort placed the Coconut in the latter half of the inventory
#Because the coconut is needed for duplication
Expand All @@ -196,7 +192,8 @@ def handleChapter5Eval(legal_moves, step_index, temp_inventory, tempOutputsFulfi
CO_place_index)

#Reverse Alphabetical Sorted Inventory
reverse_alpha_inventory = getSortedInventory(temp_inventory, full_alpha_list, True)
reverse_alpha_inventory = getSortedInventory(
temp_inventory, alpha_sort_positions, reversed=True)

#Only bother with further evaluation if the sort placed the Coconut in the latter half of the inventory
#Because the coconut is needed for duplication
Expand All @@ -217,7 +214,8 @@ def handleChapter5Eval(legal_moves, step_index, temp_inventory, tempOutputsFulfi
CO_place_index)

#Type Sorted Inventory
type_inventory = getSortedInventory(temp_inventory, full_type_list, False)
type_inventory = getSortedInventory(
temp_inventory, type_sort_positions, reversed=False)

#Only bother with further evaluation if the sort placed the Coconut in the latter half of the inventory
#Because the coconut is needed for duplication
Expand All @@ -238,7 +236,8 @@ def handleChapter5Eval(legal_moves, step_index, temp_inventory, tempOutputsFulfi
CO_place_index)

#Reverse Type Sorted Inventory
reverse_type_inventory = getSortedInventory(temp_inventory, full_type_list, True)
reverse_type_inventory = getSortedInventory(
temp_inventory, type_sort_positions, reversed=True)

#Only bother with further evaluation if the sort placed the Coconut in the latter half of the inventory
#Because the coconut is needed for duplication
Expand Down Expand Up @@ -286,7 +285,8 @@ def handleChapter5Eval(legal_moves, step_index, temp_inventory, tempOutputsFulfi

#Perform all sorts
#Alphabetically Sorted Inventory
alpha_inventory = getSortedInventory(temp_inventory, full_alpha_list, False)
alpha_inventory = getSortedInventory(
temp_inventory, alpha_sort_positions, reversed=False)

#Only bother with further evaluation if the sort placed the Coconut in the latter half of the inventory
#Because the coconut is needed for duplication
Expand All @@ -309,7 +309,8 @@ def handleChapter5Eval(legal_moves, step_index, temp_inventory, tempOutputsFulfi
KM_place_index)

#Reverse Alphabetical Sorted Inventory
reverse_alpha_inventory = getSortedInventory(temp_inventory, full_alpha_list, True)
reverse_alpha_inventory = getSortedInventory(
temp_inventory, alpha_sort_positions, reversed=True)

#Only bother further evaluation if the sort placed the Coconut in the latter half of the inventory
#Because the coconut is needed for duplication
Expand All @@ -332,7 +333,8 @@ def handleChapter5Eval(legal_moves, step_index, temp_inventory, tempOutputsFulfi
KM_place_index)

#Type Sorted Inventory
type_inventory = getSortedInventory(temp_inventory, full_type_list, False)
type_inventory = getSortedInventory(
temp_inventory, type_sort_positions, reversed=False)

#Only bother further evaluation if the sort placed the Coconut in the latter half of the inventory
#Because the coconut is needed for duplication
Expand All @@ -355,7 +357,8 @@ def handleChapter5Eval(legal_moves, step_index, temp_inventory, tempOutputsFulfi
KM_place_index)

#Reverse Type Sorted Inventory
reverse_type_inventory = getSortedInventory(temp_inventory, full_type_list, True)
reverse_type_inventory = getSortedInventory(
temp_inventory, type_sort_positions, reversed=True)

#Only bother further evaluation if the sort placed the Coconut in the latter half of the inventory
#Because the coconut is needed for duplication
Expand All @@ -380,7 +383,7 @@ def handleChapter5Eval(legal_moves, step_index, temp_inventory, tempOutputsFulfi
#Return the replaced items for the next loop
temp_inventory[KM_place_index] = KM_replacement

def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList, invFrames):
def calculateOrder(callNumber, frameRecord, startingInventory, recipeList, invFrames):
itemNames = []
#Fill the itemNames
for item in recipeList:
Expand All @@ -389,6 +392,12 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
log(5, "Calculator", "Items", "Scan", "All item names scanned successfully.")
randomise = bool(getConfig("randomise") == "True")
select = bool(getConfig("select") == "True")

alpha_sort_positions = {itemName: index for index, itemName
in enumerate(getAlphaSort())}
type_sort_positions = {itemName: index for index, itemName
in enumerate(getTypeSort())}

#===============================================================================
# GOAL
#===============================================================================
Expand Down Expand Up @@ -443,9 +452,6 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
#If the player does not toss the final output item, 5 extra frames are needed to obtain jump storage
JUMP_STORAGE_NO_TOSS_FRAMES = 5

sorted_alpha_list = getAlphaSort()
sorted_type_list = getTypeSort()

#start main loop
while(True):
stepIndex = 0
Expand Down Expand Up @@ -490,15 +496,16 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
totalFrames[-1] += JUMP_STORAGE_NO_TOSS_FRAMES
else:
writtenStep[-1] += " (Jump Storage on Tossed Item)"

if(totalFrames[stepIndex] < currentFrameRecord):

#Technically a race condition.
if(totalFrames[stepIndex] < frameRecord.value):
#New Record!
currentFrameRecord = totalFrames[stepIndex]
frameRecord.value = totalFrames[stepIndex]
#print("New Record Time: {0}".format(totalFrames[stepIndex]))
#Log the updated outcome
printResults("results/[{0}].txt".format(totalFrames[stepIndex]),writtenStep,framesTaken,totalFrames,inventory,outputCreated,itemNames,stepIndex)

return [totalFrames[stepIndex], callNumber]
yield [totalFrames[stepIndex], callNumber]

#Regardless of record status, its time to go back up and find new endstates
#Wipe away the current state
Expand Down Expand Up @@ -718,7 +725,11 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
temp_frames_CO = 0

#Handle the Allocation of the Coconut Sort, Keel Mango, and Courage Shell
handleChapter5Eval(legalMoves, stepIndex, tempInventory, tempOutputsFulfilled, recipeList, itemNames, invFrames, temp_frames_DB, temp_frames_CO, sorted_alpha_list, sorted_type_list)
handleChapter5Eval(
legalMoves, stepIndex, tempInventory,
tempOutputsFulfilled, recipeList, itemNames,
invFrames, temp_frames_DB, temp_frames_CO,
alpha_sort_positions, type_sort_positions)

elif(tempInventory.count("NULL") == 1):
#The Dried Bouquet gets Auto Placed in the 1st available NULL
Expand All @@ -741,7 +752,11 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
temp_frames_CO = TOSS_FRAMES + invFrames[viableItems][tempindexCO]

#Handle the Allocation of the Coconut Sort, Keel Mango, and Courage Shell
handleChapter5Eval(legalMoves, stepIndex, tempInventory, tempOutputsFulfilled, recipeList, itemNames, invFrames, temp_frames_DB, temp_frames_CO, sorted_alpha_list, sorted_type_list)
handleChapter5Eval(
legalMoves, stepIndex, tempInventory,
tempOutputsFulfilled, recipeList, itemNames,
invFrames, temp_frames_DB, temp_frames_CO,
alpha_sort_positions, type_sort_positions)

#Reset what was previously in the Coconut's slot
tempInventory[tempindexCO] = tossed_item_1
Expand Down Expand Up @@ -773,7 +788,13 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
temp_frames_CO = TOSS_FRAMES + invFrames[viableItems][tempindexCO]

#Handle the Allocation of the Coconut Sort, Keel Mango, and Courage Shell
handleChapter5Eval(legalMoves, stepIndex, tempInventory, tempOutputsFulfilled, recipeList, itemNames, invFrames, temp_frames_DB, temp_frames_CO, sorted_alpha_list, sorted_type_list)
handleChapter5Eval(
legalMoves, stepIndex,
tempInventory, tempOutputsFulfilled,
recipeList, itemNames, invFrames,
temp_frames_DB, temp_frames_CO,
alpha_sort_positions,
type_sort_positions)

#Reset what was preciously in the Coconut's Slot
tempInventory[tempindexCO] = tossed_item_2
Expand All @@ -797,7 +818,9 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
if(total_sorts <= 15):

#Alphabetical Sort
alpha_inventory = getSortedInventory(inventory[stepIndex], sorted_alpha_list, False)
alpha_inventory = getSortedInventory(
inventory[stepIndex], alpha_sort_positions,
reversed=False)

#Only add the legal move if the sort actually changes the inventory
if(alpha_inventory != inventory[stepIndex]):
Expand All @@ -810,7 +833,9 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
legalMoves[stepIndex].append([description,-1,ALPHA_SORT_FRAMES,alpha_inventory])

#Reverse Alphabetical Sort
reverse_alpha_inventory = getSortedInventory(inventory[stepIndex], sorted_alpha_list, True)
reverse_alpha_inventory = getSortedInventory(
inventory[stepIndex], alpha_sort_positions,
reversed=True)

#Only add the legal move if the sort actually changes the inventory
if(reverse_alpha_inventory != inventory[stepIndex]):
Expand All @@ -823,7 +848,9 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
legalMoves[stepIndex].append([description,-1,REVERSE_ALPHA_SORT_FRAMES,reverse_alpha_inventory])

#Type Sort
type_inventory = getSortedInventory(inventory[stepIndex], sorted_type_list, False)
type_inventory = getSortedInventory(
inventory[stepIndex], type_sort_positions,
reversed=False)

#Only add the legal move if the sort actually changes the inventory
if(type_inventory != inventory[stepIndex]):
Expand All @@ -836,7 +863,9 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
legalMoves[stepIndex].append([description,-1,TYPE_SORT_FRAMES,type_inventory])

#Reverse Type Sort
reverse_type_inventory = getSortedInventory(inventory[stepIndex], sorted_type_list, True)
reverse_type_inventory = getSortedInventory(
inventory[stepIndex], type_sort_positions,
reversed=True)

#Only add the legal move if the sort actually changes the inventory
if(reverse_type_inventory != inventory[stepIndex]):
Expand All @@ -853,7 +882,10 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
#=====================================

#Filter out all legal moves that would exceed the current frame limit
legalMoves[stepIndex] = list(filter(lambda x: x[2]+totalFrames[stepIndex] < currentFrameRecord, legalMoves[stepIndex]))
stepFramesCutoff = frameRecord.value - totalFrames[stepIndex]
legalMoves[stepIndex] = list(filter(lambda x: x[2]
< stepFramesCutoff,
legalMoves[stepIndex]))

#Filter out all legal moves that use 2 ingredients in the very first legal move
if(stepIndex == 0):
Expand Down Expand Up @@ -901,7 +933,10 @@ def calculateOrder(callNumber, currentFrameRecord, startingInventory, recipeList
legalMoves[stepIndex].pop(0)

#Filter out all legal moves that would exceed the current frame limit
legalMoves[stepIndex] = list(filter(lambda x: x[2]+totalFrames[stepIndex] < currentFrameRecord, legalMoves[stepIndex]))
stepFramesCutoff = frameRecord.value - totalFrames[stepIndex]
legalMoves[stepIndex] = list(filter(lambda x: x[2]
< stepFramesCutoff,
legalMoves[stepIndex]))

#Just because, if the step index is sufficiently small, just shuffle!
if(randomise):
Expand Down
55 changes: 22 additions & 33 deletions start.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,56 +1,45 @@
import multiprocessing
from ctypes import c_int
from calculator import calculateOrder
from inventory import getStartingInventory, getInventoryFrames
from recipes import getRecipeList
from config import getConfig
from logger import log
from FTPManagement import getFastestRecordOnFTP, testRecord

def worker(workQueue, doneQueue):
while(True):
job = workQueue.get(True)
#waiting for a job to appear
#in this case job refers to a single instance of calculating the recipe order
result = calculateOrder(job[0], job[1], job[2], job[3], job[4])
def worker(doneQueue, *args):
for result in calculateOrder(*args):
#write the calculated result to the "done" queue
doneQueue.put(result, False)

def work(currentFrameRecord, startingInventory, recipeList, invFrames):
#create queues
def work(frameRecord, startingInventory, recipeList, invFrames):
#create queue for results to be pushed to
doneQueue = multiprocessing.Queue(workerCount)
workQueue = multiprocessing.Queue(workerCount)
#create array for all the instances that are running
instances = []
#start instances
#start workers
for i in range(workerCount):
instance = multiprocessing.Process(target=worker, args=(workQueue, doneQueue))
instance = multiprocessing.Process(
target=worker, args=(doneQueue, i, frameRecord, startingInventory,
recipeList, invFrames))
instance.daemon = True
instance.start()
instances.append(instance)
#start jobs
for i in range (workerCount):
job = [i, currentFrameRecord, startingInventory, recipeList, invFrames]
workQueue.put(job, False)
#waiting for first result
result = doneQueue.get(True)
#terminate the other instances still running
for instance in instances:
instance.terminate()
return result
#wait for each result from the workers
while True:
yield doneQueue.get(True)

if __name__ == '__main__':
frameRecord = multiprocessing.Value(c_int)
frameRecord.value = getFastestRecordOnFTP()
cycle_count = 1
startingInventory = getStartingInventory()
recipeList = getRecipeList()
invFrames = getInventoryFrames()
workerCount = int(getConfig("workerCount"))
while(True):
currentFrameRecord = getFastestRecordOnFTP()
#start the work
result = work(currentFrameRecord, startingInventory, recipeList, invFrames)
#sanity check
if(result[0] < currentFrameRecord):
testRecord(result[0])
currentFrameRecord = result[0]
log(1, "Main", "Results", "", 'cycle {0} done, current record: {1} frames. Record on call {2}.'.format(cycle_count, currentFrameRecord, result[1]))
#start the work
for result in work(frameRecord, startingInventory, recipeList,
invFrames):
testRecord(result[0])
log(1, "Main", "Results", "",
'cycle {0} done, current record: {1} frames. Record on call {2}.'
.format(cycle_count, result[0], result[1]))
frameRecord.value = getFastestRecordOnFTP()
cycle_count += 1