diff --git a/common/data/Glicko.lua b/common/data/Glicko.lua new file mode 100644 index 00000000..82875d44 --- /dev/null +++ b/common/data/Glicko.lua @@ -0,0 +1,342 @@ +-- Credit to "SpaceCube" for implementing the lua implementations +-- https://devforum.roblox.com/t/a-lua-implementation-of-the-glicko-2-rating-algorithm-for-skill-based-matchmaking/1442673 +-- 1 small bug fix is included: to1:() returned the wrong version + +local util = require("common.lib.util") + +--http://www.glicko.net/glicko/glicko2.pdf +local c = 173.7178 -- Used for conversion between Glicko1 and 2, not to be confused with c from Glicko1 +local epsilon = 1e-6 -- Convergence + +-- Shortcuts for commonly used math functions +local exp = math.exp +local sqrt = math.sqrt +local log = math.log + +-- Glicko2 +local Glicko2 = { + Tau = 0.5, -- Slider for volatility Smaller values prevent the volatility measures from changing by large + -- amounts, which in turn prevent enormous changes in ratings based on very improbable results. + InitialVolatility = 0.06 +}; Glicko2.__index = Glicko2 + +-- Creates a Glicko rating with a specified version +function Glicko2.gv(Rating, RatingDeviation, Volatility, Version) + local self = { + Rating = Rating, + RD = RatingDeviation, + Vol = Volatility or Glicko2.InitialVolatility, + Version = Version or 2, + } + + return setmetatable(self, Glicko2) +end + +-- Creates a Glicko1 rating +function Glicko2.g1(Rating, RatingDeviation, Volatility) + return Glicko2.gv( + Rating or 1500, + RatingDeviation or 350, + Volatility, + 1 + ) +end + +-- Creates a Glicko2 rating +function Glicko2.g2(Rating, RatingDeviation, Volatility) + return Glicko2.gv( + Rating or 0, + RatingDeviation or (350/c), + Volatility, + 2 + ) +end + +function Glicko2:copy() + return Glicko2.gv(self.Rating, self.RD, self.Vol, self.Version) +end + +-- Scales glicko rating to Glicko2 +function Glicko2:to2() + if self.Version == 2 then + return self:copy() + end + + local g2 = Glicko2.g2((self.Rating - 1500)/c, self.RD/c, self.Vol) + + if self.Score then + g2.Score = self.Score + end + + return g2 +end + +-- Scales glicko rating to Glicko1 +function Glicko2:to1() + if self.Version == 1 then + return self:copy() + end + + local g1 = Glicko2.g1(self.Rating*c + 1500, self.RD*c, self.Vol) + + if self.Score then + g1.Score = self.Score + end + + return g1 +end + +function Glicko2.serialize(gv) + return { + gv.Rating, + gv.RD, + gv.Vol, + gv.Score + } +end + +function Glicko2.deserialize(gv_s, version) + local constructor = nil + + -- Finds glicko constructor for specified version + if version == 1 then + constructor = Glicko2.g1 + elseif version == 2 then + constructor = Glicko2.g2 + else + error("Version must be specified for deserialization", 2) + end + + local gv = constructor(gv_s[1], gv_s[2], gv_s[3]) + + -- Inserts a score if there is one + if gv_s[4] then + gv = gv:score(gv_s[4]) + end + + return gv +end + +function Glicko2.updatedRatings(player1, player2, matchOutcomes, elapsedRatingPeriods) + + local player1Results = {} + local player2Results = {} + for i = 1, #matchOutcomes do + local matchOutcome = matchOutcomes[i] + player1Results[#player1Results+1] = player2:score(matchOutcome) + if matchOutcome == 0 then + matchOutcome = 1 + elseif matchOutcome == 1 then + matchOutcome = 0 + end + player2Results[#player2Results+1] = player1:score(matchOutcome) + end + + local updatedPlayer1 = player1:update(player1Results, elapsedRatingPeriods) + local updatedPlayer2 = player2:update(player2Results, elapsedRatingPeriods) + + return updatedPlayer1, updatedPlayer2 +end + +-- Attaches a score to an opponent +function Glicko2:score(score) + local new_g2 = self:copy() + + --lost: 0, win: 1, tie: 0.5 + new_g2.Score = score or 0 + + return new_g2 +end + +-- Function g as described in step 3 +local function g(RD) + return 1/sqrt(1 + 3*RD^2/math.pi^2) +end + +-- Function E as described in step 3 +local function E(rating, opRating, opRD) + return 1/(1 + exp(-g(opRD)*(rating - opRating))) +end + +-- Constructor for function f described in step 5 +local function makebigf(g2, v, delta) + local a = log(g2.Vol^2) + + return function(x) + local numer = exp(x)*(delta^2 - g2.RD^2 - v - exp(x)) --numerator + local denom = 2*(g2.RD^2 + v + exp(x))^2 --denominator + local endTerm = (x - a)/(Glicko2.Tau^2) --final term + + return numer/denom - endTerm + end +end + +-- Updates a Glicko rating using the last set of matches +function Glicko2:update(matches, elapsedRatingPeriods) + local g2 = self + local originalVersion = g2.Version + + -- convert ratings to glicko2 + if originalVersion == 1 then + g2 = g2:to2() + end + + local convertedMatches = {} + for i, match in ipairs(matches) do + if match.Version == 1 then + convertedMatches[i] = match:to2() + end + end + + -- step 3: compute v + local v = 0 + + for j, match in ipairs(convertedMatches) do + local EValue = E(g2.Rating, match.Rating, match.RD) + + v = v + g(match.RD)^2*EValue*(1 - EValue) + end + + v = 1/v + + -- step 4: compute delta + local delta = 0 + + for j, match in ipairs(convertedMatches) do + local EValue = E(g2.Rating, match.Rating, match.RD) + + delta = delta + g(match.RD)*(match.Score - EValue) + end + + delta = delta*v + + -- step 5: find new volatility (iterative process) + local a = log(g2.Vol^2) + + local bigf = makebigf(g2, v, delta) + + -- step 5.2: find initial A and B values + local A = a + local B = 0 + + if delta^2 > g2.RD^2 + v then + B = log(delta^2 - g2.RD^2 - v) + else + --iterative process for solving B + local k = 1 + + while bigf(a - k*Glicko2.Tau) < 0 do + k = k + 1 + end + + B = a - k*Glicko2.Tau + end + + -- step 5.3: compute values of bigf of A and B + local fA = bigf(A) + local fB = bigf(B) + + -- step 5.4: iterates until A and B converge + while math.abs(B - A) > epsilon do + local C = A + (A - B)*fA/(fB - fA) + local fC = bigf(C) + + if fC*fB <= 0 then + A = B + fA = fB + else + fA = fA/2 + end + + B = C + fB = fC + end + + -- step 5.5: set new volatility + local newVol = g2.Vol + + if #convertedMatches > 0 then + newVol = exp(A/2) + end + + -- step 6: update the rating deviation to the new pre-rating period value + local ratingDeviation = self:calculateNewRD(g2.RD, newVol, elapsedRatingPeriods) + + -- Step 7: Update to the new rating + + local newRD = 1/sqrt(1/ratingDeviation^2 + 1/v) + local newRating = g2.Rating + + if #convertedMatches > 0 then + local accumulation = 0 + for j, match in ipairs(convertedMatches) do + local EValue = E(g2.Rating, match.Rating, match.RD) + accumulation = accumulation + g(match.RD)*(match.Score - EValue) + end + + newRating = g2.Rating + newRD^2*accumulation + end + + --wrap up results + local result = Glicko2.g2(newRating, newRD, newVol) + + if originalVersion == 1 then + result = result:to1() + end + + return result +end + +-- This is the formula defined in step 6. It is also used for players who have not competed during the rating period +function Glicko2:calculateNewRD(ratingDeviation, volatility, elapsedRatingPeriods) + local ratingDeviation = sqrt(ratingDeviation^2 + elapsedRatingPeriods * volatility^2) + return ratingDeviation +end + +function Glicko2:deviation(deviations) + deviations = deviations or 2 + local radius = self.RD*deviations + + return self.Rating - radius, self.Rating + radius +end + +function Glicko2:expectedOutcome(otherGlicko) + local g2 = self + local otherGlicko2 = otherGlicko + + -- convert ratings to glicko2 + if g2.Version == 1 then + g2 = g2:to2() + end + if otherGlicko2.Version == 1 then + otherGlicko2 = otherGlicko2:to2() + end + + local function A(glicko1, glicko2) + return g(sqrt(glicko1.RD^2+glicko2.RD^2)) * (glicko2.Rating-glicko1.Rating) + end + + local function myFunc(glicko1, glicko2) + return 1/(1 + exp(-A(glicko1, glicko2))) + end + + local result = myFunc(otherGlicko2, g2) + return result +end + +function Glicko2:range(padding) + padding = padding or 0 + local small, big = self:deviation() + + return small - padding, big + padding +end + +function Glicko2:percent(confidence) + confidence = util.bound(0, confidence, 1) + assert(confidence < 1, "Percentage cannot be equal or greater than 1") + + --This is a simple inverse erf approximation, has accuracy of +- 0.02 + return self:deviation(.5877*math.log((1 + confidence)/(1 - confidence))) +end + +return Glicko2 diff --git a/common/data/PlayerRating.lua b/common/data/PlayerRating.lua new file mode 100644 index 00000000..32d9eeb2 --- /dev/null +++ b/common/data/PlayerRating.lua @@ -0,0 +1,152 @@ +local Glicko2 = require("common.data.Glicko") +local class = require("common.lib.class") + +-- Represents the rating for a player +PlayerRating = + class( + function(self, rating, ratingDeviation, maxRatingDeviation, volatility, maxVolatility) + rating = rating or self.STARTING_RATING + ratingDeviation = ratingDeviation or self.STARTING_RATING_DEVIATION + self.maxRatingDeviation = maxRatingDeviation or PlayerRating.MAX_RATING_DEVIATION + volatility = volatility or self.STARTING_VOLATILITY + self.maxVolatility = maxVolatility or PlayerRating.MAX_VOLATILITY + self.glicko = Glicko2.g1(rating, ratingDeviation, volatility) + self.lastRatingPeriodCalculated = nil + end +) + +PlayerRating.RATING_PERIOD_IN_SECONDS = 60 * 60 * 16 +PlayerRating.ALLOWABLE_RATING_SPREAD = 400 + +PlayerRating.STARTING_RATING = 1500 + +PlayerRating.STARTING_RATING_DEVIATION = 200 +PlayerRating.MAX_RATING_DEVIATION = PlayerRating.STARTING_RATING_DEVIATION +PlayerRating.PROVISIONAL_RATING_DEVIATION = 125 + +PlayerRating.STARTING_VOLATILITY = 0.06 +PlayerRating.MAX_VOLATILITY = 0.3 + +-- Returns the rating period number for the given timestamp +function PlayerRating.ratingPeriodForTimeStamp(timestamp) + local ratingPeriod = timestamp / (PlayerRating.RATING_PERIOD_IN_SECONDS) + return ratingPeriod +end + +function PlayerRating.timestampForRatingPeriod(ratingPeriod) + local timestamp = ratingPeriod * PlayerRating.RATING_PERIOD_IN_SECONDS + return timestamp +end + +function PlayerRating:copy() + local result = deepcpy(self) + return result +end + +function PlayerRating:getRating() + return self.glicko.Rating +end + +-- Returns a percentage value (0-1) of how likely the rating thinks the player is to win +function PlayerRating:expectedOutcome(opponent) + return self.glicko:expectedOutcome(opponent.glicko) +end + +-- Returns if the player is still "provisional" +-- Provisional is only used to change how the UI looks to help the player know this rating is not accurate yet. +function PlayerRating:isProvisional() + return self.glicko.RD >= PlayerRating.PROVISIONAL_RATING_DEVIATION +end + +-- Returns an array of wins vs total games +-- The wins / losses will be distributed by the ratio to try to balance it out. +-- Really only meant for testing. +function PlayerRating.createTestSetResults(player1WinCount, gameCount) + + assert(gameCount >= player1WinCount) + + local matchSet = {} + for i = 1, gameCount - player1WinCount, 1 do + matchSet[#matchSet+1] = 0 + end + + local step = gameCount / player1WinCount + local position = 1 + for i = 1, player1WinCount, 1 do + matchSet[math.round(position)] = 1 + position = position + step + end + + return matchSet +end + +-- Helper function to create one game result with the given outcome if the players are allowed to rank. +function PlayerRating:createGameResult(opponent, matchOutcome) + local result = nil + + if math.abs(self:getRating() - opponent:getRating()) <= PlayerRating.ALLOWABLE_RATING_SPREAD then + result = opponent.glicko:score(matchOutcome) + end + + return result +end + +function PlayerRating.invertedGameResult(gameResult) + if gameResult == 0 then + return 1 + end + if gameResult == 1 then + return 0 + end + -- Ties stay 0.5 + return gameResult +end + +function PlayerRating:newRatingForResultsAndLatestRatingPeriod(gameResult, latestRatingPeriodFound) + local updatedPlayer = self + if updatedPlayer.lastRatingPeriodCalculated == nil then + updatedPlayer = self:copy() + updatedPlayer.lastRatingPeriodCalculated = latestRatingPeriodFound + end + local elapsedRatingPeriods = latestRatingPeriodFound - updatedPlayer.lastRatingPeriodCalculated + if elapsedRatingPeriods > 0 then + updatedPlayer = updatedPlayer:newRatingForResultsAndElapsedRatingPeriod(gameResult, elapsedRatingPeriods) + end + + return updatedPlayer +end + +-- Runs the given results for the player with the given elapsedRatingPeriod +function PlayerRating:newRatingForResultsAndElapsedRatingPeriod(gameResult, elapsedRatingPeriods) + local updatedPlayer = self + + if elapsedRatingPeriods > 0 then + updatedPlayer = updatedPlayer:privateNewRatingForResultWithElapsedRatingPeriod(nil, elapsedRatingPeriods) + if updatedPlayer.lastRatingPeriodCalculated then + updatedPlayer.lastRatingPeriodCalculated = self.lastRatingPeriodCalculated + elapsedRatingPeriods + end + end + + updatedPlayer = updatedPlayer:privateNewRatingForResultWithElapsedRatingPeriod(gameResult, 0) + + return updatedPlayer +end + +function PlayerRating:privateNewRatingForResultWithElapsedRatingPeriod(gameResult, elapsedRatingPeriods) + local gameResults = {} + if gameResult then + gameResults[#gameResults+1] = gameResult + end + local updatedGlicko = self.glicko:update(gameResults, elapsedRatingPeriods) + if updatedGlicko.RD > self.maxRatingDeviation then + updatedGlicko.RD = self.maxRatingDeviation + end + if updatedGlicko.Vol > self.maxVolatility then + updatedGlicko.Vol = self.maxVolatility + end + local updatedPlayer = self:copy() + updatedPlayer.glicko = updatedGlicko + return updatedPlayer +end + +return PlayerRating diff --git a/common/tests/data/GlickoTests.lua b/common/tests/data/GlickoTests.lua new file mode 100644 index 00000000..10024852 --- /dev/null +++ b/common/tests/data/GlickoTests.lua @@ -0,0 +1,108 @@ +local Glicko2 = require("common.data.Glicko") + +local function basicTest() + local player1 = Glicko2.g1(1500, 350) + local player2 = Glicko2.g1(1500, 350) + + local updatedPlayer1, updatedPlayer2 = Glicko2.updatedRatings(player1, player2, {1}, 1) + + assert(math.floor(updatedPlayer1.Rating) == 1662) + assert(math.floor(updatedPlayer2.Rating) == 1337) + + assert(player1.RD > updatedPlayer1.RD) + assert(player2.RD > updatedPlayer2.RD) +end + +basicTest() + +local function expectedOutcome() + local player1 = Glicko2.g1(1500, 350) + local player2 = Glicko2.g1(1500, 350) + + assert(player1:expectedOutcome(player2) == 0.5) + assert(player2:expectedOutcome(player1) == 0.5) + + local updatedPlayer1, updatedPlayer2 = Glicko2.updatedRatings(player1, player2, {1}, 1) + + assert(math.floor(updatedPlayer1.Rating) == 1662) + assert(math.floor(updatedPlayer2.Rating) == 1337) + + assert(player1.RD > updatedPlayer1.RD) + assert(player2.RD > updatedPlayer2.RD) + + assert(math.round(updatedPlayer1:expectedOutcome(updatedPlayer2), 2) == 0.76) + assert(math.round(updatedPlayer2:expectedOutcome(updatedPlayer1), 2) == 0.24) + + local player3 = Glicko2.g1(2000, 40) + local player4 = Glicko2.g1(1500, 350) + + assert(math.round(player3:expectedOutcome(player4), 2) == 0.87) + + local player4 = Glicko2.g1(2000, 40) + local player5 = Glicko2.g1(600, 40) + + assert(math.round(player4:expectedOutcome(player5), 4) == .9996) + + local player6 = Glicko2.g1(2500, 40) + local player7 = Glicko2.g1(1500, 40) + + assert(math.round(player6:expectedOutcome(player7), 4) == .9965) +end + +expectedOutcome() + +local function establishedVersusNew() + + local player1 = Glicko2.g1(1500, 40) + local player2 = Glicko2.g1(1500, 350) + + local updatedPlayer1, updatedPlayer2 = Glicko2.updatedRatings(player1, player2, {1, 1, 1, 1, 1, 1, 1, 1, 1, 0}, 1) + + assert(math.floor(updatedPlayer1.Rating) == 1524) + assert(math.floor(updatedPlayer2.Rating) == 1245) + + assert(math.floor(updatedPlayer1.RD) == 40) + assert(math.floor(updatedPlayer2.RD) == 105) +end + +establishedVersusNew() + +local function orderDoesntMatter() + + local player1 = Glicko2.g1(1500, 350) + local player2 = Glicko2.g1(1500, 350) + + local player1Copy = player1:copy() + local player2Copy = player2:copy() + + local updatedPlayer1, updatedPlayer2 = Glicko2.updatedRatings(player1, player2, {1, 1, 1, 0}, 1) + + local updatedPlayer1Copy, updatedPlayer2Copy = Glicko2.updatedRatings(player1Copy, player2Copy, {0, 1, 1, 1}, 1) + + assert(math.floatsEqualWithPrecision(updatedPlayer1Copy.Rating, updatedPlayer1.Rating, 10)) + assert(math.floatsEqualWithPrecision(updatedPlayer2Copy.Rating, updatedPlayer2.Rating, 10)) +end + +orderDoesntMatter() + +local function paperExample() + + local player1 = Glicko2.g1(1500, 200) + local player2 = Glicko2.g1(1400, 30) + local player3 = Glicko2.g1(1550, 100) + local player4 = Glicko2.g1(1700, 300) + + local player1Results = {} + player1Results[#player1Results+1] = player2:score(1) + player1Results[#player1Results+1] = player3:score(0) + player1Results[#player1Results+1] = player4:score(0) + + local updatedPlayer1 = player1:update(player1Results, 1) + + assert(math.round(updatedPlayer1.Rating, 2) == 1464.05) + assert(math.round(updatedPlayer1.RD, 2) == 151.52) + assert(math.round(updatedPlayer1.Vol, 2) == 0.06) +end + +paperExample() + diff --git a/common/tests/data/PlayerRatingTests.lua b/common/tests/data/PlayerRatingTests.lua new file mode 100644 index 00000000..e8483519 --- /dev/null +++ b/common/tests/data/PlayerRatingTests.lua @@ -0,0 +1,445 @@ +local PlayerRating = require("common.data.PlayerRating") +local simpleCSV = require("server.simplecsv") +local tableUtils = require("common.lib.tableUtils") +local logger = require("common.lib.logger") + +local function createPlayer(playerTable, playerID, playerRating) + playerTable[playerID] = {} + playerTable[playerID].playerRating = playerRating + assert(playerTable[playerID].playerRating:getRating() > 0) + playerTable[playerID].error = 0 + playerTable[playerID].totalGames = 0 +end + +local function playGamesWithResults(playerTable, player1ID, player2ID, gameResults, ratingPeriodElapsed) + for _, gameResult in ipairs(gameResults) do + local player1Rating = playerTable[player1ID].playerRating + local player2Rating = playerTable[player2ID].playerRating + for i = 1, 2, 1 do + local playerID = player1ID + local playerRating = player1Rating + local opponentRating = player2Rating + if i == 2 then + playerID = player2ID + playerRating = player2Rating + opponentRating = player1Rating + gameResult = PlayerRating.invertedGameResult(gameResult) + end + local result = playerRating:createGameResult(opponentRating, gameResult) + local expected = playerRating:expectedOutcome(opponentRating) + + playerTable[playerID].playerRating = playerRating:newRatingForResultsAndElapsedRatingPeriod(result, ratingPeriodElapsed) + playerTable[playerID].error = playerTable[playerID].error + (gameResult - expected) + playerTable[playerID].totalGames = playerTable[playerID].totalGames + 1 + end + end +end + +local function applyNewRatingPeriodToPlayers(playerTable, ratingPeriod) + for _, playerRow in ipairs(playerTable) do + playerRow.playerRating = playerRow.playerRating:newRatingForResultsAndLatestRatingPeriod(nil, ratingPeriod) + end +end + +local function testLowRDSimilarOpponents() + + local player1 = PlayerRating(1490, 20) + local player2 = PlayerRating(1500, 20) + local players = {} + createPlayer(players, 1, player1) + createPlayer(players, 2, player2) + + playGamesWithResults(players, 1, 2, PlayerRating.createTestSetResults(1, 1), 0) + + local player1RatingChange = players[1].playerRating:getRating() - player1:getRating() + local player2RatingChange = players[2].playerRating:getRating() - player2:getRating() + assert(player1RatingChange > 1 and player1RatingChange < 2) + assert(player2RatingChange < -1 and player2RatingChange > -2) +end +testLowRDSimilarOpponents() + +local function testLowRDFarOpponentUpset() + + local player1 = PlayerRating(1110, 20) + local player2 = PlayerRating(1500, 20) + local players = {} + createPlayer(players, 1, player1) + createPlayer(players, 2, player2) + + playGamesWithResults(players, 1, 2, PlayerRating.createTestSetResults(1, 1), 0) + + local player1RatingChange = players[1].playerRating:getRating() - player1:getRating() + local player2RatingChange = players[2].playerRating:getRating() - player2:getRating() + assert(player1RatingChange > 2 and player1RatingChange < 3) + assert(player2RatingChange < -2 and player2RatingChange > -3) +end +testLowRDFarOpponentUpset() + +local function testLowRDFarOpponentExpected() + local player1 = PlayerRating(1110, 20) + local player2 = PlayerRating(1500, 20) + local players = {} + createPlayer(players, 1, player1) + createPlayer(players, 2, player2) + + playGamesWithResults(players, 1, 2, PlayerRating.createTestSetResults(0, 1), 0) + + local player1RatingChange = players[1].playerRating:getRating() - player1:getRating() + local player2RatingChange = players[2].playerRating:getRating() - player2:getRating() + assert(player1RatingChange < -0.2 and player1RatingChange > -0.3) + assert(player2RatingChange > .2 and player2RatingChange < .3) +end +testLowRDFarOpponentExpected() + +local function testHighRDSimilarOpponents() + local player1 = PlayerRating(1490, PlayerRating.STARTING_RATING_DEVIATION) + local player2 = PlayerRating(1500, PlayerRating.STARTING_RATING_DEVIATION) + local players = {} + createPlayer(players, 1, player1) + createPlayer(players, 2, player2) + + playGamesWithResults(players, 1, 2, PlayerRating.createTestSetResults(1, 1), 0) + + local player1RatingChange = players[1].playerRating:getRating() - player1:getRating() + local player2RatingChange = players[2].playerRating:getRating() - player2:getRating() + assert(player1RatingChange > 80 and player1RatingChange < 100) + assert(player2RatingChange < -80 and player2RatingChange > -100) +end +testHighRDSimilarOpponents() + +local function testNewcomerUpset() + local player1 = PlayerRating() + local player2 = PlayerRating(1890, 20) + local players = {} + createPlayer(players, 1, player1) + createPlayer(players, 2, player2) + + playGamesWithResults(players, 1, 2, PlayerRating.createTestSetResults(1, 1), 0) + + local player1RatingChange = players[1].playerRating:getRating() - player1:getRating() + local player2RatingChange = players[2].playerRating:getRating() - player2:getRating() + assert(player1RatingChange > 180 and player1RatingChange < 200) + assert(player2RatingChange < -1 and player2RatingChange > -5) +end +testNewcomerUpset() + +local function testNewcomerExpected() + local player1 = PlayerRating() + local player2 = PlayerRating(1890, 20) + local players = {} + createPlayer(players, 1, player1) + createPlayer(players, 2, player2) + + playGamesWithResults(players, 1, 2, PlayerRating.createTestSetResults(0, 1), 0) + + local player1RatingChange = players[1].playerRating:getRating() - player1:getRating() + local player2RatingChange = players[2].playerRating:getRating() - player2:getRating() + assert(player1RatingChange < -19 and player1RatingChange > -20) + assert(player2RatingChange > .2 and player2RatingChange < .3) +end +testNewcomerExpected() + +local function testNoMatchesInRatingPeriod() + + local player1 = PlayerRating(1500, 150) + + -- We shouldn't assert or blow up in here, v goes to infinity but it is okay to divide by infinity as that is 0... + local updatedPlayer1 = player1:newRatingForResultsAndElapsedRatingPeriod({}, 1) + assert(updatedPlayer1:getRating() == 1500) -- rating shouldn't change + assert(updatedPlayer1.glicko.RD > 150) -- RD should go up +end +testNoMatchesInRatingPeriod() + +local function testRatingPeriodsForOccasionalPlayers() + local players = {} + for i = 1, 3 do + createPlayer(players, i, PlayerRating()) + end + + local ratingPeriodBetweenSets = 24 * 60 * 60 / PlayerRating.RATING_PERIOD_IN_SECONDS + local previousPlayerRatings = {} + for i = 1, 100, 1 do + for k = 1, 3 do + previousPlayerRatings[k] = players[k].playerRating:copy() + end + playGamesWithResults(players, 1, 2, PlayerRating.createTestSetResults(6, 10), 0) + playGamesWithResults(players, 1, 3, PlayerRating.createTestSetResults(4, 5), 0) + playGamesWithResults(players, 2, 3, PlayerRating.createTestSetResults(3, 5), 0) + + applyNewRatingPeriodToPlayers(players, i * ratingPeriodBetweenSets) + end + + assert(players[1].playerRating:getRating() > players[2].playerRating:getRating()) + assert(players[1].playerRating:getRating() > players[3].playerRating:getRating()) + assert(players[2].playerRating:getRating() > players[3].playerRating:getRating()) + + assert(players[1].playerRating.glicko.RD < 60) + assert(players[2].playerRating.glicko.RD < 60) + assert(players[3].playerRating.glicko.RD < 60) + + for k = 1, 3 do + -- rating and deviation should stabilize over time if players perform the same + assert(math.abs(previousPlayerRatings[k]:getRating() - players[k].playerRating:getRating()) < 6) + assert(math.abs(previousPlayerRatings[k].glicko.RD - players[k].playerRating.glicko.RD) < 1) + assert(previousPlayerRatings[k]:isProvisional() == false) + end +end +testRatingPeriodsForOccasionalPlayers() + +local function testRatingPeriodsForObsessivePlayers() + local players = {} + for i = 1, 3 do + createPlayer(players, i, PlayerRating()) + end + + local ratingPeriodBetweenSets = 24 * 60 * 60 / PlayerRating.RATING_PERIOD_IN_SECONDS + local previousPlayerRatings = {} + for i = 1, 100, 1 do + for k = 1, 3 do + previousPlayerRatings[k] = players[k].playerRating:copy() + end + playGamesWithResults(players, 1, 2, PlayerRating.createTestSetResults(60, 100), 0) + playGamesWithResults(players, 1, 3, PlayerRating.createTestSetResults(40, 50), 0) + playGamesWithResults(players, 2, 3, PlayerRating.createTestSetResults(30, 50), 0) + + applyNewRatingPeriodToPlayers(players, i * ratingPeriodBetweenSets) + end + + assert(players[1].playerRating:getRating() > players[2].playerRating:getRating()) + assert(players[1].playerRating:getRating() > players[3].playerRating:getRating()) + assert(players[2].playerRating:getRating() > players[3].playerRating:getRating()) + + assert(players[1].playerRating.glicko.RD < 30) + assert(players[2].playerRating.glicko.RD < 30) + assert(players[3].playerRating.glicko.RD < 35) -- player 3 "only" played 100 games per day + + for k = 1, 3 do + -- rating and deviation should stabilize over time if players perform the same + assert(math.abs(previousPlayerRatings[k]:getRating() - players[k].playerRating:getRating()) < 6) + assert(math.abs(previousPlayerRatings[k].glicko.RD - players[k].playerRating.glicko.RD) < 1) + assert(previousPlayerRatings[k]:isProvisional() == false) + end +end +testRatingPeriodsForObsessivePlayers() + +-- When a stable player doesn't play for a long time, we should lose some confidence in their rating, but not all. +local function testMaxRD() + local playerRating = PlayerRating(2000, 30) + + local threeMonthsInSeconds = 60 * 60 * 24 * 31 * 3 + local threeMonthsOfRatingPeriod = math.ceil(threeMonthsInSeconds / PlayerRating.RATING_PERIOD_IN_SECONDS) + for i = 1, threeMonthsOfRatingPeriod, 1 do + playerRating = playerRating:newRatingForResultsAndElapsedRatingPeriod({}, 1) + end + + assert(playerRating.glicko.RD >= 120) + + local nineMoreMonths = 60 * 60 * 24 * 31 * 9 + local oneYearOfRatingPeriod = math.ceil(nineMoreMonths / PlayerRating.RATING_PERIOD_IN_SECONDS) + for i = 1, oneYearOfRatingPeriod, 1 do + playerRating = playerRating:newRatingForResultsAndElapsedRatingPeriod({}, 1) + end + + assert(playerRating.glicko.RD >= PlayerRating.MAX_RATING_DEVIATION) +end +testMaxRD() + +local function testFarming() + local players = {} + for i = 1, 2 do + createPlayer(players, i, PlayerRating()) + end + + local ratingPeriodBetweenSets = 24 * 60 * 60 / PlayerRating.RATING_PERIOD_IN_SECONDS + for i = 1, 100, 1 do + playGamesWithResults(players, 1, 2, PlayerRating.createTestSetResults(11, 20), 0) + applyNewRatingPeriodToPlayers(players, i * ratingPeriodBetweenSets) + end + + -- Farm newcomers to see how much rating you can gain + for i = 3, 1000, 1 do + createPlayer(players, i, PlayerRating()) + playGamesWithResults(players, 1, i, PlayerRating.createTestSetResults(1, 1), 0) + end + + assert(players[1].playerRating:getRating() > PlayerRating.STARTING_RATING + PlayerRating.ALLOWABLE_RATING_SPREAD) -- Ranked high enough we can't play default players anymore + assert(players[1].playerRating:getRating() < 2000) -- Thus we couldn't farm really high +end +testFarming() + +local function testNewcomerSwing() + local players = {} + createPlayer(players, 1, PlayerRating()) + createPlayer(players, 2, PlayerRating(1121.96, 29.65)) + + -- Newcomer loses 40 times in a row... + playGamesWithResults(players, 1, 2, PlayerRating.createTestSetResults(0, 40), 0) + + assert(players[1].playerRating:getRating() > 700) -- Newcomer shouldn't go too far below other player + assert(players[1].playerRating:getRating() < 800) -- Newcomer should go down significantly though + assert(players[1].playerRating.glicko.RD > 70) -- Newcomer RD should start going down, but not too much + assert(players[1].playerRating.glicko.RD < 150) -- Newcomer RD should start going down, but not too much + assert(players[2].playerRating:getRating() > 1130) -- Normal player shouldn't gain too much rating + assert(players[2].playerRating:getRating() < 1200) + assert(players[2].playerRating.glicko.RD > 25) -- Normal player RD should stay similar + assert(players[2].playerRating.glicko.RD < 30) +end +testNewcomerSwing() + +local usedNames = {} +local publicIDMap = {} -- mapping of privateID to publicID +local function cleanNameForName(name, privateID) + name = name or "Player" + privateID = tostring(privateID) + if publicIDMap[privateID] == nil then + local result = name + while usedNames[result] ~= nil do + result = name .. math.random(1000000, 9999999) + end + usedNames[result] = true + publicIDMap[privateID] = result + end + + return publicIDMap[privateID] +end + +local function saveToResultsTable(currentRatingPeriod, players, glickoResultsTable) + for playerID, playerTable in pairs(players) do + -- Make sure the player is up to date with the current rating period + local newPlayerRating = playerTable.playerRating:newRatingForResultsAndLatestRatingPeriod({}, currentRatingPeriod) + playerTable.playerRating = newPlayerRating + local row = {} + row[#row+1] = playerTable.playerRating.lastRatingPeriodCalculated + row[#row+1] = playerID + row[#row+1] = playerTable.playerRating:getRating() + row[#row+1] = playerTable.playerRating.glicko.RD + row[#row+1] = playerTable.playerRating.glicko.Vol + row[#row+1] = playerTable.error + row[#row+1] = playerTable.totalGames + glickoResultsTable[#glickoResultsTable+1] = row + end +end + +local function runRatingPeriods(latestRatingPeriodFound, lastRatingPeriodSaved, players, playerID, gameResults, glickoResultsTable) + +end + +-- This test is to experiment with real world server data to verify the values work well. +-- put the players.txt, and GameResults.csv files in the root directory to run this test. +-- Make sure you don't keep this test enabled or commit those files! +local function testRealWorldData() + local players = {} + local glickoResultsTable = {} + local lastRatingPeriodSaved = 0 + local lastRatingPeriodFound = 0 + local gamesPlayedDays = {} + local gameResults = simpleCSV.read("GameResults.csv") + assert(gameResults) + + local tehJSON, err = love.filesystem.read("players.txt") + if tehJSON then + local playerData = json.decode(tehJSON) + if playerData then + ---@cast playerData table + for key, value in pairs(playerData) do + cleanNameForName(value, key) + end + end + end + + for row = 1, #gameResults do + local player1ID = cleanNameForName(nil, gameResults[row][1]) + local player2ID = cleanNameForName(nil, gameResults[row][2]) + local winResult = tonumber(gameResults[row][3]) + local ranked = tonumber(gameResults[row][4]) + local timestamp = tonumber(gameResults[row][5]) + local dateTable = os.date("*t", timestamp) + + assert(player1ID) + assert(player2ID) + assert(winResult) + assert(ranked) + assert(timestamp) + assert(dateTable) + + local now = os.date("*t", timestamp) + local dateKey = string.format("%02d/%02d/%04d", now.month, now.day, now.year) + + if gamesPlayedDays[dateKey] == nil then + gamesPlayedDays[dateKey] = {"",0,0} + gamesPlayedDays[dateKey][1] = dateKey + end + + if ranked == 0 then + gamesPlayedDays[dateKey][2] = gamesPlayedDays[dateKey][2] + 1 + else + gamesPlayedDays[dateKey][3] = gamesPlayedDays[dateKey][3] + 1 + end + + if ranked == 0 then + goto continue + end + + local currentRatingPeriod = PlayerRating.ratingPeriodForTimeStamp(timestamp) + lastRatingPeriodFound = currentRatingPeriod + + local now = os.date("*t", PlayerRating.timestampForRatingPeriod(currentRatingPeriod)) + logger.info("Processing " .. currentRatingPeriod .. " on " .. string.format("%02d/%02d/%04d", now.month, now.day, now.year)) + + local currentPlayerSets = {{player1ID, player2ID}, {player2ID, player1ID}} + for _, playerID in ipairs(currentPlayerSets[1]) do + -- Create a new player if one doesn't exist yet. + if not players[playerID] then + createPlayer(players, playerID, PlayerRating()) + end + players[playerID].playerRating = players[playerID].playerRating:newRatingForResultsAndLatestRatingPeriod(nil, currentRatingPeriod) + end + + playGamesWithResults(players, player1ID, player2ID, {winResult}, 0) + + -- Save off to a table for data analysis + local periodDifferenceFromLastRecord = currentRatingPeriod - lastRatingPeriodSaved + local shouldRecordRatings = (periodDifferenceFromLastRecord > 10) or (periodDifferenceFromLastRecord >= 1 and (currentRatingPeriod > 30130 or currentRatingPeriod < 29440)) + if lastRatingPeriodSaved == nil or shouldRecordRatings then + saveToResultsTable(currentRatingPeriod, players, glickoResultsTable) + lastRatingPeriodSaved = currentRatingPeriod + end + + ::continue:: + end + + saveToResultsTable(lastRatingPeriodFound, players, glickoResultsTable) + + local totalError = 0 + local totalGames = 0 + local provisionalCount = 0 + local playerCount = 0 + for playerID, playerTable in pairs(players) do + if playerTable.totalGames > 0 then + local error = math.abs(playerTable.error) + totalError = totalError + error + totalGames = totalGames + playerTable.totalGames + end + if playerTable.playerRating:isProvisional() then + provisionalCount = provisionalCount + 1 + end + playerCount = playerCount + 1 + end + local totalErrorPerGame = totalError / totalGames + + simpleCSV.write("Glicko.csv", glickoResultsTable) + -- 0.014760291450591 1 game per evaluation DEFAULT_RATING_DEVIATION:200 MAX_DEVIATION:200 DEFAULT_VOLATILITY:0.06 MAX_VOLATILITY:0.3 Tau:0.5 RATING PERIOD = 16hrs + -- 0.019420264639828 1 game per evaluation DEFAULT_RATING_DEVIATION:250 MAX_DEVIATION:250 DEFAULT_VOLATILITY:0.06 MAX_VOLATILITY:0.3 Tau:0.2 RATING PERIOD = 16hrs + -- 0.019439095055957 1 game per evaluation DEFAULT_RATING_DEVIATION:250 MAX_DEVIATION:250 DEFAULT_VOLATILITY:0.06 MAX_VOLATILITY:0.3 Tau:0.75 RATING PERIOD = 16hrs + -- 0.01944363557018 1 game per evaluation DEFAULT_RATING_DEVIATION:250 MAX_DEVIATION:350 DEFAULT_VOLATILITY:0.06 MAX_VOLATILITY:0.06 Tau:? RATING PERIOD = 16hrs + -- 0.03724587514630 all games in rating period DEFAULT_RATING_DEVIATION:250 MAX_DEVIATION:250 DEFAULT_VOLATILITY:0.06 MAX_VOLATILITY:0.06 Tau:? RATING PERIOD = 16hrs + -- 0.03792533617676 all games in rating period DEFAULT_RATING_DEVIATION:250 MAX_DEVIATION:250 DEFAULT_VOLATILITY:0.06 MAX_VOLATILITY:0.06 Tau:? RATING PERIOD = 24hrs + + local gamesPlayedData = {} + for dateKey, data in pairsSortedByKeys(gamesPlayedDays) do + gamesPlayedData[#gamesPlayedData+1] = data + end + simpleCSV.write("GamesPlayed.csv", gamesPlayedData) +end + +testRealWorldData() \ No newline at end of file diff --git a/testLauncher.lua b/testLauncher.lua index 0a8f173d..8cb4bcfe 100644 --- a/testLauncher.lua +++ b/testLauncher.lua @@ -40,6 +40,8 @@ local tests = { "client.tests.TcpClientTests", "client.tests.ThemeTests", "server.tests.ConnectionTests", + "common.tests.data.GlickoTests", + "common.tests.data.PlayerRatingTests", "common.tests.engine.GarbageQueueTests", "common.tests.engine.HealthTests", "common.tests.engine.PanelGenTests",