From ab37277b123deea05a7b9c0ede2f672fdbcf7658 Mon Sep 17 00:00:00 2001 From: andrey Date: Wed, 20 Mar 2019 22:53:22 +0300 Subject: [PATCH 01/13] initial commit --- HttpProxy/build.gradle | 21 ++++ HttpProxy/gradle.properties | 1 + HttpProxy/gradlew | 172 ++++++++++++++++++++++++++++++ HttpProxy/gradlew.bat | 84 +++++++++++++++ HttpProxy/settings.gradle | 2 + HttpProxy/src/main/kotlin/Main.kt | 3 + 6 files changed, 283 insertions(+) create mode 100644 HttpProxy/build.gradle create mode 100644 HttpProxy/gradle.properties create mode 100755 HttpProxy/gradlew create mode 100644 HttpProxy/gradlew.bat create mode 100644 HttpProxy/settings.gradle create mode 100644 HttpProxy/src/main/kotlin/Main.kt diff --git a/HttpProxy/build.gradle b/HttpProxy/build.gradle new file mode 100644 index 0000000..c8ed2cc --- /dev/null +++ b/HttpProxy/build.gradle @@ -0,0 +1,21 @@ +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.3.21' +} + +group 'http-proxy' +version '1.0-SNAPSHOT' + +repositories { + mavenCentral() +} + +dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" +} + +compileKotlin { + kotlinOptions.jvmTarget = "1.8" +} +compileTestKotlin { + kotlinOptions.jvmTarget = "1.8" +} \ No newline at end of file diff --git a/HttpProxy/gradle.properties b/HttpProxy/gradle.properties new file mode 100644 index 0000000..29e08e8 --- /dev/null +++ b/HttpProxy/gradle.properties @@ -0,0 +1 @@ +kotlin.code.style=official \ No newline at end of file diff --git a/HttpProxy/gradlew b/HttpProxy/gradlew new file mode 100755 index 0000000..cccdd3d --- /dev/null +++ b/HttpProxy/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/HttpProxy/gradlew.bat b/HttpProxy/gradlew.bat new file mode 100644 index 0000000..e95643d --- /dev/null +++ b/HttpProxy/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/HttpProxy/settings.gradle b/HttpProxy/settings.gradle new file mode 100644 index 0000000..f89ecdf --- /dev/null +++ b/HttpProxy/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'http-proxy' + diff --git a/HttpProxy/src/main/kotlin/Main.kt b/HttpProxy/src/main/kotlin/Main.kt new file mode 100644 index 0000000..400492f --- /dev/null +++ b/HttpProxy/src/main/kotlin/Main.kt @@ -0,0 +1,3 @@ +fun main() { + print("hello world") +} \ No newline at end of file From 4cf2b338496a3135b61aeba5a5e7910ea0d83f30 Mon Sep 17 00:00:00 2001 From: oquechy Date: Thu, 21 Mar 2019 22:17:50 +0300 Subject: [PATCH 02/13] kus' --- HttpProxy/src/main/kotlin/Main.kt | 3 -- HttpProxy/src/main/kotlin/ru/hse/Accepter.kt | 31 ++++++++++++++++++++ HttpProxy/src/main/kotlin/ru/hse/Cache.kt | 5 ++++ HttpProxy/src/main/kotlin/ru/hse/Main.kt | 7 +++++ 4 files changed, 43 insertions(+), 3 deletions(-) delete mode 100644 HttpProxy/src/main/kotlin/Main.kt create mode 100644 HttpProxy/src/main/kotlin/ru/hse/Accepter.kt create mode 100644 HttpProxy/src/main/kotlin/ru/hse/Cache.kt create mode 100644 HttpProxy/src/main/kotlin/ru/hse/Main.kt diff --git a/HttpProxy/src/main/kotlin/Main.kt b/HttpProxy/src/main/kotlin/Main.kt deleted file mode 100644 index 400492f..0000000 --- a/HttpProxy/src/main/kotlin/Main.kt +++ /dev/null @@ -1,3 +0,0 @@ -fun main() { - print("hello world") -} \ No newline at end of file diff --git a/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt b/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt new file mode 100644 index 0000000..f3c14f8 --- /dev/null +++ b/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt @@ -0,0 +1,31 @@ +package ru.hse + +import java.net.ServerSocket +import kotlin.concurrent.thread + +class Acceptor(cache: Cache, port: Int) { + + lateinit var thread: Thread + + init { + val serverSocket = ServerSocket(port) + serverSocket.use { + thread = thread(start = false, name = "Acceptor") { + while (!Thread.currentThread().isInterrupted) { + val socket = serverSocket.accept() + socket.use { + socket.getOutputStream() + } + } + } + } + } + + fun start() { + thread.start() + } + + fun stop() { + thread.interrupt() + } +} diff --git a/HttpProxy/src/main/kotlin/ru/hse/Cache.kt b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt new file mode 100644 index 0000000..95492b2 --- /dev/null +++ b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt @@ -0,0 +1,5 @@ +package ru.hse + +class Cache { + +} diff --git a/HttpProxy/src/main/kotlin/ru/hse/Main.kt b/HttpProxy/src/main/kotlin/ru/hse/Main.kt new file mode 100644 index 0000000..18c758e --- /dev/null +++ b/HttpProxy/src/main/kotlin/ru/hse/Main.kt @@ -0,0 +1,7 @@ +package ru.hse + +fun main() { + val cache = Cache() + val acceptor = Acceptor(cache, 111) + acceptor.start() +} \ No newline at end of file From 2a4e02b12aa9662bfffa9328a1cea275bc3574b8 Mon Sep 17 00:00:00 2001 From: andrey Date: Sat, 23 Mar 2019 22:18:50 +0300 Subject: [PATCH 03/13] cache --- HttpProxy/src/main/kotlin/ru/hse/Accepter.kt | 2 +- HttpProxy/src/main/kotlin/ru/hse/Cache.kt | 5 --- HttpProxy/src/main/kotlin/ru/hse/LruCache.kt | 41 ++++++++++++++++++++ HttpProxy/src/main/kotlin/ru/hse/Main.kt | 2 +- 4 files changed, 43 insertions(+), 7 deletions(-) delete mode 100644 HttpProxy/src/main/kotlin/ru/hse/Cache.kt create mode 100644 HttpProxy/src/main/kotlin/ru/hse/LruCache.kt diff --git a/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt b/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt index f3c14f8..6fa8e0e 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt @@ -3,7 +3,7 @@ package ru.hse import java.net.ServerSocket import kotlin.concurrent.thread -class Acceptor(cache: Cache, port: Int) { +class Acceptor(cache: LruCache, port: Int) { lateinit var thread: Thread diff --git a/HttpProxy/src/main/kotlin/ru/hse/Cache.kt b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt deleted file mode 100644 index 95492b2..0000000 --- a/HttpProxy/src/main/kotlin/ru/hse/Cache.kt +++ /dev/null @@ -1,5 +0,0 @@ -package ru.hse - -class Cache { - -} diff --git a/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt b/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt new file mode 100644 index 0000000..db0cd63 --- /dev/null +++ b/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt @@ -0,0 +1,41 @@ +package ru.hse + +import java.util.* + + +class LruCache(private val expirationTime: Int = 1000, val cacheSize: Int = 1000) : Cache { + private val cache = Collections.synchronizedMap(LinkedHashMap()); + override fun lookUp(request: String): String? { + val data = cache[request] ?: return null + if (getTime() - data.addedTime > expirationTime) { + cache.remove(request) + return null + } + + data.lookedUpTime = getTime() + return data.page + } + + override fun addPage(request: String, page: String) { + if (cache.size == cacheSize) { + cache.remove(cache.iterator().next().key); + } + val time = getTime() + cache[request] = PageData(page, time, time) + } + + // Suppose don't need it + override fun updateTime(request: String, page: String) { + if (!cache.containsKey(request)) { + val time = getTime() + cache[request] = PageData(page, time, time) + } + + val data = cache[request] + data?.lookedUpTime = getTime() + } + + private fun getTime(): Long = System.currentTimeMillis() / 1000 + + data class PageData(val page: String, val addedTime: Long, var lookedUpTime: Long) +} diff --git a/HttpProxy/src/main/kotlin/ru/hse/Main.kt b/HttpProxy/src/main/kotlin/ru/hse/Main.kt index 18c758e..32e2dfd 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/Main.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/Main.kt @@ -1,7 +1,7 @@ package ru.hse fun main() { - val cache = Cache() + val cache = LruCache() val acceptor = Acceptor(cache, 111) acceptor.start() } \ No newline at end of file From 00051cc47111970b70819be01433bb9f3a65eca8 Mon Sep 17 00:00:00 2001 From: oquechy Date: Sat, 23 Mar 2019 23:09:37 +0300 Subject: [PATCH 04/13] remove lateinit in Acceptor initialization --- HttpProxy/src/main/kotlin/ru/hse/Accepter.kt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt b/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt index f3c14f8..34a5da2 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt @@ -5,27 +5,27 @@ import kotlin.concurrent.thread class Acceptor(cache: Cache, port: Int) { - lateinit var thread: Thread + private val listener: Thread + private val workers: MutableList = ArrayList() init { val serverSocket = ServerSocket(port) - serverSocket.use { - thread = thread(start = false, name = "Acceptor") { - while (!Thread.currentThread().isInterrupted) { + listener = thread(start = false, name = "Acceptor") { + while (!Thread.currentThread().isInterrupted) { + serverSocket.use { val socket = serverSocket.accept() - socket.use { - socket.getOutputStream() - } + workers.add(startClientHandler(socket, cache)) } } } } fun start() { - thread.start() + listener.start() } fun stop() { - thread.interrupt() + listener.interrupt() + workers.forEach(Thread::interrupt) } } From b238bccd91cb5a7e8f0448ba29c43ee0c2452617 Mon Sep 17 00:00:00 2001 From: oquechy Date: Tue, 26 Mar 2019 23:29:24 +0300 Subject: [PATCH 05/13] fix acceptor closing time --- HttpProxy/src/main/kotlin/ru/hse/Accepter.kt | 31 -------------- HttpProxy/src/main/kotlin/ru/hse/Acceptor.kt | 42 +++++++++++++++++++ .../src/main/kotlin/ru/hse/ClientHandler.kt | 36 ++++++++++++++++ HttpProxy/src/main/kotlin/ru/hse/Main.kt | 3 +- 4 files changed, 79 insertions(+), 33 deletions(-) delete mode 100644 HttpProxy/src/main/kotlin/ru/hse/Accepter.kt create mode 100644 HttpProxy/src/main/kotlin/ru/hse/Acceptor.kt create mode 100644 HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt diff --git a/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt b/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt deleted file mode 100644 index 34a5da2..0000000 --- a/HttpProxy/src/main/kotlin/ru/hse/Accepter.kt +++ /dev/null @@ -1,31 +0,0 @@ -package ru.hse - -import java.net.ServerSocket -import kotlin.concurrent.thread - -class Acceptor(cache: Cache, port: Int) { - - private val listener: Thread - private val workers: MutableList = ArrayList() - - init { - val serverSocket = ServerSocket(port) - listener = thread(start = false, name = "Acceptor") { - while (!Thread.currentThread().isInterrupted) { - serverSocket.use { - val socket = serverSocket.accept() - workers.add(startClientHandler(socket, cache)) - } - } - } - } - - fun start() { - listener.start() - } - - fun stop() { - listener.interrupt() - workers.forEach(Thread::interrupt) - } -} diff --git a/HttpProxy/src/main/kotlin/ru/hse/Acceptor.kt b/HttpProxy/src/main/kotlin/ru/hse/Acceptor.kt new file mode 100644 index 0000000..b6ee375 --- /dev/null +++ b/HttpProxy/src/main/kotlin/ru/hse/Acceptor.kt @@ -0,0 +1,42 @@ +package ru.hse + +import java.net.ServerSocket +import java.util.logging.Logger +import kotlin.concurrent.thread + +class Acceptor(val port: Int, clientHandler: ClientHandler) { + + private val listener: Thread + private val workers: MutableList = ArrayList() + + init { + listener = thread(start = false, name = "Acceptor") { + try { + val serverSocket = ServerSocket(port) + serverSocket.use { + while (!Thread.currentThread().isInterrupted) { + val socket = it.accept() + workers.add(clientHandler.run(socket)) + } + } + } catch (e: Exception) { + logger.severe("Closing client acceptor: ${e.message}") + e.printStackTrace() + } + } + } + + fun start() { + listener.start() + logger.info("Proxy started on port $port") + } + + fun stop() { + listener.interrupt() + workers.forEach(Thread::interrupt) + } + + companion object { + val logger: Logger = Logger.getLogger(Acceptor::class.java.name) + } +} diff --git a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt new file mode 100644 index 0000000..ff90096 --- /dev/null +++ b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt @@ -0,0 +1,36 @@ +package ru.hse + +import java.io.BufferedReader +import java.io.InputStreamReader +import java.lang.Exception +import java.net.Socket +import java.util.logging.Logger +import kotlin.concurrent.thread + +class ClientHandler(cache: Cache, blackList: List) { + + fun run(clientSocket: Socket): Thread { + return thread { + val clientAddress = clientSocket.inetAddress + logger.info("Handling client $clientAddress") + try { + clientSocket.use { + val reader = BufferedReader(InputStreamReader(it.inputStream)) + var header = reader.readLine() + logger.info("Client data: $header") + while (true) { + header = reader.readLine() ?: break + logger.info(header) + } + reader.close() + } + } catch (e: Exception) { + logger.severe("Connection with $clientAddress interrupted: ${e.message}") + } + } + } + + companion object { + val logger = Logger.getLogger(ClientHandler::class.java.name) + } +} diff --git a/HttpProxy/src/main/kotlin/ru/hse/Main.kt b/HttpProxy/src/main/kotlin/ru/hse/Main.kt index 18c758e..b73ec70 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/Main.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/Main.kt @@ -1,7 +1,6 @@ package ru.hse fun main() { - val cache = Cache() - val acceptor = Acceptor(cache, 111) + val acceptor = Acceptor(3030, ClientHandler(Cache(), listOf())) acceptor.start() } \ No newline at end of file From 6a93f7417d4929bc730a18392f7212c06f72659b Mon Sep 17 00:00:00 2001 From: oquechy Date: Wed, 27 Mar 2019 10:37:10 +0300 Subject: [PATCH 06/13] cache --- HttpProxy/src/main/kotlin/ru/hse/Cache.kt | 7 ++ .../src/main/kotlin/ru/hse/ClientHandler.kt | 92 +++++++++++++++++-- HttpProxy/src/main/kotlin/ru/hse/LruCache.kt | 18 ++-- HttpProxy/src/main/kotlin/ru/hse/Main.kt | 2 +- HttpProxy/src/main/kotlin/ru/hse/Request.kt | 10 ++ 5 files changed, 111 insertions(+), 18 deletions(-) create mode 100644 HttpProxy/src/main/kotlin/ru/hse/Cache.kt create mode 100644 HttpProxy/src/main/kotlin/ru/hse/Request.kt diff --git a/HttpProxy/src/main/kotlin/ru/hse/Cache.kt b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt new file mode 100644 index 0000000..e243308 --- /dev/null +++ b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt @@ -0,0 +1,7 @@ +package ru.hse + +interface Cache { + fun lookUp(request: K): V? + fun addPage(request: K, response: V) + fun updateTime(request: K, response: V) +} diff --git a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt index ff90096..77473b1 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt @@ -1,13 +1,17 @@ package ru.hse import java.io.BufferedReader +import java.io.BufferedWriter import java.io.InputStreamReader +import java.io.OutputStreamWriter import java.lang.Exception import java.net.Socket +import java.net.URL import java.util.logging.Logger import kotlin.concurrent.thread +import kotlin.math.log -class ClientHandler(cache: Cache, blackList: List) { +class ClientHandler(private val cache: Cache, private val blackList: List) { fun run(clientSocket: Socket): Thread { return thread { @@ -15,14 +19,28 @@ class ClientHandler(cache: Cache, blackList: List) { logger.info("Handling client $clientAddress") try { clientSocket.use { - val reader = BufferedReader(InputStreamReader(it.inputStream)) - var header = reader.readLine() - logger.info("Client data: $header") - while (true) { - header = reader.readLine() ?: break - logger.info(header) + val request = extractDetails(it) + if (blackList.contains(request.host)) { + logger.severe("Host ${request.host} is blacklisted") + responseWithError(it) + return@use + } + if (request.method == "GET") { + val cached = cache.lookUp(request) + if (cached != null) { + logger.info("Cached copy of response from ${request.host} was sent") + responseWith(cached, it) + } else { + logger.info("Fresh HTTP response from ${request.host}") + val response = retrieveResponse(request) + logger.info(response) + cache.addPage(request, response) + responseWith(response, it) + } + + } else { + responseWith(retrieveResponse(request), it) } - reader.close() } } catch (e: Exception) { logger.severe("Connection with $clientAddress interrupted: ${e.message}") @@ -30,7 +48,65 @@ class ClientHandler(cache: Cache, blackList: List) { } } + private fun retrieveResponse(request: Request): String { + try { + Socket(request.host, request.port).use { + val writer = BufferedWriter(OutputStreamWriter(it.outputStream)) + writer.append(request.data) + writer.flush() + logger.info("Sent ${request.data}") + + return extractDetails(it).data + } + } catch (e: Exception) { + logger.severe("Request to ${request.host} failed: ${e.message}") + return ERROR_RESPONSE + } + } + + private fun responseWith(response: String, socket: Socket) { + val writer = BufferedWriter(OutputStreamWriter(socket.outputStream)) + writer.append(response) + writer.flush() + } + + private fun responseWithError(socket: Socket) = responseWith(ERROR_RESPONSE, socket) + + private fun extractDetails(it: Socket): Request { + val lines = mutableListOf() + val reader = BufferedReader(InputStreamReader(it.inputStream)) + reader.use { + do { + val line = it.readLine() + lines.add(line) + } while (lines.size < 2 || !lines.last().trim().isEmpty() || !line.trim().isEmpty()) + } + + val headerLine = lines[0].split(' ') + val url = URL(headerLine[1]) + val protocol = url.protocol + val port = if (url.port == -1) 80 else url.port + val authB64 = lines.find { it.contains("Authorization") }?.split(' ')?.get(2) + val details = Request( + method = headerLine[0], + host = url.host, + port = port, + protocol = protocol, + url = url.toString(), + data = lines.joinToString(separator = "\r\n"), + auth = authB64 + ) + return details + } + companion object { + const val ERROR_RESPONSE = "HTTP/1.0 200 OK\r\n" + + "Content-Length: 11\r\n" + + "\r\n" + + "Error\r\n" + + "\r\n" + + "\r\n" + val logger = Logger.getLogger(ClientHandler::class.java.name) } } diff --git a/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt b/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt index db0cd63..0a9d888 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt @@ -3,9 +3,9 @@ package ru.hse import java.util.* -class LruCache(private val expirationTime: Int = 1000, val cacheSize: Int = 1000) : Cache { - private val cache = Collections.synchronizedMap(LinkedHashMap()); - override fun lookUp(request: String): String? { +class LruCache(val expirationTime: Int = 1000, val cacheSize: Int = 1000) : Cache { + private val cache = Collections.synchronizedMap(LinkedHashMap>()) + override fun lookUp(request: K): V? { val data = cache[request] ?: return null if (getTime() - data.addedTime > expirationTime) { cache.remove(request) @@ -16,19 +16,19 @@ class LruCache(private val expirationTime: Int = 1000, val cacheSize: Int = 1000 return data.page } - override fun addPage(request: String, page: String) { + override fun addPage(request: K, response: V) { if (cache.size == cacheSize) { - cache.remove(cache.iterator().next().key); + cache.remove(cache.iterator().next().key) } val time = getTime() - cache[request] = PageData(page, time, time) + cache[request] = PageData(response, time, time) } // Suppose don't need it - override fun updateTime(request: String, page: String) { + override fun updateTime(request: K, response: V) { if (!cache.containsKey(request)) { val time = getTime() - cache[request] = PageData(page, time, time) + cache[request] = PageData(response, time, time) } val data = cache[request] @@ -37,5 +37,5 @@ class LruCache(private val expirationTime: Int = 1000, val cacheSize: Int = 1000 private fun getTime(): Long = System.currentTimeMillis() / 1000 - data class PageData(val page: String, val addedTime: Long, var lookedUpTime: Long) + data class PageData(val page: V, val addedTime: Long, var lookedUpTime: Long) } diff --git a/HttpProxy/src/main/kotlin/ru/hse/Main.kt b/HttpProxy/src/main/kotlin/ru/hse/Main.kt index b73ec70..ec2e54b 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/Main.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/Main.kt @@ -1,6 +1,6 @@ package ru.hse fun main() { - val acceptor = Acceptor(3030, ClientHandler(Cache(), listOf())) + val acceptor = Acceptor(3030, ClientHandler(LruCache(), listOf())) acceptor.start() } \ No newline at end of file diff --git a/HttpProxy/src/main/kotlin/ru/hse/Request.kt b/HttpProxy/src/main/kotlin/ru/hse/Request.kt new file mode 100644 index 0000000..d985510 --- /dev/null +++ b/HttpProxy/src/main/kotlin/ru/hse/Request.kt @@ -0,0 +1,10 @@ +package ru.hse + +data class Request( + val method: String, val host: String, + val port: Int, + val protocol: String, + val url: String, + val data: String, + val auth: String? +) \ No newline at end of file From 92eccef71db9027b49405fd676d713f127a7707e Mon Sep 17 00:00:00 2001 From: oquechy Date: Wed, 27 Mar 2019 15:01:08 +0300 Subject: [PATCH 07/13] add two empty lines to parsed request --- .../src/main/kotlin/ru/hse/ClientHandler.kt | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt index 77473b1..828ee7a 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt @@ -4,12 +4,10 @@ import java.io.BufferedReader import java.io.BufferedWriter import java.io.InputStreamReader import java.io.OutputStreamWriter -import java.lang.Exception import java.net.Socket import java.net.URL import java.util.logging.Logger import kotlin.concurrent.thread -import kotlin.math.log class ClientHandler(private val cache: Cache, private val blackList: List) { @@ -19,7 +17,7 @@ class ClientHandler(private val cache: Cache, private val black logger.info("Handling client $clientAddress") try { clientSocket.use { - val request = extractDetails(it) + val request = extractRequestDetails(it) if (blackList.contains(request.host)) { logger.severe("Host ${request.host} is blacklisted") responseWithError(it) @@ -56,7 +54,7 @@ class ClientHandler(private val cache: Cache, private val black writer.flush() logger.info("Sent ${request.data}") - return extractDetails(it).data + return extractResponse(it) } } catch (e: Exception) { logger.severe("Request to ${request.host} failed: ${e.message}") @@ -72,15 +70,8 @@ class ClientHandler(private val cache: Cache, private val black private fun responseWithError(socket: Socket) = responseWith(ERROR_RESPONSE, socket) - private fun extractDetails(it: Socket): Request { - val lines = mutableListOf() - val reader = BufferedReader(InputStreamReader(it.inputStream)) - reader.use { - do { - val line = it.readLine() - lines.add(line) - } while (lines.size < 2 || !lines.last().trim().isEmpty() || !line.trim().isEmpty()) - } + private fun extractRequestDetails(socket: Socket): Request { + val lines = readLines(socket) val headerLine = lines[0].split(' ') val url = URL(headerLine[1]) @@ -93,12 +84,32 @@ class ClientHandler(private val cache: Cache, private val black port = port, protocol = protocol, url = url.toString(), - data = lines.joinToString(separator = "\r\n"), + data = joinLines(lines), auth = authB64 ) return details } + private fun extractResponse(socket: Socket): String { + val lines = readLines(socket) + return joinLines(lines) + } + + private fun joinLines(lines: MutableList) = + lines.joinToString(separator = "\r\n", postfix = "\r\n\r\n") + + private fun readLines(socket: Socket): MutableList { + val lines = mutableListOf() + val reader = BufferedReader(InputStreamReader(socket.inputStream)) + reader.use { + do { + val line = it.readLine() + lines.add(line) + } while (lines.size < 2 || !lines.last().trim().isEmpty() || !line.trim().isEmpty()) + } + return lines + } + companion object { const val ERROR_RESPONSE = "HTTP/1.0 200 OK\r\n" + "Content-Length: 11\r\n" + From afb99061f70b1a549a6842bc75690fd7f6fc1234 Mon Sep 17 00:00:00 2001 From: oquechy Date: Wed, 27 Mar 2019 17:34:28 +0300 Subject: [PATCH 08/13] fukit --- .../src/main/kotlin/ru/hse/ClientHandler.kt | 29 ++++++++++--------- HttpProxy/src/main/kotlin/ru/hse/Main.kt | 2 +- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt index 828ee7a..218f8ab 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt @@ -54,7 +54,9 @@ class ClientHandler(private val cache: Cache, private val black writer.flush() logger.info("Sent ${request.data}") - return extractResponse(it) + val response = extractResponse(it) + logger.info("Received $response") + return response } } catch (e: Exception) { logger.severe("Request to ${request.host} failed: ${e.message}") @@ -78,7 +80,7 @@ class ClientHandler(private val cache: Cache, private val black val protocol = url.protocol val port = if (url.port == -1) 80 else url.port val authB64 = lines.find { it.contains("Authorization") }?.split(' ')?.get(2) - val details = Request( + return Request( method = headerLine[0], host = url.host, port = port, @@ -87,37 +89,38 @@ class ClientHandler(private val cache: Cache, private val black data = joinLines(lines), auth = authB64 ) - return details } private fun extractResponse(socket: Socket): String { - val lines = readLines(socket) + val lines = readLines(socket, 2) return joinLines(lines) } - private fun joinLines(lines: MutableList) = - lines.joinToString(separator = "\r\n", postfix = "\r\n\r\n") - - private fun readLines(socket: Socket): MutableList { + private fun readLines(socket: Socket, times: Int = 1): List { val lines = mutableListOf() val reader = BufferedReader(InputStreamReader(socket.inputStream)) - reader.use { + repeat(times) { do { - val line = it.readLine() + val line: String = reader.readLine() ?: return@repeat lines.add(line) - } while (lines.size < 2 || !lines.last().trim().isEmpty() || !line.trim().isEmpty()) + logger.info(line) + } while (!line.isEmpty()) } return lines } + private fun joinLines(lines: List): String { + return lines.joinToString(separator = SEPARATOR, postfix = SEPARATOR + SEPARATOR) + } + companion object { const val ERROR_RESPONSE = "HTTP/1.0 200 OK\r\n" + "Content-Length: 11\r\n" + "\r\n" + "Error\r\n" + - "\r\n" + "\r\n" + const val SEPARATOR = "\r\n" - val logger = Logger.getLogger(ClientHandler::class.java.name) + val logger = Logger.getLogger(ClientHandler::class.java.name)!! } } diff --git a/HttpProxy/src/main/kotlin/ru/hse/Main.kt b/HttpProxy/src/main/kotlin/ru/hse/Main.kt index ec2e54b..a16ea50 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/Main.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/Main.kt @@ -1,6 +1,6 @@ package ru.hse fun main() { - val acceptor = Acceptor(3030, ClientHandler(LruCache(), listOf())) + val acceptor = Acceptor(3030, ClientHandler(LruCache(), listOf("google.com", "ya.ru"))) acceptor.start() } \ No newline at end of file From 4fd93e8bcaf9efaa0ef55affc5e75610f85fe630 Mon Sep 17 00:00:00 2001 From: oquechy Date: Wed, 27 Mar 2019 17:49:57 +0300 Subject: [PATCH 09/13] http proxy --- HttpProxy/src/main/kotlin/ru/hse/Cache.kt | 7 ++ .../src/main/kotlin/ru/hse/ClientHandler.kt | 110 ++++++++++++++++-- HttpProxy/src/main/kotlin/ru/hse/LruCache.kt | 18 +-- HttpProxy/src/main/kotlin/ru/hse/Main.kt | 2 +- HttpProxy/src/main/kotlin/ru/hse/Request.kt | 10 ++ 5 files changed, 127 insertions(+), 20 deletions(-) create mode 100644 HttpProxy/src/main/kotlin/ru/hse/Cache.kt create mode 100644 HttpProxy/src/main/kotlin/ru/hse/Request.kt diff --git a/HttpProxy/src/main/kotlin/ru/hse/Cache.kt b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt new file mode 100644 index 0000000..e243308 --- /dev/null +++ b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt @@ -0,0 +1,7 @@ +package ru.hse + +interface Cache { + fun lookUp(request: K): V? + fun addPage(request: K, response: V) + fun updateTime(request: K, response: V) +} diff --git a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt index ff90096..218f8ab 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt @@ -1,13 +1,15 @@ package ru.hse import java.io.BufferedReader +import java.io.BufferedWriter import java.io.InputStreamReader -import java.lang.Exception +import java.io.OutputStreamWriter import java.net.Socket +import java.net.URL import java.util.logging.Logger import kotlin.concurrent.thread -class ClientHandler(cache: Cache, blackList: List) { +class ClientHandler(private val cache: Cache, private val blackList: List) { fun run(clientSocket: Socket): Thread { return thread { @@ -15,14 +17,28 @@ class ClientHandler(cache: Cache, blackList: List) { logger.info("Handling client $clientAddress") try { clientSocket.use { - val reader = BufferedReader(InputStreamReader(it.inputStream)) - var header = reader.readLine() - logger.info("Client data: $header") - while (true) { - header = reader.readLine() ?: break - logger.info(header) + val request = extractRequestDetails(it) + if (blackList.contains(request.host)) { + logger.severe("Host ${request.host} is blacklisted") + responseWithError(it) + return@use + } + if (request.method == "GET") { + val cached = cache.lookUp(request) + if (cached != null) { + logger.info("Cached copy of response from ${request.host} was sent") + responseWith(cached, it) + } else { + logger.info("Fresh HTTP response from ${request.host}") + val response = retrieveResponse(request) + logger.info(response) + cache.addPage(request, response) + responseWith(response, it) + } + + } else { + responseWith(retrieveResponse(request), it) } - reader.close() } } catch (e: Exception) { logger.severe("Connection with $clientAddress interrupted: ${e.message}") @@ -30,7 +46,81 @@ class ClientHandler(cache: Cache, blackList: List) { } } + private fun retrieveResponse(request: Request): String { + try { + Socket(request.host, request.port).use { + val writer = BufferedWriter(OutputStreamWriter(it.outputStream)) + writer.append(request.data) + writer.flush() + logger.info("Sent ${request.data}") + + val response = extractResponse(it) + logger.info("Received $response") + return response + } + } catch (e: Exception) { + logger.severe("Request to ${request.host} failed: ${e.message}") + return ERROR_RESPONSE + } + } + + private fun responseWith(response: String, socket: Socket) { + val writer = BufferedWriter(OutputStreamWriter(socket.outputStream)) + writer.append(response) + writer.flush() + } + + private fun responseWithError(socket: Socket) = responseWith(ERROR_RESPONSE, socket) + + private fun extractRequestDetails(socket: Socket): Request { + val lines = readLines(socket) + + val headerLine = lines[0].split(' ') + val url = URL(headerLine[1]) + val protocol = url.protocol + val port = if (url.port == -1) 80 else url.port + val authB64 = lines.find { it.contains("Authorization") }?.split(' ')?.get(2) + return Request( + method = headerLine[0], + host = url.host, + port = port, + protocol = protocol, + url = url.toString(), + data = joinLines(lines), + auth = authB64 + ) + } + + private fun extractResponse(socket: Socket): String { + val lines = readLines(socket, 2) + return joinLines(lines) + } + + private fun readLines(socket: Socket, times: Int = 1): List { + val lines = mutableListOf() + val reader = BufferedReader(InputStreamReader(socket.inputStream)) + repeat(times) { + do { + val line: String = reader.readLine() ?: return@repeat + lines.add(line) + logger.info(line) + } while (!line.isEmpty()) + } + return lines + } + + private fun joinLines(lines: List): String { + return lines.joinToString(separator = SEPARATOR, postfix = SEPARATOR + SEPARATOR) + } + companion object { - val logger = Logger.getLogger(ClientHandler::class.java.name) + const val ERROR_RESPONSE = "HTTP/1.0 200 OK\r\n" + + "Content-Length: 11\r\n" + + "\r\n" + + "Error\r\n" + + "\r\n" + const val SEPARATOR = "\r\n" + + val logger = Logger.getLogger(ClientHandler::class.java.name)!! } } diff --git a/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt b/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt index db0cd63..0a9d888 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt @@ -3,9 +3,9 @@ package ru.hse import java.util.* -class LruCache(private val expirationTime: Int = 1000, val cacheSize: Int = 1000) : Cache { - private val cache = Collections.synchronizedMap(LinkedHashMap()); - override fun lookUp(request: String): String? { +class LruCache(val expirationTime: Int = 1000, val cacheSize: Int = 1000) : Cache { + private val cache = Collections.synchronizedMap(LinkedHashMap>()) + override fun lookUp(request: K): V? { val data = cache[request] ?: return null if (getTime() - data.addedTime > expirationTime) { cache.remove(request) @@ -16,19 +16,19 @@ class LruCache(private val expirationTime: Int = 1000, val cacheSize: Int = 1000 return data.page } - override fun addPage(request: String, page: String) { + override fun addPage(request: K, response: V) { if (cache.size == cacheSize) { - cache.remove(cache.iterator().next().key); + cache.remove(cache.iterator().next().key) } val time = getTime() - cache[request] = PageData(page, time, time) + cache[request] = PageData(response, time, time) } // Suppose don't need it - override fun updateTime(request: String, page: String) { + override fun updateTime(request: K, response: V) { if (!cache.containsKey(request)) { val time = getTime() - cache[request] = PageData(page, time, time) + cache[request] = PageData(response, time, time) } val data = cache[request] @@ -37,5 +37,5 @@ class LruCache(private val expirationTime: Int = 1000, val cacheSize: Int = 1000 private fun getTime(): Long = System.currentTimeMillis() / 1000 - data class PageData(val page: String, val addedTime: Long, var lookedUpTime: Long) + data class PageData(val page: V, val addedTime: Long, var lookedUpTime: Long) } diff --git a/HttpProxy/src/main/kotlin/ru/hse/Main.kt b/HttpProxy/src/main/kotlin/ru/hse/Main.kt index b73ec70..a16ea50 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/Main.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/Main.kt @@ -1,6 +1,6 @@ package ru.hse fun main() { - val acceptor = Acceptor(3030, ClientHandler(Cache(), listOf())) + val acceptor = Acceptor(3030, ClientHandler(LruCache(), listOf("google.com", "ya.ru"))) acceptor.start() } \ No newline at end of file diff --git a/HttpProxy/src/main/kotlin/ru/hse/Request.kt b/HttpProxy/src/main/kotlin/ru/hse/Request.kt new file mode 100644 index 0000000..d985510 --- /dev/null +++ b/HttpProxy/src/main/kotlin/ru/hse/Request.kt @@ -0,0 +1,10 @@ +package ru.hse + +data class Request( + val method: String, val host: String, + val port: Int, + val protocol: String, + val url: String, + val data: String, + val auth: String? +) \ No newline at end of file From 95313f91c6acff596a607eb2acdeab33399641c4 Mon Sep 17 00:00:00 2001 From: andrey Date: Wed, 27 Mar 2019 18:34:56 +0300 Subject: [PATCH 10/13] checking cache headers --- HttpProxy/src/main/kotlin/ru/hse/Cache.kt | 8 ++++++++ HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt | 14 ++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/HttpProxy/src/main/kotlin/ru/hse/Cache.kt b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt index e243308..635c822 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/Cache.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt @@ -4,4 +4,12 @@ interface Cache { fun lookUp(request: K): V? fun addPage(request: K, response: V) fun updateTime(request: K, response: V) + + companion object { + fun canBeCached(headers: List): Boolean { + val cachePolicy = headers.find { it.contains("Cache-Control") }?.split(' ')?.get(2) ?: return true + return !cachePolicy.contains("no-cache") && !cachePolicy.contains("no-store") + } + } + } diff --git a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt index 218f8ab..c18a988 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt @@ -9,7 +9,7 @@ import java.net.URL import java.util.logging.Logger import kotlin.concurrent.thread -class ClientHandler(private val cache: Cache, private val blackList: List) { +class ClientHandler(private val cache: Cache, private val blackList: List) { fun run(clientSocket: Socket): Thread { return thread { @@ -24,15 +24,17 @@ class ClientHandler(private val cache: Cache, private val black return@use } if (request.method == "GET") { - val cached = cache.lookUp(request) - if (cached != null) { - logger.info("Cached copy of response from ${request.host} was sent") - responseWith(cached, it) + if (Cache.canBeCached(request.data.split("\n"))) { + val cached = cache.lookUp(request.host) + if (cached != null) { + logger.info("Cached copy of response from ${request.host} was sent") + responseWith(cached, it) + } } else { logger.info("Fresh HTTP response from ${request.host}") val response = retrieveResponse(request) logger.info(response) - cache.addPage(request, response) + cache.addPage(request.host, response) responseWith(response, it) } From 4be74cb4279e09f899454e5c4a3b017e311aade7 Mon Sep 17 00:00:00 2001 From: andrey Date: Wed, 27 Mar 2019 19:17:04 +0300 Subject: [PATCH 11/13] checking cache headers --- HttpProxy/src/main/kotlin/ru/hse/LruCache.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt b/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt index 0a9d888..c8e8711 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt @@ -3,7 +3,7 @@ package ru.hse import java.util.* -class LruCache(val expirationTime: Int = 1000, val cacheSize: Int = 1000) : Cache { +class LruCache(private val expirationTime: Int = 1000, private val cacheSize: Int = 1000) : Cache { private val cache = Collections.synchronizedMap(LinkedHashMap>()) override fun lookUp(request: K): V? { val data = cache[request] ?: return null From 59b728310def3c1364d18595e95bb2a109e39936 Mon Sep 17 00:00:00 2001 From: andrey Date: Wed, 27 Mar 2019 20:24:44 +0300 Subject: [PATCH 12/13] caching only if can --- HttpProxy/src/main/kotlin/ru/hse/Cache.kt | 3 +- .../src/main/kotlin/ru/hse/ClientHandler.kt | 15 +++--- HttpProxy/src/main/kotlin/ru/hse/LruCache.kt | 46 +++++++++++-------- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/HttpProxy/src/main/kotlin/ru/hse/Cache.kt b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt index 635c822..e1bf96f 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/Cache.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt @@ -2,8 +2,7 @@ package ru.hse interface Cache { fun lookUp(request: K): V? - fun addPage(request: K, response: V) - fun updateTime(request: K, response: V) + fun addPage(request: K, response: V, expiration: Long = 0) companion object { fun canBeCached(headers: List): Boolean { diff --git a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt index c18a988..ef04c4b 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/ClientHandler.kt @@ -24,17 +24,18 @@ class ClientHandler(private val cache: Cache, private val blackL return@use } if (request.method == "GET") { - if (Cache.canBeCached(request.data.split("\n"))) { - val cached = cache.lookUp(request.host) - if (cached != null) { - logger.info("Cached copy of response from ${request.host} was sent") - responseWith(cached, it) - } + val cached = cache.lookUp(request.host) + val canCache = Cache.canBeCached(request.data.split("\n")) + if (canCache && cached != null) { + logger.info("Cached copy of response from ${request.host} was sent") + responseWith(cached, it) } else { logger.info("Fresh HTTP response from ${request.host}") val response = retrieveResponse(request) logger.info(response) - cache.addPage(request.host, response) + if (canCache) { + cache.addPage(request.host, response) + } responseWith(response, it) } diff --git a/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt b/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt index 0a9d888..477eb2a 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/LruCache.kt @@ -1,41 +1,49 @@ package ru.hse - import java.util.* - -class LruCache(val expirationTime: Int = 1000, val cacheSize: Int = 1000) : Cache { - private val cache = Collections.synchronizedMap(LinkedHashMap>()) +/** + * An implementation of last recently used cache. + * After each access to the element the time of last access is updated. + * So if we want to remove an element because cache is full we remove one that was accessed last. + */ +class LruCache(val expirationTime: Long = 1000, val cacheSize: Int = 1000) : Cache { + private val cache = Collections.synchronizedMap(HashMap>()) + private val order = PriorityQueue>() override fun lookUp(request: K): V? { val data = cache[request] ?: return null - if (getTime() - data.addedTime > expirationTime) { + if (getTime() > data.validTill) { cache.remove(request) return null } - data.lookedUpTime = getTime() + val newTime = getTime() + synchronized(order) { + order.remove(Expiration(data.lookedUpTime, request)) + order.add(Expiration(newTime, request)) + } + data.lookedUpTime = newTime return data.page } - override fun addPage(request: K, response: V) { + override fun addPage(request: K, response: V, expiration: Long) { if (cache.size == cacheSize) { - cache.remove(cache.iterator().next().key) + synchronized(order) { + val first = order.poll() + cache.remove(first.key) + } } val time = getTime() - cache[request] = PageData(response, time, time) - } - // Suppose don't need it - override fun updateTime(request: K, response: V) { - if (!cache.containsKey(request)) { - val time = getTime() - cache[request] = PageData(response, time, time) + val tillTime = time + (if (expiration != 0L) expiration else this.expirationTime) + synchronized(order) { + order.add(Expiration(time, request)) } - - val data = cache[request] - data?.lookedUpTime = getTime() + cache[request] = PageData(response, time, tillTime) } private fun getTime(): Long = System.currentTimeMillis() / 1000 - data class PageData(val page: V, val addedTime: Long, var lookedUpTime: Long) + data class PageData(val page: V, var lookedUpTime: Long, val validTill: Long) + + data class Expiration(val time: Long, val key: K) } From e2b4276dbeffbf679e8b62b22c59e5c6437d72c0 Mon Sep 17 00:00:00 2001 From: andrey Date: Wed, 27 Mar 2019 21:27:26 +0300 Subject: [PATCH 13/13] expiration check --- HttpProxy/src/main/kotlin/ru/hse/Cache.kt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/HttpProxy/src/main/kotlin/ru/hse/Cache.kt b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt index e1bf96f..54c4678 100644 --- a/HttpProxy/src/main/kotlin/ru/hse/Cache.kt +++ b/HttpProxy/src/main/kotlin/ru/hse/Cache.kt @@ -9,6 +9,13 @@ interface Cache { val cachePolicy = headers.find { it.contains("Cache-Control") }?.split(' ')?.get(2) ?: return true return !cachePolicy.contains("no-cache") && !cachePolicy.contains("no-store") } + + fun getExpirationTime(headers: List): Long { + val expiration = headers.find { it.contains("Cache-Control") }?.split(' ')?.get(2) ?: return 0 + if (expiration.contains("max-age=")) + return expiration.split('=')[2].toLong() + return 0 + } } }