From fff5e3f4cdc5a8ca4f0a2e87885de7468b022dbc Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Sat, 2 Jul 2016 18:31:42 +0200 Subject: [PATCH 1/4] Experimental support for chunk cache. (with bugs) --- lib/include/client.h | 4 +++ lib/include/util.h | 1 + lib/src/client.cpp | 84 ++++++++++++++++++++++++++++++++++++-------- lib/src/util.cpp | 5 +++ 4 files changed, 80 insertions(+), 14 deletions(-) diff --git a/lib/include/client.h b/lib/include/client.h index 202a8ed..5eef18b 100644 --- a/lib/include/client.h +++ b/lib/include/client.h @@ -87,6 +87,10 @@ namespace konstructs { void process_error(Packet *packet); void process_chunk(Packet *packet); void process_chunk_updated(Packet *packet); + void cache_chunk(Vector3i pos, Packet *packet); + std::string cached_chunk_path(Vector3i pos); + bool is_chunk_cached(Vector3i pos); + void load_cached_chunk(Vector3i pos); void recv_worker(); void send_worker(); bool is_empty_chunk(Vector3i pos); diff --git a/lib/include/util.h b/lib/include/util.h index 54c5e29..de24103 100644 --- a/lib/include/util.h +++ b/lib/include/util.h @@ -30,5 +30,6 @@ GLuint gen_faces(int components, int faces, GLfloat *data); void load_png_texture(const char *file_name); void load_png_texture_from_buffer(const char *in, int size); int file_exist(const char *filename); +bool make_dir(const char *filename); #endif diff --git a/lib/src/client.cpp b/lib/src/client.cpp index 4568fea..6e5c0e0 100644 --- a/lib/src/client.cpp +++ b/lib/src/client.cpp @@ -13,10 +13,13 @@ #include #include #include +#include +#include #include #include #include #include "client.h" +#include "util.h" #define PROTOCOL_VERSION 8 #define MAX_RECV_SIZE 4096*1024 @@ -124,12 +127,59 @@ namespace konstructs { Vector3i position(p, q, k); received_chunk(position); + cache_chunk(position, packet); const int blocks_size = packet->size - 3 * sizeof(int); auto chunk = make_shared(position, pos, blocks_size, (uint8_t*)inflation_buffer); std::lock_guard lock_packets(packets_mutex); chunks.push_back(chunk); } + /* Write a chunk to local disk cache */ + void Client::cache_chunk(Vector3i pos, Packet *packet) { + std::string p_path = "cache/chunk/" + std::to_string(pos[0]); + std::string q_path = p_path + "/" + std::to_string(pos[1]); + std::string k_path = q_path + "/" + std::to_string(pos[2]); + + if (!file_exist("cache")) make_dir("cache"); + if (!file_exist("cache/chunk")) make_dir("cache/chunk"); + if (!file_exist(p_path.c_str())) make_dir(p_path.c_str()); + if (!file_exist(q_path.c_str())) make_dir(q_path.c_str()); + + ofstream cf(k_path, ofstream::binary); + cf.write(packet->buffer(),packet->size); + cf.close(); + } + + std::string Client::cached_chunk_path(Vector3i pos) { + stringstream cache_path; + cache_path << "cache/chunk/" << pos[0] << "/" << pos[1] << "/" << pos[2]; + + return cache_path.str(); + } + + bool Client::is_chunk_cached(Vector3i pos) { + return file_exist(cached_chunk_path(pos).c_str()); + } + + void Client::load_cached_chunk(Vector3i pos) { + string path(cached_chunk_path(pos)); + + // Get size of file + ifstream cf(path, ofstream::binary); + cf.seekg(0,cf.end); + long size = cf.tellg(); + cf.seekg(0); + + // make a package + auto packet = make_shared('C', size); + + // load the files content + cf.read(packet->buffer(), size); + cf.close(); + + process_chunk(packet.get()); + } + void Client::recv_worker() { std::cout<<"[Recv worker]: started"<= old_r) { + int distance = (pos - p_chunk).norm(); + + // This checks removes edges so that we request a sphere not a cube + // It also rejects chunks that was already previously added to the queue + // that is chunks within the old radius + if(distance <= r && distance >= old_r) { + if(is_empty_chunk(pos) && is_chunk_cached(pos)) { + // Missing chunk, and we have the chunk cached on disk. + load_cached_chunk(pos); + } else if(is_empty_chunk(pos)) { + // Request missing chunks with no local cache. chunks_to_fetch.push({distance, pos}); } } diff --git a/lib/src/util.cpp b/lib/src/util.cpp index f8f6ed1..0c96601 100644 --- a/lib/src/util.cpp +++ b/lib/src/util.cpp @@ -96,3 +96,8 @@ int file_exist(const char *filename) { int result = stat(filename, &st); return result == 0; } + +// TODO: Will will not work under windows, I think ... +bool make_dir(const char *filename) { + return (mkdir(filename, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != -1); +} From ceae91b7f85a9d0897a70bea0a0c50c7df70d900 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Sun, 3 Jul 2016 18:30:53 +0200 Subject: [PATCH 2/4] Fix bug with floating chunks. process_chunk used the same inflation_buffer on both threads. --- lib/include/client.h | 5 ++--- lib/src/client.cpp | 24 +++++++++++++++--------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/lib/include/client.h b/lib/include/client.h index 5eef18b..f4051f2 100644 --- a/lib/include/client.h +++ b/lib/include/client.h @@ -85,12 +85,12 @@ namespace konstructs { void send_string(const string &str); size_t recv_all(char* out_buf, const size_t size); void process_error(Packet *packet); - void process_chunk(Packet *packet); + void process_chunk(Packet *packet, char *inflation_buffer); void process_chunk_updated(Packet *packet); void cache_chunk(Vector3i pos, Packet *packet); std::string cached_chunk_path(Vector3i pos); bool is_chunk_cached(Vector3i pos); - void load_cached_chunk(Vector3i pos); + void load_cached_chunk(Vector3i pos, char* inflation_buffer); void recv_worker(); void send_worker(); bool is_empty_chunk(Vector3i pos); @@ -117,7 +117,6 @@ namespace konstructs { bool connected; bool logged_in; std::string error_message; - char *inflation_buffer; /* Chunk worker */ Vector3i player_chunk; int radius; diff --git a/lib/src/client.cpp b/lib/src/client.cpp index 6e5c0e0..242c898 100644 --- a/lib/src/client.cpp +++ b/lib/src/client.cpp @@ -37,7 +37,6 @@ namespace konstructs { recv_thread = new std::thread(&Client::recv_worker, this); send_thread = new std::thread(&Client::send_worker, this); chunk_thread = new std::thread(&Client::chunk_worker, this); - inflation_buffer = new char[BLOCK_BUFFER_SIZE]; } string Client::get_error_message() { @@ -112,7 +111,7 @@ namespace konstructs { force_close(); } - void Client::process_chunk(Packet *packet) { + void Client::process_chunk(Packet *packet, char *inflation_buffer) { int p, q, k; char *pos = packet->buffer(); @@ -161,7 +160,7 @@ namespace konstructs { return file_exist(cached_chunk_path(pos).c_str()); } - void Client::load_cached_chunk(Vector3i pos) { + void Client::load_cached_chunk(Vector3i pos, char* inflation_buffer) { string path(cached_chunk_path(pos)); // Get size of file @@ -177,11 +176,12 @@ namespace konstructs { cf.read(packet->buffer(), size); cf.close(); - process_chunk(packet.get()); + process_chunk(packet.get(), inflation_buffer); } void Client::recv_worker() { std::cout<<"[Recv worker]: started"<buffer(), packet->size); // move data over to packet_buffer - if(packet->type == 'C') - process_chunk(packet.get()); - else if(packet->type == 'E') + if(packet->type == 'C') { + process_chunk(packet.get(), inflation_buffer); + } else if(packet->type == 'E') process_error(packet.get()); else if(packet->type == 'c') process_chunk_updated(packet.get()); @@ -224,6 +224,7 @@ namespace konstructs { } } } catch(const std::exception& ex) { + delete[] inflation_buffer; std::cout << "[Recv worker]: Caught exception: " << ex.what() << std::endl; std::cout << "[Recv worker]: Will assume connection is down: " << ex.what() << std::endl; error_message = "Disconnected from server."; @@ -489,6 +490,7 @@ namespace konstructs { bool chunk_changed = false; // Stores if the chunk the player is in changed // Stores the chunks that needs to be fetched in priority order priority_queue, LessThanByScore> chunks_to_fetch; + char *inflation_buffer = new char[BLOCK_BUFFER_SIZE]; while(connected && logged_in) { @@ -564,7 +566,8 @@ namespace konstructs { if(distance <= r) { if(is_empty_chunk(pos) && is_chunk_cached(pos)) { // Missing chunk, and we have the chunk cached on disk. - load_cached_chunk(pos); + load_cached_chunk(pos, inflation_buffer); + set_loaded_radius(distance); } else if(is_empty_chunk(pos)) { // Request missing chunks with no local cache. chunks_to_fetch.push({distance, pos}); @@ -593,7 +596,8 @@ namespace konstructs { if(distance <= r && distance >= old_r) { if(is_empty_chunk(pos) && is_chunk_cached(pos)) { // Missing chunk, and we have the chunk cached on disk. - load_cached_chunk(pos); + load_cached_chunk(pos, inflation_buffer); + set_loaded_radius(distance); } else if(is_empty_chunk(pos)) { // Request missing chunks with no local cache. chunks_to_fetch.push({distance, pos}); @@ -663,6 +667,8 @@ namespace konstructs { } std::this_thread::sleep_for(std::chrono::milliseconds(15)); } + + delete[] inflation_buffer; } } }; From c4fc75db71defb66f89aeee5134144bfd61eef72 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Sun, 3 Jul 2016 18:40:08 +0200 Subject: [PATCH 3/4] undo change --- lib/src/client.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/src/client.cpp b/lib/src/client.cpp index 242c898..4817be7 100644 --- a/lib/src/client.cpp +++ b/lib/src/client.cpp @@ -212,9 +212,9 @@ namespace konstructs { // read 'size' bytes from the network int r = recv_all(packet->buffer(), packet->size); // move data over to packet_buffer - if(packet->type == 'C') { + if(packet->type == 'C') process_chunk(packet.get(), inflation_buffer); - } else if(packet->type == 'E') + else if(packet->type == 'E') process_error(packet.get()); else if(packet->type == 'c') process_chunk_updated(packet.get()); @@ -567,7 +567,6 @@ namespace konstructs { if(is_empty_chunk(pos) && is_chunk_cached(pos)) { // Missing chunk, and we have the chunk cached on disk. load_cached_chunk(pos, inflation_buffer); - set_loaded_radius(distance); } else if(is_empty_chunk(pos)) { // Request missing chunks with no local cache. chunks_to_fetch.push({distance, pos}); @@ -597,7 +596,6 @@ namespace konstructs { if(is_empty_chunk(pos) && is_chunk_cached(pos)) { // Missing chunk, and we have the chunk cached on disk. load_cached_chunk(pos, inflation_buffer); - set_loaded_radius(distance); } else if(is_empty_chunk(pos)) { // Request missing chunks with no local cache. chunks_to_fetch.push({distance, pos}); From 2afc948f9f388bf709c92c0d50bf7d7ff8669ac8 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Sun, 3 Jul 2016 18:40:41 +0200 Subject: [PATCH 4/4] Update loaded_radius when we uses cache. --- lib/src/client.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/src/client.cpp b/lib/src/client.cpp index 4817be7..4321543 100644 --- a/lib/src/client.cpp +++ b/lib/src/client.cpp @@ -567,6 +567,7 @@ namespace konstructs { if(is_empty_chunk(pos) && is_chunk_cached(pos)) { // Missing chunk, and we have the chunk cached on disk. load_cached_chunk(pos, inflation_buffer); + set_loaded_radius(distance); } else if(is_empty_chunk(pos)) { // Request missing chunks with no local cache. chunks_to_fetch.push({distance, pos}); @@ -596,6 +597,7 @@ namespace konstructs { if(is_empty_chunk(pos) && is_chunk_cached(pos)) { // Missing chunk, and we have the chunk cached on disk. load_cached_chunk(pos, inflation_buffer); + set_loaded_radius(distance); } else if(is_empty_chunk(pos)) { // Request missing chunks with no local cache. chunks_to_fetch.push({distance, pos});