From 1859d5c5125a638746518459aed28c80011cc660 Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Wed, 12 Jan 2022 15:46:23 +0800
Subject: [PATCH 01/23] feature: preserve body after parse

---
 lib/resty/upload.lua |  60 +++++++++++++++--
 t/body_preserve.t    | 153 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 209 insertions(+), 4 deletions(-)
 create mode 100644 t/body_preserve.t

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index 37da961..8375321 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -7,6 +7,9 @@ local match = string.match
 local setmetatable = setmetatable
 local type = type
 local ngx_var = ngx.var
+local ngx_init_body = ngx.req.init_body
+local ngx_finish_body = ngx.req.finish_body
+local ngx_append_body = ngx.req.append_body
 -- local print = print
 
 
@@ -21,12 +24,46 @@ local STATE_READING_HEADER = 2
 local STATE_READING_BODY = 3
 local STATE_EOF = 4
 
+local function warped_receiveuntil(self, until_str)
+    local iter, err_outer = self:old_receiveuntil(until_str)
+    if iter == nil then
+        ngx_finish_body()
+    end
+
+    local function warped(size)
+        local ret, err = iter(size)
+        if ret then
+            ngx_append_body(ret)
+        end
+        -- successful size call and nil ret, or non-nil ret
+        if not err then
+            ngx_append_body(until_str)
+        end
+        return ret, err
+    end
+    return warped, err_outer
+end
+
+local function warped_receive(self, arg)
+    local ret, err = self:old_receive(arg)
+    if ret == nil then
+        ngx_finish_body()
+    end
+    return ret, err
+end
+
+local function req_socket_body_collector(sock)
+    sock.old_receiveuntil = sock.receiveuntil
+    sock.old_receive = sock.receive
+    sock.receiveuntil = warped_receiveuntil
+    sock.receive = warped_receive
+end
+
 
 local mt = { __index = _M }
 
 local state_handlers
 
-
 local function get_boundary()
     local header = ngx_var.content_type
     if not header then
@@ -46,7 +83,7 @@ local function get_boundary()
 end
 
 
-function _M.new(self, chunk_size, max_line_size)
+function _M.new(self, chunk_size, max_line_size, preserve_body)
     local boundary = get_boundary()
 
     -- print("boundary: ", boundary)
@@ -62,6 +99,11 @@ function _M.new(self, chunk_size, max_line_size)
         return nil, err
     end
 
+    if preserve_body then
+        ngx_init_body(chunk_size)
+        req_socket_body_collector(sock)
+    end
+
     local read2boundary, err = sock:receiveuntil("--" .. boundary)
     if not read2boundary then
         return nil, err
@@ -79,7 +121,8 @@ function _M.new(self, chunk_size, max_line_size)
         read2boundary = read2boundary,
         read_line = read_line,
         boundary = boundary,
-        state = STATE_BEGIN
+        state = STATE_BEGIN,
+        preserve_body = preserve_body
     }, mt)
 end
 
@@ -104,6 +147,9 @@ local function discard_line(self)
 
     local dummy, err = read_line(1)
     if dummy then
+        if self.preserve_body then
+            ngx_finish_body()
+        end
         return nil, "line too long: " .. line .. dummy .. "..."
     end
 
@@ -179,6 +225,9 @@ local function read_header(self)
 
     local dummy, err = read_line(1)
     if dummy then
+        if self.preserve_body then
+            ngx_finish_body()
+        end
         return nil, nil, "line too long: " .. line .. dummy .. "..."
     end
 
@@ -203,7 +252,10 @@ local function read_header(self)
 end
 
 
-local function eof()
+local function eof(self)
+    if self.preserve_body then
+        ngx_finish_body()
+    end
     return "eof", nil
 end
 
diff --git a/t/body_preserve.t b/t/body_preserve.t
new file mode 100644
index 0000000..4cb5df3
--- /dev/null
+++ b/t/body_preserve.t
@@ -0,0 +1,153 @@
+# vim:set ft= ts=4 sw=4 et:
+
+use Test::Nginx::Socket::Lua;
+use Cwd qw(cwd);
+
+repeat_each(2);
+
+plan tests => repeat_each() * (3 * blocks());
+
+my $pwd = cwd();
+
+our $HttpConfig = qq{
+    lua_package_path "$pwd/lib/?.lua;;";
+};
+
+$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
+
+no_long_string();
+#no_diff();
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: body preserve off
+--- http_config eval: $::HttpConfig
+--- config
+    location /t {
+        content_by_lua '
+            local upload = require "resty.upload"
+            local ljson = require "ljson"
+
+            local form = upload:new(5)
+
+            form:set_timeout(1000) -- 1 sec
+
+            while true do
+                local typ, res, err = form:read()
+                if not typ then
+                    ngx.say("failed to read: ", err)
+                    return
+                end
+
+                ngx.say("read: ", ljson.encode({typ, res}))
+
+                if typ == "eof" then
+                    break
+                end
+            end
+
+            local typ, res, err = form:read()
+            ngx.say("read: ", ljson.encode({typ, res}))
+
+            ngx.say("remain body: ", ngx.req.get_body_data())
+        ';
+    }
+--- more_headers
+Content-Type: multipart/form-data; boundary=---------------------------820127721219505131303151179
+--- request eval
+qq{POST /t\n-----------------------------820127721219505131303151179\r
+Content-Disposition: form-data; name="file1"; filename="a.txt"\r
+Content-Type: text/plain\r
+\r
+Hello, world\r\n-----------------------------820127721219505131303151179\r
+Content-Disposition: form-data; name="test"\r
+\r
+value\r
+\r\n-----------------------------820127721219505131303151179--\r
+}
+--- response_body
+read: ["header",["Content-Disposition","form-data; name=\"file1\"; filename=\"a.txt\"","Content-Disposition: form-data; name=\"file1\"; filename=\"a.txt\""]]
+read: ["header",["Content-Type","text/plain","Content-Type: text/plain"]]
+read: ["body","Hello"]
+read: ["body",", wor"]
+read: ["body","ld"]
+read: ["part_end"]
+read: ["header",["Content-Disposition","form-data; name=\"test\"","Content-Disposition: form-data; name=\"test\""]]
+read: ["body","value"]
+read: ["body","\r\n"]
+read: ["part_end"]
+read: ["eof"]
+read: ["eof"]
+remain body: 
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: body preserve on
+--- http_config eval: $::HttpConfig
+--- config
+    location /t {
+        content_by_lua '
+            local upload = require "resty.upload"
+            local ljson = require "ljson"
+
+            local form = upload:new(5, nil, true)
+
+            form:set_timeout(1000) -- 1 sec
+
+            while true do
+                local typ, res, err = form:read()
+                if not typ then
+                    ngx.say("failed to read: ", err)
+                    return
+                end
+
+                ngx.say("read: ", ljson.encode({typ, res}))
+
+                if typ == "eof" then
+                    break
+                end
+            end
+
+            local typ, res, err = form:read()
+            ngx.say("read: ", ljson.encode({typ, res}))
+
+            ngx.say("remain body length changed: ", #ngx.req.get_body_data() ~= ngx.req.get_headers()["Content-Length"])
+        ';
+    }
+--- more_headers
+Content-Type: multipart/form-data; boundary=---------------------------820127721219505131303151179
+--- request eval
+qq{POST /t\n-----------------------------820127721219505131303151179\r
+Content-Disposition: form-data; name="file1"; filename="a.txt"\r
+Content-Type: text/plain\r
+\r
+Hello, world\r\n-----------------------------820127721219505131303151179\r
+Content-Disposition: form-data; name="test"\r
+\r
+value\r
+\r\n-----------------------------820127721219505131303151179--\r
+}
+--- response_body
+read: ["header",["Content-Disposition","form-data; name=\"file1\"; filename=\"a.txt\"","Content-Disposition: form-data; name=\"file1\"; filename=\"a.txt\""]]
+read: ["header",["Content-Type","text/plain","Content-Type: text/plain"]]
+read: ["body","Hello"]
+read: ["body",", wor"]
+read: ["body","ld"]
+read: ["part_end"]
+read: ["header",["Content-Disposition","form-data; name=\"test\"","Content-Disposition: form-data; name=\"test\""]]
+read: ["body","value"]
+read: ["body","\r\n"]
+read: ["part_end"]
+read: ["eof"]
+read: ["eof"]
+remain body length changed: false
+
+--- no_error_log
+[error]
+
+
+

From 411e889a154cced4df4c27097a81a7bc9f26fb63 Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Wed, 12 Jan 2022 17:05:26 +0800
Subject: [PATCH 02/23] more strict check

---
 t/body_preserve.t | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/t/body_preserve.t b/t/body_preserve.t
index 4cb5df3..4905d4f 100644
--- a/t/body_preserve.t
+++ b/t/body_preserve.t
@@ -91,6 +91,7 @@ remain body:
 --- config
     location /t {
         content_by_lua '
+            local original_len = ngx.req.get_headers()["Content-Length"]
             local upload = require "resty.upload"
             local ljson = require "ljson"
 
@@ -115,7 +116,7 @@ remain body:
             local typ, res, err = form:read()
             ngx.say("read: ", ljson.encode({typ, res}))
 
-            ngx.say("remain body length changed: ", #ngx.req.get_body_data() ~= ngx.req.get_headers()["Content-Length"])
+            ngx.say("remain body length changed: ", #ngx.req.get_body_data() ~= original_len)
         ';
     }
 --- more_headers

From b35c2c2a69cb7c971680c957112a6c842c7cf780 Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Wed, 12 Jan 2022 17:12:05 +0800
Subject: [PATCH 03/23] this test does not really need ljson

---
 t/body_preserve.t | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/t/body_preserve.t b/t/body_preserve.t
index 4905d4f..70cb799 100644
--- a/t/body_preserve.t
+++ b/t/body_preserve.t
@@ -93,7 +93,6 @@ remain body:
         content_by_lua '
             local original_len = ngx.req.get_headers()["Content-Length"]
             local upload = require "resty.upload"
-            local ljson = require "ljson"
 
             local form = upload:new(5, nil, true)
 
@@ -106,7 +105,6 @@ remain body:
                     return
                 end
 
-                ngx.say("read: ", ljson.encode({typ, res}))
 
                 if typ == "eof" then
                     break
@@ -114,7 +112,6 @@ remain body:
             end
 
             local typ, res, err = form:read()
-            ngx.say("read: ", ljson.encode({typ, res}))
 
             ngx.say("remain body length changed: ", #ngx.req.get_body_data() ~= original_len)
         ';
@@ -133,18 +130,6 @@ value\r
 \r\n-----------------------------820127721219505131303151179--\r
 }
 --- response_body
-read: ["header",["Content-Disposition","form-data; name=\"file1\"; filename=\"a.txt\"","Content-Disposition: form-data; name=\"file1\"; filename=\"a.txt\""]]
-read: ["header",["Content-Type","text/plain","Content-Type: text/plain"]]
-read: ["body","Hello"]
-read: ["body",", wor"]
-read: ["body","ld"]
-read: ["part_end"]
-read: ["header",["Content-Disposition","form-data; name=\"test\"","Content-Disposition: form-data; name=\"test\""]]
-read: ["body","value"]
-read: ["body","\r\n"]
-read: ["part_end"]
-read: ["eof"]
-read: ["eof"]
 remain body length changed: false
 
 --- no_error_log

From 25923250e520f7647db6121884bfab840a72354b Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Wed, 12 Jan 2022 17:18:42 +0800
Subject: [PATCH 04/23] test should be on same file

---
 t/body_preserve.t | 139 ----------------------------------------------
 t/sanity.t        | 116 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+), 139 deletions(-)
 delete mode 100644 t/body_preserve.t

diff --git a/t/body_preserve.t b/t/body_preserve.t
deleted file mode 100644
index 70cb799..0000000
--- a/t/body_preserve.t
+++ /dev/null
@@ -1,139 +0,0 @@
-# vim:set ft= ts=4 sw=4 et:
-
-use Test::Nginx::Socket::Lua;
-use Cwd qw(cwd);
-
-repeat_each(2);
-
-plan tests => repeat_each() * (3 * blocks());
-
-my $pwd = cwd();
-
-our $HttpConfig = qq{
-    lua_package_path "$pwd/lib/?.lua;;";
-};
-
-$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
-
-no_long_string();
-#no_diff();
-
-run_tests();
-
-__DATA__
-
-=== TEST 1: body preserve off
---- http_config eval: $::HttpConfig
---- config
-    location /t {
-        content_by_lua '
-            local upload = require "resty.upload"
-            local ljson = require "ljson"
-
-            local form = upload:new(5)
-
-            form:set_timeout(1000) -- 1 sec
-
-            while true do
-                local typ, res, err = form:read()
-                if not typ then
-                    ngx.say("failed to read: ", err)
-                    return
-                end
-
-                ngx.say("read: ", ljson.encode({typ, res}))
-
-                if typ == "eof" then
-                    break
-                end
-            end
-
-            local typ, res, err = form:read()
-            ngx.say("read: ", ljson.encode({typ, res}))
-
-            ngx.say("remain body: ", ngx.req.get_body_data())
-        ';
-    }
---- more_headers
-Content-Type: multipart/form-data; boundary=---------------------------820127721219505131303151179
---- request eval
-qq{POST /t\n-----------------------------820127721219505131303151179\r
-Content-Disposition: form-data; name="file1"; filename="a.txt"\r
-Content-Type: text/plain\r
-\r
-Hello, world\r\n-----------------------------820127721219505131303151179\r
-Content-Disposition: form-data; name="test"\r
-\r
-value\r
-\r\n-----------------------------820127721219505131303151179--\r
-}
---- response_body
-read: ["header",["Content-Disposition","form-data; name=\"file1\"; filename=\"a.txt\"","Content-Disposition: form-data; name=\"file1\"; filename=\"a.txt\""]]
-read: ["header",["Content-Type","text/plain","Content-Type: text/plain"]]
-read: ["body","Hello"]
-read: ["body",", wor"]
-read: ["body","ld"]
-read: ["part_end"]
-read: ["header",["Content-Disposition","form-data; name=\"test\"","Content-Disposition: form-data; name=\"test\""]]
-read: ["body","value"]
-read: ["body","\r\n"]
-read: ["part_end"]
-read: ["eof"]
-read: ["eof"]
-remain body: 
---- no_error_log
-[error]
-
-
-
-=== TEST 2: body preserve on
---- http_config eval: $::HttpConfig
---- config
-    location /t {
-        content_by_lua '
-            local original_len = ngx.req.get_headers()["Content-Length"]
-            local upload = require "resty.upload"
-
-            local form = upload:new(5, nil, true)
-
-            form:set_timeout(1000) -- 1 sec
-
-            while true do
-                local typ, res, err = form:read()
-                if not typ then
-                    ngx.say("failed to read: ", err)
-                    return
-                end
-
-
-                if typ == "eof" then
-                    break
-                end
-            end
-
-            local typ, res, err = form:read()
-
-            ngx.say("remain body length changed: ", #ngx.req.get_body_data() ~= original_len)
-        ';
-    }
---- more_headers
-Content-Type: multipart/form-data; boundary=---------------------------820127721219505131303151179
---- request eval
-qq{POST /t\n-----------------------------820127721219505131303151179\r
-Content-Disposition: form-data; name="file1"; filename="a.txt"\r
-Content-Type: text/plain\r
-\r
-Hello, world\r\n-----------------------------820127721219505131303151179\r
-Content-Disposition: form-data; name="test"\r
-\r
-value\r
-\r\n-----------------------------820127721219505131303151179--\r
-}
---- response_body
-remain body length changed: false
-
---- no_error_log
-[error]
-
-
-
diff --git a/t/sanity.t b/t/sanity.t
index 6521a92..befe3be 100644
--- a/t/sanity.t
+++ b/t/sanity.t
@@ -648,3 +648,119 @@ read: ["eof"]
 --- no_error_log
 [error]
 
+
+
+=== TEST 11: body preserve off
+--- http_config eval: $::HttpConfig
+--- config
+    location /t {
+        content_by_lua '
+            local upload = require "resty.upload"
+            local ljson = require "ljson"
+
+            local form = upload:new(5)
+
+            form:set_timeout(1000) -- 1 sec
+
+            while true do
+                local typ, res, err = form:read()
+                if not typ then
+                    ngx.say("failed to read: ", err)
+                    return
+                end
+
+                ngx.say("read: ", ljson.encode({typ, res}))
+
+                if typ == "eof" then
+                    break
+                end
+            end
+
+            local typ, res, err = form:read()
+            ngx.say("read: ", ljson.encode({typ, res}))
+
+            ngx.say("remain body: ", ngx.req.get_body_data())
+        ';
+    }
+--- more_headers
+Content-Type: multipart/form-data; boundary=---------------------------820127721219505131303151179
+--- request eval
+qq{POST /t\n-----------------------------820127721219505131303151179\r
+Content-Disposition: form-data; name="file1"; filename="a.txt"\r
+Content-Type: text/plain\r
+\r
+Hello, world\r\n-----------------------------820127721219505131303151179\r
+Content-Disposition: form-data; name="test"\r
+\r
+value\r
+\r\n-----------------------------820127721219505131303151179--\r
+}
+--- response_body
+read: ["header",["Content-Disposition","form-data; name=\"file1\"; filename=\"a.txt\"","Content-Disposition: form-data; name=\"file1\"; filename=\"a.txt\""]]
+read: ["header",["Content-Type","text/plain","Content-Type: text/plain"]]
+read: ["body","Hello"]
+read: ["body",", wor"]
+read: ["body","ld"]
+read: ["part_end"]
+read: ["header",["Content-Disposition","form-data; name=\"test\"","Content-Disposition: form-data; name=\"test\""]]
+read: ["body","value"]
+read: ["body","\r\n"]
+read: ["part_end"]
+read: ["eof"]
+read: ["eof"]
+remain body: 
+--- no_error_log
+[error]
+
+
+
+=== TEST 12: body preserve on
+--- http_config eval: $::HttpConfig
+--- config
+    location /t {
+        content_by_lua '
+            local original_len = ngx.req.get_headers()["Content-Length"]
+            local upload = require "resty.upload"
+
+            local form = upload:new(5, nil, true)
+
+            form:set_timeout(1000) -- 1 sec
+
+            while true do
+                local typ, res, err = form:read()
+                if not typ then
+                    ngx.say("failed to read: ", err)
+                    return
+                end
+
+
+                if typ == "eof" then
+                    break
+                end
+            end
+
+            local typ, res, err = form:read()
+
+            ngx.say("remain body length changed: ", #ngx.req.get_body_data() ~= original_len)
+        ';
+    }
+--- more_headers
+Content-Type: multipart/form-data; boundary=---------------------------820127721219505131303151179
+--- request eval
+qq{POST /t\n-----------------------------820127721219505131303151179\r
+Content-Disposition: form-data; name="file1"; filename="a.txt"\r
+Content-Type: text/plain\r
+\r
+Hello, world\r\n-----------------------------820127721219505131303151179\r
+Content-Disposition: form-data; name="test"\r
+\r
+value\r
+\r\n-----------------------------820127721219505131303151179--\r
+}
+--- response_body
+remain body length changed: false
+
+--- no_error_log
+[error]
+
+

From e6be711aa7b22eb3a9bd6d3a394ef907d8d15a73 Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Thu, 13 Jan 2022 09:53:41 +0800
Subject: [PATCH 05/23] test: handle in file request body

---
 t/sanity.t | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/t/sanity.t b/t/sanity.t
index befe3be..e1b9b8c 100644
--- a/t/sanity.t
+++ b/t/sanity.t
@@ -741,7 +741,14 @@ remain body:
 
             local typ, res, err = form:read()
 
-            ngx.say("remain body length changed: ", #ngx.req.get_body_data() ~= original_len)
+            local body = ngx.req.get_body_data()
+            local new_len
+            if body then
+                new_len = #body
+            else
+                new_len = io.open(ngx.req.get_body_file(), "r"):seek("end")
+            end
+            ngx.say("remain body length changed: ", new_len ~= original_len)
         ';
     }
 --- more_headers

From 30937af3ff3cbed53eb1cbe65f26515774487cb9 Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Thu, 13 Jan 2022 10:29:30 +0800
Subject: [PATCH 06/23] bug fix

---
 lib/resty/upload.lua | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index 8375321..068f930 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -35,8 +35,8 @@ local function warped_receiveuntil(self, until_str)
         if ret then
             ngx_append_body(ret)
         end
-        -- successful size call and nil ret, or non-nil ret
-        if not err then
+        -- non-nil ret for call with no size or successful size call and nil ret
+        if (not size and ret) or (size and not ret and not err) then
             ngx_append_body(until_str)
         end
         return ret, err
@@ -48,6 +48,8 @@ local function warped_receive(self, arg)
     local ret, err = self:old_receive(arg)
     if ret == nil then
         ngx_finish_body()
+    else
+        ngx_append_body(ret)
     end
     return ret, err
 end

From fd7c347e2f73741b461b9cdaeb238e060eb8f9cb Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Thu, 13 Jan 2022 10:46:29 +0800
Subject: [PATCH 07/23] test: no body remained, not even empty string

---
 t/sanity.t | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/t/sanity.t b/t/sanity.t
index e1b9b8c..9529ebe 100644
--- a/t/sanity.t
+++ b/t/sanity.t
@@ -679,7 +679,7 @@ read: ["eof"]
             local typ, res, err = form:read()
             ngx.say("read: ", ljson.encode({typ, res}))
 
-            ngx.say("remain body: ", ngx.req.get_body_data())
+            ngx.say("remain body: ", ngx.req.get_body_data(), ',', ngx.req.get_body_file())
         ';
     }
 --- more_headers
@@ -708,7 +708,7 @@ read: ["body","\r\n"]
 read: ["part_end"]
 read: ["eof"]
 read: ["eof"]
-remain body: 
+remain body: nil,nil
 --- no_error_log
 [error]
 

From bb89a8956d170aadfda2a13103634299c438443c Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Thu, 13 Jan 2022 11:11:58 +0800
Subject: [PATCH 08/23] test: typo

---
 t/sanity.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/t/sanity.t b/t/sanity.t
index 9529ebe..9cc345c 100644
--- a/t/sanity.t
+++ b/t/sanity.t
@@ -679,7 +679,7 @@ read: ["eof"]
             local typ, res, err = form:read()
             ngx.say("read: ", ljson.encode({typ, res}))
 
-            ngx.say("remain body: ", ngx.req.get_body_data(), ',', ngx.req.get_body_file())
+            ngx.say("remain body: ", ngx.req.get_body_data(), ",", ngx.req.get_body_file())
         ';
     }
 --- more_headers

From d99414a3e471c8643f5e95e6dec42d5c02cd8897 Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Mon, 24 Jan 2022 14:26:10 +0800
Subject: [PATCH 09/23] bug fix: handling of partial data

---
 lib/resty/upload.lua | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index 068f930..bf2dd98 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -45,11 +45,15 @@ local function warped_receiveuntil(self, until_str)
 end
 
 local function warped_receive(self, arg)
-    local ret, err = self:old_receive(arg)
+    local ret, err, partial = self:old_receive(arg)
+    if ret then
+        ngx_append_body(ret)
+    elseif partial then
+        ngx_append_body(partial)
+    end
+
     if ret == nil then
         ngx_finish_body()
-    else
-        ngx_append_body(ret)
     end
     return ret, err
 end

From 12d588b34347a65dbdc4e45181719bcd12016a46 Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Mon, 24 Jan 2022 14:46:33 +0800
Subject: [PATCH 10/23] bug fix: make sure all read

---
 lib/resty/upload.lua | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index bf2dd98..d70c1a4 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -260,6 +260,7 @@ end
 
 local function eof(self)
     if self.preserve_body then
+        self.sock:receive('*a')
         ngx_finish_body()
     end
     return "eof", nil

From 061a1a4d8a170952891620e66bc43bc144bda639 Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Mon, 24 Jan 2022 14:48:06 +0800
Subject: [PATCH 11/23] specific length of content

---
 t/sanity.t | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/t/sanity.t b/t/sanity.t
index 9cc345c..d9b37d9 100644
--- a/t/sanity.t
+++ b/t/sanity.t
@@ -748,7 +748,8 @@ remain body: nil,nil
             else
                 new_len = io.open(ngx.req.get_body_file(), "r"):seek("end")
             end
-            ngx.say("remain body length changed: ", new_len ~= original_len)
+            ngx.say("content_length: ", original_len)
+            ngx.say("remain body length: ", new_len)
         ';
     }
 --- more_headers
@@ -765,7 +766,8 @@ value\r
 \r\n-----------------------------820127721219505131303151179--\r
 }
 --- response_body
-remain body length changed: false
+content_length: 338
+remain body length: 338
 
 --- no_error_log
 [error]

From dfecdac1caf8c1cdd68c2a60ca9026600129c485 Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Mon, 24 Jan 2022 15:00:51 +0800
Subject: [PATCH 12/23] easier way to regen garbage after last boundary

---
 lib/resty/upload.lua | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index d70c1a4..ddd836b 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -260,8 +260,7 @@ end
 
 local function eof(self)
     if self.preserve_body then
-        self.sock:receive('*a')
-        ngx_finish_body()
+        discard_rest(self)
     end
     return "eof", nil
 end

From 6f9e3093caa04621b725b82eff24df655be43b1f Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Mon, 24 Jan 2022 15:06:56 +0800
Subject: [PATCH 13/23] fix test: length

---
 t/sanity.t | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/t/sanity.t b/t/sanity.t
index d9b37d9..6f71202 100644
--- a/t/sanity.t
+++ b/t/sanity.t
@@ -766,8 +766,8 @@ value\r
 \r\n-----------------------------820127721219505131303151179--\r
 }
 --- response_body
-content_length: 338
-remain body length: 338
+content_length: 336
+remain body length: 336
 
 --- no_error_log
 [error]

From bd36281c091ae285406a21a885f9d03598076dbd Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Mon, 24 Jan 2022 15:19:53 +0800
Subject: [PATCH 14/23] eof should do nothing. read_body_part handles the
 garbage part.

---
 lib/resty/upload.lua | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index ddd836b..7cdd30b 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -259,9 +259,6 @@ end
 
 
 local function eof(self)
-    if self.preserve_body then
-        discard_rest(self)
-    end
     return "eof", nil
 end
 

From 948089f91c2b0cdcf43736b4e0a8921dc1ae1dcb Mon Sep 17 00:00:00 2001
From: suika <872957+suikabreaker@users.noreply.github.com>
Date: Mon, 24 Jan 2022 16:06:13 +0800
Subject: [PATCH 15/23] make less modification to original code

---
 lib/resty/upload.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index 7cdd30b..22d43d5 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -258,7 +258,7 @@ local function read_header(self)
 end
 
 
-local function eof(self)
+local function eof()
     return "eof", nil
 end
 

From f167921ececaff4b8bc34a1a7336c6542324259e Mon Sep 17 00:00:00 2001
From: Suika <872957+suikabreaker@users.noreply.github.com>
Date: Wed, 23 Mar 2022 13:33:21 +0800
Subject: [PATCH 16/23] Apply suggestions from code review

Co-authored-by: Johnny Wang <wangjiahao@openresty.com>
---
 lib/resty/upload.lua | 9 ++++++++-
 t/sanity.t           | 3 ---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index 22d43d5..2f760f2 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -24,7 +24,7 @@ local STATE_READING_HEADER = 2
 local STATE_READING_BODY = 3
 local STATE_EOF = 4
 
-local function warped_receiveuntil(self, until_str)
+local function warpped_receiveuntil(self, until_str)
     local iter, err_outer = self:old_receiveuntil(until_str)
     if iter == nil then
         ngx_finish_body()
@@ -35,12 +35,14 @@ local function warped_receiveuntil(self, until_str)
         if ret then
             ngx_append_body(ret)
         end
+
         -- non-nil ret for call with no size or successful size call and nil ret
         if (not size and ret) or (size and not ret and not err) then
             ngx_append_body(until_str)
         end
         return ret, err
     end
+
     return warped, err_outer
 end
 
@@ -48,6 +50,7 @@ local function warped_receive(self, arg)
     local ret, err, partial = self:old_receive(arg)
     if ret then
         ngx_append_body(ret)
+
     elseif partial then
         ngx_append_body(partial)
     end
@@ -55,9 +58,11 @@ local function warped_receive(self, arg)
     if ret == nil then
         ngx_finish_body()
     end
+
     return ret, err
 end
 
+
 local function req_socket_body_collector(sock)
     sock.old_receiveuntil = sock.receiveuntil
     sock.old_receive = sock.receive
@@ -156,6 +161,7 @@ local function discard_line(self)
         if self.preserve_body then
             ngx_finish_body()
         end
+
         return nil, "line too long: " .. line .. dummy .. "..."
     end
 
@@ -234,6 +240,7 @@ local function read_header(self)
         if self.preserve_body then
             ngx_finish_body()
         end
+ 
         return nil, nil, "line too long: " .. line .. dummy .. "..."
     end
 
diff --git a/t/sanity.t b/t/sanity.t
index 6f71202..9fac009 100644
--- a/t/sanity.t
+++ b/t/sanity.t
@@ -733,7 +733,6 @@ remain body: nil,nil
                     return
                 end
 
-
                 if typ == "eof" then
                     break
                 end
@@ -771,5 +770,3 @@ remain body length: 336
 
 --- no_error_log
 [error]
-
-

From d2f9befdae52586ab810113585cc2c986c7ac4d4 Mon Sep 17 00:00:00 2001
From: Suika <872957+suikabreaker@users.noreply.github.com>
Date: Wed, 23 Mar 2022 13:39:37 +0800
Subject: [PATCH 17/23] bug fix

---
 lib/resty/upload.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index 2f760f2..31eaa4c 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -66,7 +66,7 @@ end
 local function req_socket_body_collector(sock)
     sock.old_receiveuntil = sock.receiveuntil
     sock.old_receive = sock.receive
-    sock.receiveuntil = warped_receiveuntil
+    sock.receiveuntil = warpped_receiveuntil
     sock.receive = warped_receive
 end
 

From 66c0040fb3cf6971f04aee83bd4e04db10c449ed Mon Sep 17 00:00:00 2001
From: Suika <872957+suikabreaker@users.noreply.github.com>
Date: Wed, 23 Mar 2022 13:43:43 +0800
Subject: [PATCH 18/23] Apply suggestion

---
 lib/resty/upload.lua | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index 31eaa4c..640dff0 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -24,6 +24,10 @@ local STATE_READING_HEADER = 2
 local STATE_READING_BODY = 3
 local STATE_EOF = 4
 
+local mt = { __index = _M }
+
+local state_handlers
+
 local function warpped_receiveuntil(self, until_str)
     local iter, err_outer = self:old_receiveuntil(until_str)
     if iter == nil then
@@ -46,6 +50,7 @@ local function warpped_receiveuntil(self, until_str)
     return warped, err_outer
 end
 
+
 local function warped_receive(self, arg)
     local ret, err, partial = self:old_receive(arg)
     if ret then
@@ -71,10 +76,6 @@ local function req_socket_body_collector(sock)
 end
 
 
-local mt = { __index = _M }
-
-local state_handlers
-
 local function get_boundary()
     local header = ngx_var.content_type
     if not header then

From 9c011311d83e9d2881ca79d9a7a38212013dccf3 Mon Sep 17 00:00:00 2001
From: Suika <872957+suikabreaker@users.noreply.github.com>
Date: Wed, 23 Mar 2022 14:08:40 +0800
Subject: [PATCH 19/23] Update README.markdown

---
 README.markdown | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/README.markdown b/README.markdown
index e841603..7999969 100644
--- a/README.markdown
+++ b/README.markdown
@@ -161,6 +161,19 @@ not want to save the data on local file systems.
 
 [Back to TOC](#table-of-contents)
 
+Usage
+=====
+
+```lua
+local upload = require "resty.upload"
+local form, err = upload:new(self, chunk_size, max_line_size, preserve_body)
+```
+`chunk_size` defaults to 4096. This will be the size used to read the socket.
+
+`max_line_size` defaults to 512. This will be the size limit to read the chunked body header.
+
+By Default, `lua-resty-upload` will consume the request body. In proxy mode this means upstream will not see the body. When `preserve_body` is set to true, the request body will be preserved.
+
 Author
 ======
 

From 16f9ffb21a362cf0cc56b18bcdfbdd67b08d074c Mon Sep 17 00:00:00 2001
From: Suika <872957+suikabreaker@users.noreply.github.com>
Date: Wed, 23 Mar 2022 14:12:29 +0800
Subject: [PATCH 20/23] Update README.markdown

---
 README.markdown | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.markdown b/README.markdown
index 7999969..ebdd291 100644
--- a/README.markdown
+++ b/README.markdown
@@ -168,11 +168,11 @@ Usage
 local upload = require "resty.upload"
 local form, err = upload:new(self, chunk_size, max_line_size, preserve_body)
 ```
-`chunk_size` defaults to 4096. This will be the size used to read the socket.
+`chunk_size` defaults to 4096. It will be the size used to read the socket.
 
-`max_line_size` defaults to 512. This will be the size limit to read the chunked body header.
+`max_line_size` defaults to 512. It will be the size limit to read the chunked body header.
 
-By Default, `lua-resty-upload` will consume the request body. In proxy mode this means upstream will not see the body. When `preserve_body` is set to true, the request body will be preserved.
+By Default, `lua-resty-upload` will consume the request body. For proxy mode this means upstream will not see the body. When `preserve_body` is set to true, the request body will be preserved. Note that this option is not free. When enabled, it will double the memory usage of `resty.upload`.
 
 Author
 ======

From 4892c7687b90b8ff2cb0716544b2060a7009bcd8 Mon Sep 17 00:00:00 2001
From: Suika <872957+suikabreaker@users.noreply.github.com>
Date: Thu, 24 Mar 2022 13:14:23 +0800
Subject: [PATCH 21/23] fix typo

---
 lib/resty/upload.lua | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index 640dff0..944086d 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -28,7 +28,7 @@ local mt = { __index = _M }
 
 local state_handlers
 
-local function warpped_receiveuntil(self, until_str)
+local function wrapped_receiveuntil(self, until_str)
     local iter, err_outer = self:old_receiveuntil(until_str)
     if iter == nil then
         ngx_finish_body()
@@ -71,7 +71,7 @@ end
 local function req_socket_body_collector(sock)
     sock.old_receiveuntil = sock.receiveuntil
     sock.old_receive = sock.receive
-    sock.receiveuntil = warpped_receiveuntil
+    sock.receiveuntil = wrapped_receiveuntil
     sock.receive = warped_receive
 end
 

From 3fd4fa608a58e3f7c9d18b94fd35c8854f6861d8 Mon Sep 17 00:00:00 2001
From: Suika <872957+suikabreaker@users.noreply.github.com>
Date: Thu, 24 Mar 2022 17:08:51 +0800
Subject: [PATCH 22/23] fix typo

---
 lib/resty/upload.lua | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/resty/upload.lua b/lib/resty/upload.lua
index 944086d..7a6c805 100644
--- a/lib/resty/upload.lua
+++ b/lib/resty/upload.lua
@@ -34,7 +34,7 @@ local function wrapped_receiveuntil(self, until_str)
         ngx_finish_body()
     end
 
-    local function warped(size)
+    local function wrapped(size)
         local ret, err = iter(size)
         if ret then
             ngx_append_body(ret)
@@ -47,11 +47,11 @@ local function wrapped_receiveuntil(self, until_str)
         return ret, err
     end
 
-    return warped, err_outer
+    return wrapped, err_outer
 end
 
 
-local function warped_receive(self, arg)
+local function wrapped_receive(self, arg)
     local ret, err, partial = self:old_receive(arg)
     if ret then
         ngx_append_body(ret)
@@ -72,7 +72,7 @@ local function req_socket_body_collector(sock)
     sock.old_receiveuntil = sock.receiveuntil
     sock.old_receive = sock.receive
     sock.receiveuntil = wrapped_receiveuntil
-    sock.receive = warped_receive
+    sock.receive = wrapped_receive
 end
 
 

From 9b96757d399ed07c6a7ca7c6b933f5d56b091cfe Mon Sep 17 00:00:00 2001
From: Suika <872957+suikabreaker@users.noreply.github.com>
Date: Tue, 29 Mar 2022 22:53:23 +0800
Subject: [PATCH 23/23] Apply suggestions from code review

Co-authored-by: lijunlong <lijunlong@openresty.com>
---
 README.markdown | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.markdown b/README.markdown
index ebdd291..0fb69b4 100644
--- a/README.markdown
+++ b/README.markdown
@@ -168,9 +168,9 @@ Usage
 local upload = require "resty.upload"
 local form, err = upload:new(self, chunk_size, max_line_size, preserve_body)
 ```
-`chunk_size` defaults to 4096. It will be the size used to read the socket.
+`chunk_size` defaults to 4096. It is the size used to read data from the socket.
 
-`max_line_size` defaults to 512. It will be the size limit to read the chunked body header.
+`max_line_size` defaults to 512. It is the size limit to read the chunked body header.
 
 By Default, `lua-resty-upload` will consume the request body. For proxy mode this means upstream will not see the body. When `preserve_body` is set to true, the request body will be preserved. Note that this option is not free. When enabled, it will double the memory usage of `resty.upload`.