handling 'too many siblings' #920
Description
CS respond '500 Internal Server Error' when too many siblings occurs. '503' seems to be better.
To reproduce:
set max_siblings=1
(riak1@127.0.0.1)1> application:set_env(riak_kv,max_siblings,1).
ok
put a object from 10 processes using s3curl.pl
% for i in `seq 1 10`;do s3curl.pl --id user --put testfile -- -x localhost:8080 'http://test.s3.amazonaws.com/siblings' -i&; done
then we can see 500 error.
HTTP/1.1 500 Internal Server Error
Server: Riak CS
Date: Thu, 17 Jul 2014 03:38:23 GMT
Content-Type: text/html
Content-Length: 170
<html><head><title>500 Internal Server Error</title></head><body><h1>Internal Server Error</h1>The server encountered an error while processing this request</body></html><html><head><title>500 Internal Server Error</title></head><body><h1>Internal Server Error</h1>The server encountered an error while processing this request</body></html>
console.log
2:41:19.513 [error] CRASH REPORT Process <0.17172.1> with 1 neighbours exited with reason: no function clause matching riakc_obj:get_update_metadata(<<"{too_many_siblings,2}">>) line 262 in gen_fsm:terminate/7 line 611
12:41:19.514 [error] Lager event handler error_logger_lager_h exited with reason {'EXIT',{{badmatch,["/buckets/test/objects/siblings",{error,{exit,{{function_clause,[{riakc_obj,get_update_metadata,[<<"{too_many_siblings,2}">>],[{file,"src/riakc_obj.erl"},{line,262}]},{riak_cs_utils,manifests_from_riak_object,1,[{file,"src/riak_cs_utils.erl"},{line,599}]},{riak_cs_manifest_fsm,get_and_update,4,[{file,"src/riak_cs_manifest_fsm.erl"},{line,328}]},{riak_cs_manifest_fsm,waiting_command,2,[{file,"src/riak_cs_manifest_fsm.erl"},{line,174}]},{gen_fsm,handle_msg,7,[{file,"gen_..."},...]},...]},...},...}},...]},...}}
crash.log
2014-07-17 12:41:19 =ERROR REPORT====
** State machine <0.17172.1> terminating
** Last event in was {add_new_dict,[{<<161,183,11,23,154,158,71,114,189,136,220,218,163,43,15,149>>,{lfs_manifest_v3,3,1048576,{<<"test">>,<<"siblings">>},[],"2014-07-17T03:41:19.000Z",<<161,183,11,23,154,158,71,114,189,136,220,218,163,43,15,149>>,0,<<"application/octet-stream">>,undefined,writing,{1405,568479,495845},undefined,[],undefined,undefined,undefined,undefined,{acl_v2,{"user","465ee4d8f62039bcb4899f1b3198c2f938d54958ab5ac9fc8fbe93b181dd89bc","X0WBIZ1ACIPAIMYP3WZ_"},[{{"user","465ee4d8f62039bcb4899f1b3198c2f938d54958ab5ac9fc8fbe93b181dd89bc"},['FULL_CONTROL']}],{1405,568479,495569}},[],undefined}}]}
** When State == waiting_command
** Data == {state,<<"test">>,<<"siblings">>,undefined,undefined,<0.263.0>}
** Reason for termination =
** {function_clause,[{riakc_obj,get_update_metadata,[<<"{too_many_siblings,2}">>],[{file,"src/riakc_obj.erl"},{line,262}]},{riak_cs_utils,manifests_from_riak_object,1,[{file,"src/riak_cs_utils.erl"},{line,599}]},{riak_cs_manifest_fsm,get_and_update,4,[{file,"src/riak_cs_manifest_fsm.erl"},{line,328}]},{riak_cs_manifest_fsm,waiting_command,2,[{file,"src/riak_cs_manifest_fsm.erl"},{line,174}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,494}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]}
2014-07-17 12:41:19 =CRASH REPORT====
crasher:
initial call: riak_cs_manifest_fsm:init/1
pid: <0.17172.1>
registered_name: []
exception exit: {{function_clause,[{riakc_obj,get_update_metadata,[<<"{too_many_siblings,2}">>],[{file,"src/riakc_obj.erl"},{line,262}]},{riak_cs_utils,manifests_from_riak_object,1,[{file,"src/riak_cs_utils.erl"},{line,599}]},{riak_cs_manifest_fsm,get_and_update,4,[{file,"src/riak_cs_manifest_fsm.erl"},{line,328}]},{riak_cs_manifest_fsm,waiting_command,2,[{file,"src/riak_cs_manifest_fsm.erl"},{line,174}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,494}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]},[{gen_fsm,terminate,7,[{file,"gen_fsm.erl"},{line,611}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]}
ancestors: [<0.17171.1>,riak_cs_put_fsm_sup,riak_cs_sup,<0.142.0>]
messages: [{'$gen_sync_event',{<0.17171.1>,#Ref<0.0.3.86169>},{update_manifests_with_confirmation,[{<<161,183,11,23,154,158,71,114,189,136,220,218,163,43,15,149>>,{lfs_manifest_v3,3,1048576,{<<"test">>,<<"siblings">>},[],"2014-07-17T03:41:19.000Z",<<161,183,11,23,154,158,71,114,189,136,220,218,163,43,15,149>>,0,<<"application/octet-stream">>,<<212,29,140,217,143,0,178,4,233,128,9,152,236,248,66,126>>,active,{1405,568479,495845},{1405,568479,495854},[],undefined,undefined,undefined,undefined,{acl_v2,{"user","465ee4d8f62039bcb4899f1b3198c2f938d54958ab5ac9fc8fbe93b181dd89bc","X0WBIZ1ACIPAIMYP3WZ_"},[{{"user","465ee4d8f62039bcb4899f1b3198c2f938d54958ab5ac9fc8fbe93b181dd89bc"},['FULL_CONTROL']}],{1405,568479,495569}},[],undefined}}]}}]
links: [<0.17171.1>]
dictionary: [{random_seed,{28543,11274,1722}}]
trap_exit: true
status: running
heap_size: 10946
stack_size: 24
reductions: 2015
neighbours:
neighbour: [{pid,<0.17171.1>},{registered_name,[]},{initial_call,{riak_cs_put_fsm,init,['Argument__1']}},{current_function,{gen,do_call,4}},{ancestors,[riak_cs_put_fsm_sup,riak_cs_sup,<0.142.0>]},{messages,[]},{links,[<0.293.0>,<0.17172.1>]},{dictionary,[]},{trap_exit,false},{status,waiting},{heap_size,987},{stack_size,29},{reductions,475}]
2014-07-17 12:41:19 =ERROR REPORT====
webmachine error: path="/buckets/test/objects/siblings"
{error,{exit,{{function_clause,[{riakc_obj,get_update_metadata,[<<"{too_many_siblings,2}">>],[{file,"src/riakc_obj.erl"},{line,262}]},{riak_cs_utils,manifests_from_riak_object,1,[{file,"src/riak_cs_utils.erl"},{line,599}]},{riak_cs_manifest_fsm,get_and_update,4,[{file,"src/riak_cs_manifest_fsm.erl"},{line,328}]},{riak_cs_manifest_fsm,waiting_command,2,[{file,"src/riak_cs_manifest_fsm.erl"},{line,174}]},{gen_fsm,handle_msg,7,[{file,"gen_fsm.erl"},{line,494}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,227}]}]},{gen_fsm,sync_send_event,[<0.17171.1>,{finalize,undefined},infinity]}},[{gen_fsm,sync_send_event,3,[{file,"gen_fsm.erl"},{line,214}]},{riak_cs_wm_object,finalize_request,3,[{file,"src/riak_cs_wm_object.erl"},{line,378}]},{riak_cs_wm_common,accept_body,2,[{file,"src/riak_cs_wm_common.erl"},{line,337}]},{webmachine_resource,resource_call,3,[{file,"src/webmachine_resource.erl"},{line,186}]},{webmachine_resource,do,3,[{file,"src/webmachine_resource.erl"},{line,142}]},{webmachine_decision_core,resource_call,1,[{file,"src/webmachine_decision_core.erl"},{line,48}]},{webmachine_decision_core,accept_helper,1,[{file,"src/webmachine_decision_core.erl"},{line,612}]},{webmachine_decision_core,decision,1,[{file,"src/webmachine_decision_core.erl"},{line,517}]}]}}
[{gen_fsm,sync_send_event,3,[{file,"gen_fsm.erl"},{line,214}]},{riak_cs_wm_object,finalize_request,3,[{file,"src/riak_cs_wm_object.erl"},{line,378}]},{riak_cs_wm_common,accept_body,2,[{file,"src/riak_cs_wm_common.erl"},{line,337}]},{webmachine_resource,resource_call,3,[{file,"src/webmachine_resource.erl"},{line,186}]},{webmachine_resource,do,3,[{file,"src/webmachine_resource.erl"},{line,142}]},{webmachine_decision_core,resource_call,1,[{file,"src/webmachine_decision_core.erl"},{line,48}]},{webmachine_decision_core,accept_helper,1,[{file,"src/webmachine_decision_core.erl"},{line,612}]},{webmachine_decision_core,decision,1,[{file,"src/webmachine_decision_core.erl"},{line,517}]}]