freezes occasionally #36
Description
suspect blocking thread in async function :(
Stack Trace (10 threads)
============================================================
Thread 1 (blade)
----------------------------------------
# 0 0x00007e7a8380a43d
at __GI___poll
# 1 0x00007e7a83a8080f
at pqSocketCheck
# 2 0x00007e7a83a79c3e
at PQgetResult
# 3 0x000058a13b924e7b
at diesel::pg::connection::stmt::Statement::execute::hf517f6e21624669a
# 4 0x000058a13b767195
at <diesel::pg::connection::PgConnection as diesel::connection::Connection>::execute_returning_count::h09fde098937bf17b
# 5 0x000058a13b6ffbef
at <diesel_tracing::pg::InstrumentedPgConnection as diesel::connection::Connection>::execute_returning_count::h44542f7286b29aa4
# 6 0x000058a13b75318c
at <db::postgres::Postgres as state::DB>::delete_invocations_since::h0bc48411eeb177ce
# 7 0x000058a13ab88b34
at <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::hdd2fc3d1a3b99245
# 8 0x000058a13af4a793
at blade::main::{{closure}}::{{closure}}::ha3b6d3568341f75e
# 9 0x000058a13af462be
at blade::main::{{closure}}::h040e2c6f7a04e438
#10 0x000058a13af2c2df
at std::thread::local::LocalKey<T>::with::h833c94ece7a1b473
#11 0x000058a13ab85d81
at <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h7137c7de1b2d7692
#12 0x000058a13af67620
at tokio::runtime::runtime::Runtime::block_on::h519bb677f34cc99c
#13 0x000058a13af03999
at blade::main::hca51cb5d85083aa7
#14 0x000058a13aee38d3
at std::sys::backtrace::__rust_begin_short_backtrace::h93f6eea6e322004d
#15 0x000058a13acda19d
at std::rt::lang_start::{{closure}}::h195e2aea8ceda785
#16 0x000058a13c2fc510
at std::rt::lang_start_internal::h7d5fc939eaf0226f
#17 0x000058a13af1bad5
at main
#18 0x00007e7a837204d8
at __libc_start_call_main
#19 0x00007e7a8372059b
at __libc_start_main_alias_1
#20 0x000058a13a835ca5
at _start
Thread 7 (console_subscri)
----------------------------------------
# 0 0x00007e7a838191fa
at epoll_wait
# 1 0x000058a13c2d70cb
at mio::poll::Poll::poll::h6c432822664708dd
# 2 0x000058a13c2c0159
at tokio::runtime::io::driver::Driver::turn::h4d8d037f3238eebf
# 3 0x000058a13c2c8504
at tokio::runtime::time::Driver::park_internal::hbb375db930a1493f
# 4 0x000058a13c2a9817
at tokio::runtime::scheduler::current_thread::Context::park::h50cc4eda67258b98
# 5 0x000058a13af666b8
at tokio::runtime::runtime::Runtime::block_on::h1191f8d8a779599a
# 6 0x000058a13aee3a09
at std::sys::backtrace::__rust_begin_short_backtrace::haf3f7742c347f573
# 7 0x000058a13ad24a1f
at core::ops::function::FnOnce::call_once{{vtable.shim}}::h98d3fbc1fab828d1
# 8 0x000058a13c30310f
at std::sys::thread::unix::Thread::new::thread_start::hb28ad79a6a69b48c
# 9 0x00007e7a8379097a
at start_thread
#10 0x00007e7a83818af4
at __clone
Thread 8 (r2d2-worker-0)
----------------------------------------
# 0 0x00007e7a838168bd
at syscall
# 1 0x000058a13c2e0f1e
at parking_lot::condvar::Condvar::wait_until_internal::hf904a85e051755b7
# 2 0x000058a13b93b190
at scheduled_thread_pool::Worker::run::h3f8a64084ae994c4
# 3 0x000058a13b93bdd4
at std::sys::backtrace::__rust_begin_short_backtrace::h4ca95780529bd463
# 4 0x000058a13b93d2ce
at core::ops::function::FnOnce::call_once{{vtable.shim}}::h55db5451dad90209
# 5 0x000058a13c30310f
at std::sys::thread::unix::Thread::new::thread_start::hb28ad79a6a69b48c
# 6 0x00007e7a8379097a
at start_thread
# 7 0x00007e7a83818af4
at __clone
Thread 9 (r2d2-worker-1)
----------------------------------------
# 0 0x00007e7a838168bd
at syscall
# 1 0x000058a13c2e0f1e
at parking_lot::condvar::Condvar::wait_until_internal::hf904a85e051755b7
# 2 0x000058a13b93b190
at scheduled_thread_pool::Worker::run::h3f8a64084ae994c4
# 3 0x000058a13b93bdd4
at std::sys::backtrace::__rust_begin_short_backtrace::h4ca95780529bd463
# 4 0x000058a13b93d2ce
at core::ops::function::FnOnce::call_once{{vtable.shim}}::h55db5451dad90209
# 5 0x000058a13c30310f
at std::sys::thread::unix::Thread::new::thread_start::hb28ad79a6a69b48c
# 6 0x00007e7a8379097a
at start_thread
# 7 0x00007e7a83818af4
at __clone
Thread 10 (r2d2-worker-2)
----------------------------------------
# 0 0x00007e7a838168bd
at syscall
# 1 0x000058a13c2e0f1e
at parking_lot::condvar::Condvar::wait_until_internal::hf904a85e051755b7
# 2 0x000058a13b93b190
at scheduled_thread_pool::Worker::run::h3f8a64084ae994c4
# 3 0x000058a13b93bdd4
at std::sys::backtrace::__rust_begin_short_backtrace::h4ca95780529bd463
# 4 0x000058a13b93d2ce
at core::ops::function::FnOnce::call_once{{vtable.shim}}::h55db5451dad90209
# 5 0x000058a13c30310f
at std::sys::thread::unix::Thread::new::thread_start::hb28ad79a6a69b48c
# 6 0x00007e7a8379097a
at start_thread
# 7 0x00007e7a83818af4
at __clone
Thread 11 (actix-rt|system)
----------------------------------------
# 0 0x00007e7a838191fa
at epoll_wait
# 1 0x000058a13c2d70cb
at mio::poll::Poll::poll::h6c432822664708dd
# 2 0x000058a13c2c0159
at tokio::runtime::io::driver::Driver::turn::h4d8d037f3238eebf
# 3 0x000058a13c2c8504
at tokio::runtime::time::Driver::park_internal::hbb375db930a1493f
# 4 0x000058a13c2a9817
at tokio::runtime::scheduler::current_thread::Context::park::h50cc4eda67258b98
# 5 0x000058a13c09259c
at tokio::runtime::context::scoped::Scoped<T>::set::h62a28c81d3c3ba81
# 6 0x000058a13c0c76e3
at tokio::runtime::scheduler::current_thread::CurrentThread::block_on::hb6fc2b08c0ca9d43
# 7 0x000058a13c0970db
at tokio::runtime::runtime::Runtime::block_on::hdc0a8fbd85aa364a
# 8 0x000058a13c0bd2b7
at std::sys::backtrace::__rust_begin_short_backtrace::h4225196023ff82aa
# 9 0x000058a13c0b5265
at core::ops::function::FnOnce::call_once{{vtable.shim}}::h8ae413c0de681245
#10 0x000058a13c30310f
at std::sys::thread::unix::Thread::new::thread_start::hb28ad79a6a69b48c
#11 0x00007e7a8379097a
at start_thread
#12 0x00007e7a83818af4
at __clone
Thread 12 (actix-rt|system)
----------------------------------------
# 0 0x00007e7a838191fa
at epoll_wait
# 1 0x000058a13c2d70cb
at mio::poll::Poll::poll::h6c432822664708dd
# 2 0x000058a13c2c0159
at tokio::runtime::io::driver::Driver::turn::h4d8d037f3238eebf
# 3 0x000058a13c2c8504
at tokio::runtime::time::Driver::park_internal::hbb375db930a1493f
# 4 0x000058a13c2a9817
at tokio::runtime::scheduler::current_thread::Context::park::h50cc4eda67258b98
# 5 0x000058a13c09259c
at tokio::runtime::context::scoped::Scoped<T>::set::h62a28c81d3c3ba81
# 6 0x000058a13c0c76e3
at tokio::runtime::scheduler::current_thread::CurrentThread::block_on::hb6fc2b08c0ca9d43
# 7 0x000058a13c0970db
at tokio::runtime::runtime::Runtime::block_on::hdc0a8fbd85aa364a
# 8 0x000058a13c0bd2b7
at std::sys::backtrace::__rust_begin_short_backtrace::h4225196023ff82aa
# 9 0x000058a13c0b5265
at core::ops::function::FnOnce::call_once{{vtable.shim}}::h8ae413c0de681245
#10 0x000058a13c30310f
at std::sys::thread::unix::Thread::new::thread_start::hb28ad79a6a69b48c
#11 0x00007e7a8379097a
at start_thread
#12 0x00007e7a83818af4
at __clone
Thread 13 (actix-server ac)
----------------------------------------
# 0 0x00007e7a838191fa
at epoll_wait
# 1 0x000058a13c2d70cb
at mio::poll::Poll::poll::h6c432822664708dd
# 2 0x000058a13c0cb552
at actix_server::accept::Accept::poll_with::h2e149ca92ee7113c
# 3 0x000058a13c0bcff7
at std::sys::backtrace::__rust_begin_short_backtrace::h0aed569804848b64
# 4 0x000058a13c0b4eac
at core::ops::function::FnOnce::call_once{{vtable.shim}}::h338a5625f67cf270
# 5 0x000058a13c30310f
at std::sys::thread::unix::Thread::new::thread_start::hb28ad79a6a69b48c
# 6 0x00007e7a8379097a
at start_thread
# 7 0x00007e7a83818af4
at __clone
Thread 14 (actix-rt|system)
----------------------------------------
# 0 0x00007e7a8380aaca
at __libc_read
# 1 0x000058a13c31eb34
at <std::process::ChildStderr as std::io::Read>::read_buf::h079407cb163aaec1
# 2 0x000058a13b22e998
at std::io::default_read_exact::h00399d4b919466f1
# 3 0x000058a13b22e72b
at <serde_core::de::impls::<impl serde_core::de::Deserialize for core::result::Result<T,E>>::deserialize::ResultVisitor<T,E> as serde_core::de::Visitor>::visit_enum::h0870da3577085a7f
# 4 0x000058a13b22a820
at rstack_self::TraceOptions::trace::h990299a290394bf9
# 5 0x000058a13b22b50b
at rstack_self::trace::h96cb1998428c7818
# 6 0x000058a13ab89c6f
at <blade::admin::debug_stackz_handler as actix_web::service::HttpServiceFactory>::register::debug_stackz_handler::{{closure}}::{{closure}}::h9d9be3ad8f185e5a
# 7 0x000058a13ab934b2
at actix_web::handler::handler_service::{{closure}}::{{closure}}::h1ce033cb8d35890e
# 8 0x000058a13ab8fa2c
at <actix_web::resource::Resource<T> as actix_web::service::HttpServiceFactory>::register::{{closure}}::{{closure}}::hd1c69276db00d63a
# 9 0x000058a13ad68eab
at <tracing_actix_web::middleware::TracingResponse<F,RootSpanType> as core::future::future::Future>::poll::h175a27d024823943
#10 0x000058a13ad5a939
at <actix_service::map_err::MapErrFuture<A,Req,F,E> as core::future::future::Future>::poll::he983486573f32472
#11 0x000058a13ad5d023
at actix_http::h1::dispatcher::InnerDispatcher<T,S,B,X,U>::poll_request::heb6c8baa90b391f6
#12 0x000058a13ac40c77
at <actix_http::h1::dispatcher::Dispatcher<T,S,B,X,U> as core::future::future::Future>::poll::hbc96fbb1b7b6cdc2
#13 0x000058a13ac43964
at <actix_http::service::HttpServiceHandlerResponse<T,S,B,X,U> as core::future::future::Future>::poll::hf016737f863f3816
#14 0x000058a13ab3c98c
at <actix_server::service::StreamService<S,I> as actix_service::Service<(actix_server::worker::WorkerCounterGuard,actix_server::socket::MioStream)>>::call::{{closure}}::hcbfdf8904fa0bc39
#15 0x000058a13ab89820
at <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::hf23727555df90549
#16 0x000058a13ad3a1a2
at tokio::runtime::task::raw::poll::ha002f2635c74820b
#17 0x000058a13c2c2cb3
at tokio::task::local::LocalSet::tick::h75d6cd2a3d669027
#18 0x000058a13c09a8df
at std::thread::local::LocalKey<T>::with::he7e76a1edd3fb283
#19 0x000058a13c0aa2df
at <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::he57bb31e36efa0da
#20 0x000058a13c0c7d10
at tokio::runtime::scheduler::current_thread::Context::enter::h00a15ad4934f8be3
#21 0x000058a13c0924ae
at tokio::runtime::context::scoped::Scoped<T>::set::h62a28c81d3c3ba81
#22 0x000058a13c0c76e3
at tokio::runtime::scheduler::current_thread::CurrentThread::block_on::hb6fc2b08c0ca9d43
#23 0x000058a13c0970db
at tokio::runtime::runtime::Runtime::block_on::hdc0a8fbd85aa364a
#24 0x000058a13c0bd2b7
at std::sys::backtrace::__rust_begin_short_backtrace::h4225196023ff82aa
#25 0x000058a13c0b5265
at core::ops::function::FnOnce::call_once{{vtable.shim}}::h8ae413c0de681245
#26 0x000058a13c30310f
at std::sys::thread::unix::Thread::new::thread_start::hb28ad79a6a69b48c
#27 0x00007e7a8379097a
at start_thread
#28 0x00007e7a83818af4
at __clone
Thread 15 (actix-server ac)
----------------------------------------
# 0 0x00007e7a838191fa
at epoll_wait
# 1 0x000058a13c2d70cb
at mio::poll::Poll::poll::h6c432822664708dd
# 2 0x000058a13c0cb552
at actix_server::accept::Accept::poll_with::h2e149ca92ee7113c
# 3 0x000058a13c0bcff7
at std::sys::backtrace::__rust_begin_short_backtrace::h0aed569804848b64
# 4 0x000058a13c0b4eac
at core::ops::function::FnOnce::call_once{{vtable.shim}}::h338a5625f67cf270
# 5 0x000058a13c30310f
at std::sys::thread::unix::Thread::new::thread_start::hb28ad79a6a69b48c
# 6 0x00007e7a8379097a
at start_thread
# 7 0x00007e7a83818af4
at __clone