merge to stable-25-3 YQ-4312 fixes for ydb streaming

ydb/apps/ydbd/exports.symlist

@@ -0,0 +1,45 @@
+{
+global:
+
+__free_hook;
+__libc_calloc;
+__libc_free;
+__libc_malloc;
+__libc_memalign;
+__libc_pvalloc;
+__libc_realloc;
+__libc_valloc;
+__malloc_hook;
+__memalign_hook;
+__realloc_hook;
+aligned_alloc;
+calloc;
+cfree;
+free;
+mallinfo;
+malloc;
+malloc_info;
+malloc_stats;
+malloc_trim;
+malloc_usable_size;
+mallopt;
+memalign;
+posix_memalign;
+pthread_equal;
+pvalloc;
+realloc;
+valloc;
+
+UdfAllocate;
+UdfFree;
+UdfAllocateWithSize;
+UdfFreeWithSize;
+UdfTerminate;
+UdfRegisterObject;
+UdfUnregisterObject;
+UdfArrowAllocate;
+UdfArrowReallocate;
+UdfArrowFree;
+
+local: *;
+};

ydb/apps/ydbd/ya.make

@@ -1,7 +1,14 @@
 PROGRAM(ydbd)
 
 IF (NOT SANITIZER_TYPE)  # for some reasons some tests with asan are failed, see comment in CPPCOM-32
-    NO_EXPORT_DYNAMIC_SYMBOLS()
+    # Disabling export of dynamic symbols allows to significantly reduce size of the stripped binary,
+    # however, to be able to use dynamic UDFs (the --udfs-dir flag of ydbd server),
+    # required explicit export of symbols from yql/essentials/public/udf/service/exception_policy/udf_service.cpp
+    IF (OS_LINUX)
+        EXPORTS_SCRIPT(ydb/apps/ydbd/exports.symlist)
+    ELSE()
+        NO_EXPORT_DYNAMIC_SYMBOLS()
+    ENDIF()
 ENDIF()
 
 IF (OS_LINUX)

ydb/core/kqp/common/events/script_executions.h

@@ -357,15 +357,15 @@ struct TEvGetScriptPhysicalGraphResponse : public TEventLocal<TEvGetScriptPhysic
         , Issues(std::move(issues))
     {}
 
-    TEvGetScriptPhysicalGraphResponse(Ydb::StatusIds::StatusCode status, NKikimrKqp::TQueryPhysicalGraph&& physicalGraph, i64 generation, NYql::TIssues issues)
+    TEvGetScriptPhysicalGraphResponse(Ydb::StatusIds::StatusCode status, std::optional<NKikimrKqp::TQueryPhysicalGraph>&& physicalGraph, i64 generation, NYql::TIssues issues)
         : Status(status)
         , PhysicalGraph(std::move(physicalGraph))
         , Generation(generation)
         , Issues(std::move(issues))
     {}
 
     Ydb::StatusIds::StatusCode Status;
-    NKikimrKqp::TQueryPhysicalGraph PhysicalGraph;
+    std::optional<NKikimrKqp::TQueryPhysicalGraph> PhysicalGraph;
     i64 Generation = 0;
     NYql::TIssues Issues;
 };

ydb/core/kqp/finalize_script_service/kqp_check_script_lease_actor.cpp

@@ -20,17 +20,18 @@ class TScriptExecutionLeaseCheckActor : public TActorBootstrapped<TScriptExecuti
     };
 
 public:
-    TScriptExecutionLeaseCheckActor(const NKikimrConfig::TQueryServiceConfig& queryServiceConfig, TIntrusivePtr<TKqpCounters> counters)
+    TScriptExecutionLeaseCheckActor(const NKikimrConfig::TQueryServiceConfig& queryServiceConfig, TDuration startupTimeout, TIntrusivePtr<TKqpCounters> counters)
         : QueryServiceConfig(queryServiceConfig)
         , Counters(counters)
+        , StartupTimeout(startupTimeout)
     {}
 
     void Bootstrap() {
         LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Bootstrap");
         Become(&TScriptExecutionLeaseCheckActor::MainState);
 
         RefreshNodesInfo();
-        ScheduleRefreshScriptExecutions();
+        Schedule(StartupTimeout, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::ScheduleRefreshScriptExecutions)));
     }
 
     STRICT_STFUNC(MainState,
@@ -64,6 +65,7 @@ class TScriptExecutionLeaseCheckActor : public TActorBootstrapped<TScriptExecuti
 
         const auto nodesCount = ev->Get()->AssignedNodes.size();
         RefreshLeasePeriod = std::max(nodesCount, static_cast<size_t>(1)) * CHECK_PERIOD;
+        HasNodesInfo = true;
 
         LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Handle discover tenant nodes result, number of nodes #" << nodesCount << ", new RefreshLeasePeriod: " << RefreshLeasePeriod);
     }
@@ -92,6 +94,11 @@ class TScriptExecutionLeaseCheckActor : public TActorBootstrapped<TScriptExecuti
         LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Do ScheduleRefreshScriptExecutions (WaitRefreshScriptExecutions: " << WaitRefreshScriptExecutions << "), next refresh after " << RefreshLeasePeriod);
         Schedule(RefreshLeasePeriod, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::ScheduleRefreshScriptExecutions)));
 
+        if (!HasNodesInfo) {
+            LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Skip ScheduleRefreshScriptExecutions, node info is not arrived");
+            return;
+        }
+
         if (!WaitRefreshScriptExecutions) {
             WaitRefreshScriptExecutions = true;
 
@@ -110,17 +117,19 @@ class TScriptExecutionLeaseCheckActor : public TActorBootstrapped<TScriptExecuti
 private:
     const NKikimrConfig::TQueryServiceConfig QueryServiceConfig;
     const TIntrusivePtr<TKqpCounters> Counters;
+    const TDuration StartupTimeout;
 
     TDuration RefreshLeasePeriod = CHECK_PERIOD;
 
     bool WaitRefreshNodes = false;
     bool WaitRefreshScriptExecutions = false;
+    bool HasNodesInfo = false;
 };
 
 }  // anonymous namespace
 
-IActor* CreateScriptExecutionLeaseCheckActor(const NKikimrConfig::TQueryServiceConfig& queryServiceConfig, TIntrusivePtr<TKqpCounters> counters) {
-    return new TScriptExecutionLeaseCheckActor(queryServiceConfig, counters);
+IActor* CreateScriptExecutionLeaseCheckActor(const NKikimrConfig::TQueryServiceConfig& queryServiceConfig, TDuration startupTimeout, TIntrusivePtr<TKqpCounters> counters) {
+    return new TScriptExecutionLeaseCheckActor(queryServiceConfig, startupTimeout, counters);
 }
 
 }  // namespace NKikimr::NKqp

ydb/core/kqp/finalize_script_service/kqp_check_script_lease_actor.h

@@ -9,6 +9,6 @@ namespace NKikimrConfig {
 
 namespace NKikimr::NKqp {
 
-NActors::IActor* CreateScriptExecutionLeaseCheckActor(const NKikimrConfig::TQueryServiceConfig& queryServiceConfig, TIntrusivePtr<TKqpCounters> counters);
+NActors::IActor* CreateScriptExecutionLeaseCheckActor(const NKikimrConfig::TQueryServiceConfig& queryServiceConfig, TDuration startupTimeout, TIntrusivePtr<TKqpCounters> counters);
 
 }  // namespace NKikimr::NKqp

ydb/core/kqp/finalize_script_service/kqp_finalize_script_service.cpp

@@ -21,9 +21,11 @@ class TKqpFinalizeScriptService : public TActorBootstrapped<TKqpFinalizeScriptSe
     TKqpFinalizeScriptService(const NKikimrConfig::TQueryServiceConfig& queryServiceConfig,
         IKqpFederatedQuerySetupFactory::TPtr federatedQuerySetupFactory,
         std::shared_ptr<NYql::NDq::IS3ActorsFactory> s3ActorsFactory,
-        bool enableBackgroundLeaseChecks)
+        bool enableBackgroundLeaseChecks,
+        TDuration leaseCheckStartupTimeout)
         : QueryServiceConfig(queryServiceConfig)
         , EnableBackgroundLeaseChecks(enableBackgroundLeaseChecks)
+        , LeaseCheckStartupTimeout(leaseCheckStartupTimeout)
         , FederatedQuerySetupFactory(federatedQuerySetupFactory)
         , S3ActorsFactory(std::move(s3ActorsFactory))
     {}
@@ -58,7 +60,7 @@ class TKqpFinalizeScriptService : public TActorBootstrapped<TKqpFinalizeScriptSe
             return;
         }
 
-        ScriptExecutionLeaseCheckActor = Register(CreateScriptExecutionLeaseCheckActor(QueryServiceConfig, Counters));
+        ScriptExecutionLeaseCheckActor = Register(CreateScriptExecutionLeaseCheckActor(QueryServiceConfig, LeaseCheckStartupTimeout, Counters));
     }
 
     STRICT_STFUNC(MainState,
@@ -187,6 +189,7 @@ class TKqpFinalizeScriptService : public TActorBootstrapped<TKqpFinalizeScriptSe
 private:
     const NKikimrConfig::TQueryServiceConfig QueryServiceConfig;
     const bool EnableBackgroundLeaseChecks = true;
+    const TDuration LeaseCheckStartupTimeout;
 
     TIntrusivePtr<TKqpCounters> Counters;
     IKqpFederatedQuerySetupFactory::TPtr FederatedQuerySetupFactory;
@@ -206,8 +209,9 @@ class TKqpFinalizeScriptService : public TActorBootstrapped<TKqpFinalizeScriptSe
 IActor* CreateKqpFinalizeScriptService(const NKikimrConfig::TQueryServiceConfig& queryServiceConfig,
     IKqpFederatedQuerySetupFactory::TPtr federatedQuerySetupFactory,
     std::shared_ptr<NYql::NDq::IS3ActorsFactory> s3ActorsFactory,
-    bool enableBackgroundLeaseChecks) {
-    return new TKqpFinalizeScriptService(queryServiceConfig, std::move(federatedQuerySetupFactory), std::move(s3ActorsFactory), enableBackgroundLeaseChecks);
+    bool enableBackgroundLeaseChecks,
+    TDuration leaseCheckStartupTimeout) {
+    return new TKqpFinalizeScriptService(queryServiceConfig, std::move(federatedQuerySetupFactory), std::move(s3ActorsFactory), enableBackgroundLeaseChecks, leaseCheckStartupTimeout);
 }
 
 }  // namespace NKikimr::NKqp

ydb/core/kqp/finalize_script_service/kqp_finalize_script_service.h

@@ -13,6 +13,7 @@ namespace NKikimr::NKqp {
 IActor* CreateKqpFinalizeScriptService(const NKikimrConfig::TQueryServiceConfig& queryServiceConfig,
     IKqpFederatedQuerySetupFactory::TPtr federatedQuerySetupFactory,
     std::shared_ptr<NYql::NDq::IS3ActorsFactory> s3ActorsFactory,
-    bool enableBackgroundLeaseChecks = true);
+    bool enableBackgroundLeaseChecks = true,
+    TDuration leaseCheckStartupTimeout = TDuration::Seconds(15));
 
 }  // namespace NKikimr::NKqp