differentiate leaf and root timeout (#6255)

Author: trinity-1686a

quickwit/quickwit-config/src/node_config/mod.rs

@@ -295,6 +295,8 @@ pub struct SearcherConfig {
     pub split_cache: Option<SplitCacheLimits>,
     #[serde(default = "SearcherConfig::default_request_timeout_secs")]
     request_timeout_secs: NonZeroU64,
+    #[serde(default = "SearcherConfig::default_request_timeout_secs")]
+    leaf_request_timeout_secs: NonZeroU64,
     #[serde(default)]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub storage_timeout_policy: Option<StorageTimeoutPolicy>,
@@ -519,6 +521,7 @@ impl Default for SearcherConfig {
             aggregation_bucket_limit: 65000,
             split_cache: None,
             request_timeout_secs: Self::default_request_timeout_secs(),
+            leaf_request_timeout_secs: Self::default_request_timeout_secs(),
             storage_timeout_policy: None,
             warmup_memory_budget: ByteSize::gb(100),
             warmup_single_split_initial_allocation: ByteSize::mb(300),
@@ -528,10 +531,14 @@ impl Default for SearcherConfig {
 }
 
 impl SearcherConfig {
-    /// The timeout after which a search should be cancelled
+    /// The timeout applied at the gRPC layer for search requests
     pub fn request_timeout(&self) -> Duration {
         Duration::from_secs(self.request_timeout_secs.get())
     }
+    /// The timeout applied at the leaf search layer
+    pub fn leaf_request_timeout(&self) -> Duration {
+        Duration::from_secs(self.leaf_request_timeout_secs.get())
+    }
     fn default_request_timeout_secs() -> NonZeroU64 {
         NonZeroU64::new(30).unwrap()
     }

quickwit/quickwit-config/src/node_config/serialize.rs

@@ -680,6 +680,7 @@ mod tests {
                 _max_num_concurrent_split_streams: Some(serde::de::IgnoredAny),
                 split_cache: None,
                 request_timeout_secs: NonZeroU64::new(30).unwrap(),
+                leaf_request_timeout_secs: NonZeroU64::new(30).unwrap(),
                 storage_timeout_policy: Some(crate::StorageTimeoutPolicy {
                     min_throughtput_bytes_per_secs: 100_000,
                     timeout_millis: 2_000,

quickwit/quickwit-search/src/service.rs

@@ -198,7 +198,7 @@ impl SearchService for SearchServiceImpl {
             targeted_splits: num_splits,
             status: None,
         };
-        let timeout = self.searcher_context.searcher_config.request_timeout();
+        let timeout = self.searcher_context.searcher_config.leaf_request_timeout();
         tokio::time::timeout(timeout, tracked_future).await?
     }