Skip to main content

relay_server/services/projects/source/
upstream.rs

1use std::borrow::Cow;
2use std::collections::HashMap;
3use std::collections::HashSet;
4use std::collections::hash_map::Entry;
5use std::sync::Arc;
6use std::time::Duration;
7
8use futures::stream::FuturesUnordered;
9use futures::{StreamExt, future};
10use itertools::Itertools;
11use relay_base_schema::project::ProjectKey;
12use relay_config::{Config, ConfigSnapshot};
13use relay_dynamic_config::ErrorBoundary;
14use relay_statsd::metric;
15use relay_system::{
16    Addr, BroadcastChannel, BroadcastResponse, BroadcastSender, FromMessage, Interface, Service,
17};
18use serde::{Deserialize, Serialize};
19use tokio::task::JoinHandle;
20use tokio::time::Instant;
21
22use crate::services::projects::project::Revision;
23use crate::services::projects::project::{IncomingProjectState, ProjectState};
24use crate::services::projects::source::{FetchProjectState, SourceProjectState};
25use crate::services::upstream::{
26    Method, RequestPriority, SendQuery, UpstreamQuery, UpstreamRelay, UpstreamRequestError,
27};
28use crate::statsd::{RelayCounters, RelayDistributions, RelayTimers};
29use crate::utils::{RetryBackoff, SleepHandle};
30
31/// A query to retrieve a batch of project states from upstream.
32///
33/// This query does not implement `Deserialize`. To parse the query, use a wrapper that skips
34/// invalid project keys instead of failing the entire batch.
35#[derive(Debug, Serialize)]
36#[serde(rename_all = "camelCase")]
37pub struct GetProjectStates {
38    /// List of requested project keys.
39    public_keys: Vec<ProjectKey>,
40    /// List of revisions for each project key.
41    ///
42    /// The revisions are mapped by index to the project key,
43    /// this is a separate field to keep the API compatible.
44    revisions: Vec<Revision>,
45    /// If `true` the upstream should return a full configuration.
46    ///
47    /// Upstreams will ignore this for non-internal Relays.
48    full_config: bool,
49    /// If `true` the upstream should not serve from cache.
50    no_cache: bool,
51}
52
53/// The response of the projects states requests.
54///
55/// A [`ProjectKey`] is either pending or has a result, it can not appear in both and doing
56/// so is undefined.
57#[derive(Debug, Deserialize)]
58#[serde(rename_all = "camelCase")]
59pub struct GetProjectStatesResponse {
60    /// Map of [`ProjectKey`] to [`IncomingProjectState`] that was fetched from the upstream.
61    #[serde(default)]
62    configs: HashMap<ProjectKey, ErrorBoundary<Option<IncomingProjectState>>>,
63    /// The [`ProjectKey`]'s that couldn't be immediately retrieved from the upstream.
64    #[serde(default)]
65    pending: HashSet<ProjectKey>,
66    /// The [`ProjectKey`]'s that the upstream has no updates for.
67    ///
68    /// List is only populated when the request contains revision information
69    /// for all requested configurations.
70    #[serde(default)]
71    unchanged: HashSet<ProjectKey>,
72}
73
74impl UpstreamQuery for GetProjectStates {
75    type Response = GetProjectStatesResponse;
76
77    fn method(&self) -> Method {
78        Method::POST
79    }
80
81    fn path(&self) -> Cow<'static, str> {
82        Cow::Borrowed("/api/0/relays/projectconfigs/?version=3")
83    }
84
85    fn priority() -> RequestPriority {
86        RequestPriority::High
87    }
88
89    fn retry() -> bool {
90        false
91    }
92
93    fn route(&self) -> &'static str {
94        "project_configs"
95    }
96}
97
98/// Error returned to the requester.
99#[derive(Copy, Clone, Debug, thiserror::Error)]
100pub enum Error {
101    /// Fetching the project state exceeded the configured deadline.
102    #[error("deadline exceeded while fetching project state")]
103    DeadlineExceeded,
104}
105
106/// The wrapper struct for the incoming external requests which also keeps addition information.
107#[derive(Debug)]
108struct ProjectStateChannel {
109    // Main broadcast channel.
110    channel: BroadcastChannel<Message>,
111    // Additional broadcast channels tracked from merge operations.
112    merged: Vec<BroadcastChannel<Message>>,
113    revision: Revision,
114    deadline: Instant,
115    no_cache: bool,
116    attempts: u64,
117    /// How often the request failed.
118    errors: usize,
119    /// How often a "pending" response was received for this project state.
120    pending: usize,
121}
122
123impl ProjectStateChannel {
124    pub fn new(
125        sender: BroadcastSender<Message>,
126        revision: Revision,
127        timeout: Duration,
128        no_cache: bool,
129    ) -> Self {
130        let now = Instant::now();
131        Self {
132            no_cache,
133            channel: sender.into_channel(),
134            merged: Vec::new(),
135            revision,
136            deadline: now + timeout,
137            attempts: 0,
138            errors: 0,
139            pending: 0,
140        }
141    }
142
143    pub fn no_cache(&mut self) {
144        self.no_cache = true;
145    }
146
147    /// Attaches a new sender to the same channel.
148    ///
149    /// Also makes sure the new sender's revision matches the already requested revision.
150    /// If the new revision is different from the contained revision this clears the revision.
151    /// To not have multiple fetches per revision per batch, we need to find a common denominator
152    /// for requests with different revisions, which is always to fetch the full project config.
153    pub fn attach(&mut self, sender: BroadcastSender<Message>, revision: Revision) {
154        self.channel.attach(sender);
155        if self.revision != revision {
156            self.revision = Revision::default();
157        }
158    }
159
160    pub fn send(self, state: SourceProjectState) {
161        self.do_send(Ok(state));
162    }
163
164    pub fn error(self, err: Error) {
165        self.do_send(Err(err));
166    }
167
168    pub fn expired(&self, now: Instant) -> bool {
169        now > self.deadline
170    }
171
172    pub fn merge(&mut self, channel: ProjectStateChannel) {
173        let ProjectStateChannel {
174            channel,
175            merged,
176            revision,
177            deadline,
178            no_cache,
179            attempts,
180            errors,
181            pending,
182        } = channel;
183
184        self.merged.push(channel);
185        self.merged.extend(merged);
186        if self.revision != revision {
187            self.revision = Revision::default();
188        }
189        self.deadline = self.deadline.max(deadline);
190        self.no_cache |= no_cache;
191        self.attempts += attempts;
192        self.errors += errors;
193        self.pending += pending;
194    }
195
196    fn do_send(self, message: Message) {
197        for channel in self.merged {
198            channel.send(message.clone());
199        }
200        self.channel.send(message)
201    }
202}
203
204/// The map of project keys with their project state channels.
205type ProjectStateChannels = HashMap<ProjectKey, ProjectStateChannel>;
206
207/// The message used to communicate with the requester.
208type Message = Result<SourceProjectState, Error>;
209
210/// This is the [`UpstreamProjectSourceService`] interface.
211///
212/// The service is responsible for fetching the [`IncomingProjectState`] from the upstream.
213/// Internally it maintains the buffer queue of the incoming requests, which got scheduled to fetch the
214/// state and takes care of the backoff in case there is a problem with the requests.
215#[derive(Debug)]
216pub struct UpstreamProjectSource(FetchProjectState, BroadcastSender<Message>);
217
218impl Interface for UpstreamProjectSource {}
219
220impl FromMessage<FetchProjectState> for UpstreamProjectSource {
221    type Response = BroadcastResponse<Message>;
222
223    fn from_message(message: FetchProjectState, sender: BroadcastSender<Message>) -> Self {
224        Self(message, sender)
225    }
226}
227
228/// The batch of the channels which used to fetch the project states.
229struct ChannelsBatch {
230    nocache_channels: Vec<(ProjectKey, ProjectStateChannel)>,
231    cache_channels: Vec<(ProjectKey, ProjectStateChannel)>,
232}
233
234/// Collected Upstream responses, with associated project state channels.
235struct UpstreamResponse {
236    channels_batch: ProjectStateChannels,
237    response: Result<GetProjectStatesResponse, UpstreamRequestError>,
238}
239
240/// The service which handles the fetching of the [`IncomingProjectState`] from upstream.
241#[derive(Debug)]
242pub struct UpstreamProjectSourceService {
243    backoff: RetryBackoff,
244    config: Arc<Config>,
245    upstream_relay: Addr<UpstreamRelay>,
246    state_channels: ProjectStateChannels,
247    fetch_tasks: FuturesUnordered<JoinHandle<Vec<Option<UpstreamResponse>>>>,
248    fetch_handle: SleepHandle,
249    /// Instant when the last fetch failed, `None` if there aren't any failures.
250    ///
251    /// Relay updates this value to the instant when the first fetch fails, and
252    /// resets it to `None` on successful responses. Relay does nothing during
253    /// long times without requests.
254    last_failed_fetch: Option<Instant>,
255    /// Duration of continued fetch fails before emitting an error.
256    ///
257    /// Relay emits an error if all requests for at least this interval fail.
258    failure_interval: Duration,
259}
260
261impl UpstreamProjectSourceService {
262    /// Creates a new [`UpstreamProjectSourceService`] instance.
263    pub fn new(config: Arc<Config>, upstream_relay: Addr<UpstreamRelay>) -> Self {
264        let current_config = config.current();
265
266        Self {
267            backoff: RetryBackoff::new(current_config.http_max_retry_interval()),
268            state_channels: HashMap::new(),
269            fetch_handle: SleepHandle::idle(),
270            upstream_relay,
271            fetch_tasks: FuturesUnordered::new(),
272            last_failed_fetch: None,
273            failure_interval: current_config.http_project_failure_interval(),
274            config,
275        }
276    }
277
278    /// Returns the backoff timeout for a batched upstream query.
279    ///
280    /// If previous queries succeeded, this will be the general batch interval. Additionally, an
281    /// exponentially increasing backoff is used for retrying the upstream request.
282    fn next_backoff(&mut self) -> Duration {
283        self.config.current().query_batch_interval() + self.backoff.next_backoff()
284    }
285
286    /// Prepares the batches of the cache and nocache channels which could be used to request the
287    /// project states.
288    fn prepare_batches(&mut self) -> ChannelsBatch {
289        let now = Instant::now();
290        let config = self.config.current();
291        let batch_size = config.query_batch_size();
292        let num_batches = config.max_concurrent_queries();
293
294        // Pop N items from state_channels. Intuitively, we would use
295        // `state_channels.drain().take(n)`, but that clears the entire hashmap regardless how
296        // much of the iterator is consumed.
297        //
298        // Instead, we have to collect the keys we want into a separate vector and pop them
299        // one-by-one.
300        let projects: Vec<_> = (self.state_channels.keys().copied())
301            .take(batch_size * num_batches)
302            .collect();
303
304        let fresh_channels = (projects.iter())
305            .filter_map(|id| Some((*id, self.state_channels.remove(id)?)))
306            .filter_map(|(id, channel)| {
307                if !channel.expired(now) {
308                    return Some((id, channel));
309                }
310
311                // Channel is expired, emit telemetry and notify the other end.
312                metric!(
313                    distribution(RelayDistributions::ProjectStateAttempts) = channel.attempts,
314                    result = "timeout",
315                );
316                metric!(
317                    counter(RelayCounters::ProjectUpstreamCompleted) += 1,
318                    result = "timeout",
319                );
320                relay_log::warn!(
321                    errors = channel.errors,
322                    pending = channel.pending,
323                    tags.did_error = channel.errors > 0,
324                    tags.was_pending = channel.pending > 0,
325                    tags.project_key = %id,
326                    "error fetching project state {id}: deadline exceeded",
327                );
328                channel.error(Error::DeadlineExceeded);
329
330                None
331            });
332
333        // Separate regular channels from those with the `nocache` flag. The latter go in separate
334        // requests, since the upstream will block the response.
335        let (nocache_channels, cache_channels): (Vec<_>, Vec<_>) =
336            fresh_channels.partition(|(_id, channel)| channel.no_cache);
337
338        let total_count = cache_channels.len() + nocache_channels.len();
339
340        metric!(
341            distribution(RelayDistributions::ProjectStatePending) =
342                self.state_channels.len() as u64
343        );
344
345        relay_log::debug!(
346            "updating project states for {}/{} projects (attempt {})",
347            total_count,
348            total_count + self.state_channels.len(),
349            self.backoff.attempt(),
350        );
351
352        ChannelsBatch {
353            nocache_channels,
354            cache_channels,
355        }
356    }
357
358    /// Merges a [`ProjectStateChannel`] into the existing list of tracked channels.
359    ///
360    /// A channel is removed when querying the upstream for the project,
361    /// when the upstream returns pending for this project it needs to be returned to
362    /// the list of channels. If there is already another request for the same project
363    /// outstanding those two requests must be merged.
364    fn merge_channel(&mut self, key: ProjectKey, channel: ProjectStateChannel) {
365        match self.state_channels.entry(key) {
366            Entry::Vacant(e) => {
367                e.insert(channel);
368            }
369            Entry::Occupied(mut e) => {
370                e.get_mut().merge(channel);
371            }
372        }
373    }
374
375    /// Executes an upstream request to fetch project configs.
376    ///
377    /// This assumes that currently no request is running. If the upstream request fails or new
378    /// channels are pushed in the meanwhile, this will reschedule automatically.
379    async fn fetch_states(
380        config: ConfigSnapshot,
381        upstream_relay: Addr<UpstreamRelay>,
382        channels: ChannelsBatch,
383    ) -> Vec<Option<UpstreamResponse>> {
384        let request_start = Instant::now();
385        let batch_size = config.query_batch_size();
386        let cache_batches = channels.cache_channels.into_iter().chunks(batch_size);
387        let nocache_batches = channels.nocache_channels.into_iter().chunks(batch_size);
388
389        let mut requests = vec![];
390        // The `nocache_batches.into_iter()` still must be called here, since compiler produces the
391        // error: `that nocache_batches is not an iterator`.
392        // Since `IntoChunks` is not an iterator itself but only implements `IntoIterator` trait.
393        #[allow(clippy::useless_conversion)]
394        for channels_batch in cache_batches.into_iter().chain(nocache_batches.into_iter()) {
395            let mut channels_batch: ProjectStateChannels = channels_batch.collect();
396            for channel in channels_batch.values_mut() {
397                channel.attempts += 1;
398            }
399            relay_log::debug!("sending request of size {}", channels_batch.len());
400            metric!(
401                distribution(RelayDistributions::ProjectStateRequestBatchSize) =
402                    channels_batch.len() as u64
403            );
404
405            let query = GetProjectStates {
406                public_keys: channels_batch.keys().copied().collect(),
407                revisions: channels_batch
408                    .values()
409                    .map(|c| c.revision.clone())
410                    .collect(),
411                full_config: config.processing_enabled() || config.request_full_project_config(),
412                no_cache: channels_batch.values().any(|c| c.no_cache),
413            };
414
415            // count number of http requests for project states
416            metric!(counter(RelayCounters::ProjectStateRequest) += 1);
417
418            let upstream_relay = upstream_relay.clone();
419            requests.push(async move {
420                match upstream_relay.send(SendQuery(query)).await {
421                    Ok(response) => Some(UpstreamResponse {
422                        channels_batch,
423                        response,
424                    }),
425                    // If sending of the request to upstream fails:
426                    // - drop the current batch of the channels
427                    // - report the error, since this is the case we should not have in proper
428                    //   workflow
429                    // - return `None` to signal that we do not have any response from the Upstream
430                    //   and we should ignore this.
431                    Err(_err) => {
432                        relay_log::error!(
433                            "upstream relay stopped while project upstream cache task is running"
434                        );
435                        None
436                    }
437                }
438            });
439        }
440
441        // Wait on results of all fanouts, and return the resolved responses.
442        let responses = future::join_all(requests).await;
443        metric!(timer(RelayTimers::ProjectStateRequestDuration) = request_start.elapsed());
444        responses
445    }
446
447    /// Schedules the next trigger for fetching the project states.
448    ///
449    /// The next trigger will be scheduled only if the current handle is idle.
450    fn schedule_fetch(&mut self) {
451        if self.fetch_handle.is_idle() {
452            let wait = self.next_backoff();
453            self.fetch_handle.set(wait);
454        }
455    }
456
457    /// Handles the responses from the upstream.
458    fn handle_responses(&mut self, responses: Vec<Option<UpstreamResponse>>) {
459        // Iterate only over the returned responses.
460        for response in responses.into_iter().flatten() {
461            let UpstreamResponse {
462                channels_batch,
463                response,
464            } = response;
465
466            match response {
467                Ok(mut response) => {
468                    // If a single request succeeded we reset the backoff. We decided to
469                    // only backoff if we see that the project config endpoint is
470                    // completely down and did not answer a single request successfully.
471                    //
472                    // Otherwise we might refuse to fetch any project configs because of a
473                    // single, reproducible 500 we observed for a particular project.
474                    self.backoff.reset();
475                    self.last_failed_fetch = None;
476
477                    // Count number of project states returned (via http requests).
478                    metric!(
479                        distribution(RelayDistributions::ProjectStateReceived) =
480                            response.configs.len() as u64
481                    );
482                    for (key, mut channel) in channels_batch {
483                        if response.pending.contains(&key) {
484                            channel.pending += 1;
485                            self.merge_channel(key, channel);
486                            continue;
487                        }
488
489                        let mut result = "ok";
490                        let state = if response.unchanged.contains(&key) {
491                            result = "ok_unchanged";
492                            SourceProjectState::NotModified
493                        } else {
494                            let state = response
495                                .configs
496                                .remove(&key)
497                                .unwrap_or(ErrorBoundary::Ok(None));
498
499                            let state = match state {
500                                ErrorBoundary::Err(error) => {
501                                    result = "invalid";
502                                    let error = &error as &dyn std::error::Error;
503                                    relay_log::error!(error, "error fetching project state {key}");
504                                    ProjectState::Pending
505                                }
506                                ErrorBoundary::Ok(None) => ProjectState::Disabled,
507                                ErrorBoundary::Ok(Some(state)) => state.into(),
508                            };
509
510                            SourceProjectState::New(state)
511                        };
512
513                        metric!(
514                            distribution(RelayDistributions::ProjectStateAttempts) =
515                                channel.attempts,
516                            result = result,
517                        );
518                        metric!(
519                            counter(RelayCounters::ProjectUpstreamCompleted) += 1,
520                            result = result,
521                        );
522
523                        channel.send(state);
524                    }
525                }
526                Err(err) => {
527                    self.track_failed_response();
528
529                    let attempts = channels_batch
530                        .values()
531                        .map(|b| b.attempts)
532                        .max()
533                        .unwrap_or(0);
534                    // Only log an error if the request failed more than once.
535                    // We are not interested in single failures. Our retry mechanism is able to
536                    // handle those.
537                    if attempts >= 2 {
538                        relay_log::error!(
539                            error = &err as &dyn std::error::Error,
540                            attempts = attempts,
541                            "error fetching project states",
542                        );
543                    }
544
545                    metric!(
546                        distribution(RelayDistributions::ProjectStatePending) =
547                            self.state_channels.len() as u64
548                    );
549                    // Put the channels back into the queue, we will retry again shortly.
550                    self.state_channels.extend(channels_batch.into_iter().map(
551                        |(key, mut channel)| {
552                            channel.errors += 1;
553                            (key, channel)
554                        },
555                    ))
556                }
557            }
558        }
559
560        if !self.state_channels.is_empty() {
561            self.schedule_fetch()
562        } else {
563            // No open channels left, if this is because we fetched everything we
564            // have already reset the backoff. If however, this is because we had
565            // failures but the channels have been cleaned up because the requests
566            // expired we need to reset the backoff so that the next request is not
567            // simply ignored (by handle) and does a schedule_fetch().
568            // Explanation 2: We use the backoff member for two purposes:
569            //  - 1 to schedule repeated fetch requests (at less and less frequent intervals)
570            //  - 2 as a flag to know if a fetch is already scheduled.
571            // Resetting it in here signals that we don't have a backoff scheduled (either
572            // because everything went fine or because all the requests have expired).
573            // Next time a user wants a project it should schedule fetch requests.
574            self.backoff.reset();
575        }
576    }
577
578    /// Tracks the last failed fetch, and emits an error if it exceeds the failure interval.
579    fn track_failed_response(&mut self) {
580        match self.last_failed_fetch {
581            None => self.last_failed_fetch = Some(Instant::now()),
582            Some(last_failed) => {
583                let failure_duration = last_failed.elapsed();
584                if failure_duration >= self.failure_interval {
585                    relay_log::error!(
586                        failure_duration = format!("{} seconds", failure_duration.as_secs()),
587                        backoff_attempts = self.backoff.attempt(),
588                        "can't fetch project states"
589                    );
590                }
591            }
592        }
593        metric!(counter(RelayCounters::ProjectUpstreamFailed) += 1);
594    }
595
596    /// Creates the async task to fetch the project states.
597    fn do_fetch(&mut self) {
598        self.fetch_handle.reset();
599
600        if self.state_channels.is_empty() {
601            relay_log::error!("project state schedule fetch request without projects");
602            return;
603        }
604
605        let config = self.config.current();
606        let channels = self.prepare_batches();
607        let upstream_relay = self.upstream_relay.clone();
608
609        let fetch_task = relay_system::spawn!(async move {
610            Self::fetch_states(config, upstream_relay, channels).await
611        });
612        self.fetch_tasks.push(fetch_task);
613    }
614
615    /// Handles the incoming external messages.
616    fn handle_message(&mut self, message: UpstreamProjectSource) {
617        let UpstreamProjectSource(
618            FetchProjectState {
619                project_key,
620                current_revision,
621                no_cache,
622            },
623            sender,
624        ) = message;
625
626        let query_timeout = self.config.current().query_timeout();
627
628        // If there is already channel for the requested project key, we attach to it,
629        // otherwise create a new one.
630        match self.state_channels.entry(project_key) {
631            Entry::Vacant(entry) => {
632                entry.insert(ProjectStateChannel::new(
633                    sender,
634                    current_revision,
635                    query_timeout,
636                    no_cache,
637                ));
638            }
639            Entry::Occupied(mut entry) => {
640                let channel = entry.get_mut();
641                channel.attach(sender, current_revision);
642                // Ensure upstream skips caches if one of the recipients requests an uncached response. This
643                // operation is additive across requests.
644                if no_cache {
645                    channel.no_cache();
646                }
647            }
648        }
649
650        // Schedule the fetch if there is nothing running at this moment.
651        if !self.backoff.started() {
652            self.backoff.reset();
653            self.schedule_fetch();
654        }
655    }
656}
657
658impl Service for UpstreamProjectSourceService {
659    type Interface = UpstreamProjectSource;
660
661    async fn run(mut self, mut rx: relay_system::Receiver<Self::Interface>) {
662        relay_log::info!("project upstream cache started");
663        let mut expects_input = true;
664        while expects_input || !self.state_channels.is_empty() || !self.fetch_tasks.is_empty() {
665            tokio::select! {
666                biased;
667
668                () = &mut self.fetch_handle => self.do_fetch(),
669                Some(result) = self.fetch_tasks.next() => {
670                    let responses = match result {
671                        Ok(responses) => responses,
672                        Err(error) => {
673                            relay_log::error!(
674                                error = &error as &dyn std::error::Error,
675                                "fetch task panicked or was cancelled"
676                            );
677                            Vec::new()
678                        }
679                    };
680                    self.handle_responses(responses);
681                }
682                signal = rx.recv(), if expects_input => match signal {
683                    Some(message) => self.handle_message(message),
684                    None => expects_input = false,
685                }
686            }
687        }
688        relay_log::info!("project upstream cache stopped");
689    }
690}
691
692#[cfg(test)]
693mod tests {
694    use crate::http::Response;
695    use futures::future::poll_immediate;
696
697    use super::*;
698
699    fn to_response(body: &impl serde::Serialize) -> Response {
700        let body = serde_json::to_vec(body).unwrap();
701        let response = http::response::Response::builder()
702            .status(http::StatusCode::OK)
703            .header(http::header::CONTENT_LENGTH, body.len())
704            .body(body)
705            .unwrap();
706
707        Response(response.into())
708    }
709
710    #[tokio::test(start_paused = true)]
711    async fn test_shutdown_with_closed_upstream() {
712        let (upstream, upstream_rx) = Addr::custom();
713        drop(upstream_rx);
714        let service = UpstreamProjectSourceService::new(Arc::new(Config::default()), upstream);
715        let (addr, rx) = relay_system::channel(UpstreamProjectSourceService::name());
716        let response = addr.send(FetchProjectState {
717            project_key: ProjectKey::parse("abd0f232775f45feab79864e580d160b").unwrap(),
718            current_revision: Revision::default(),
719            no_cache: false,
720        });
721        drop(addr);
722
723        tokio::time::timeout(Duration::from_secs(10), service.run(rx))
724            .await
725            .expect("project upstream cache should stop when the upstream service is gone");
726        assert!(response.await.is_err());
727    }
728
729    #[tokio::test(start_paused = true)]
730    async fn test_shutdown_drains_pending_requests() {
731        tokio::time::timeout(Duration::from_secs(10), async {
732            let (upstream, mut upstream_rx) = Addr::custom();
733            let config = Arc::new(Config::default());
734            let service = UpstreamProjectSourceService::new(Arc::clone(&config), upstream);
735            let (addr, rx) = relay_system::channel(UpstreamProjectSourceService::name());
736            let task = relay_system::spawn!(service.run(rx));
737            let project_key = ProjectKey::parse("abd0f232775f45feab79864e580d160b").unwrap();
738            let response = addr.send(FetchProjectState {
739                project_key,
740                current_revision: "123".into(),
741                no_cache: false,
742            });
743
744            let UpstreamRelay::SendRequest(mut request) = upstream_rx.recv().await.unwrap() else {
745                panic!()
746            };
747            // Close the input while a fetch is in flight. Pending projects must still be retried.
748            drop(addr);
749            request.configure(&config.current());
750            request
751                .respond(Ok(to_response(&serde_json::json!({
752                    "pending": [project_key],
753                }))))
754                .await;
755
756            let UpstreamRelay::SendRequest(mut request) = upstream_rx.recv().await.unwrap() else {
757                panic!()
758            };
759            request.configure(&config.current());
760            request
761                .respond(Ok(to_response(&serde_json::json!({
762                    "unchanged": [project_key],
763                }))))
764                .await;
765
766            assert!(matches!(
767                response.await,
768                Ok(Ok(SourceProjectState::NotModified))
769            ));
770            task.await.unwrap();
771        })
772        .await
773        .expect("project upstream cache should drain pending requests and stop");
774    }
775
776    #[tokio::test]
777    async fn test_schedule_merge_channels() {
778        let (upstream_addr, mut upstream_rx) = Addr::custom();
779        let config = Arc::new(Config::from_json_value(serde_json::json!({})).unwrap());
780        let project_key = ProjectKey::parse("abd0f232775f45feab79864e580d160b").unwrap();
781
782        macro_rules! next_send_request {
783            () => {{
784                let UpstreamRelay::SendRequest(mut req) = upstream_rx.recv().await.unwrap() else {
785                    panic!()
786                };
787                req.configure(&config.current());
788                req
789            }};
790        }
791
792        let service =
793            UpstreamProjectSourceService::new(Arc::clone(&config), upstream_addr).start_detached();
794
795        let mut response1 = service.send(FetchProjectState {
796            project_key,
797            current_revision: "123".into(),
798            no_cache: false,
799        });
800
801        // Wait for the upstream request to make sure we're in the pending state.
802        let request1 = next_send_request!();
803
804        // Add another request for the same project, which should be combined into a single
805        // request, after responding to the first inflight request.
806        let mut response2 = service.send(FetchProjectState {
807            project_key,
808            current_revision: Revision::default(),
809            no_cache: false,
810        });
811
812        // Return pending to the service.
813        // Now the two requests should be combined.
814        request1
815            .respond(Ok(to_response(&serde_json::json!({
816                "pending": [project_key],
817            }))))
818            .await;
819
820        // Make sure there is no response yet.
821        assert!(poll_immediate(&mut response1).await.is_none());
822        assert!(poll_immediate(&mut response2).await.is_none());
823
824        // Send a response to the second request which should successfully resolve both responses.
825        next_send_request!()
826            .respond(Ok(to_response(&serde_json::json!({
827                "unchanged": [project_key],
828            }))))
829            .await;
830
831        let (response1, response2) = futures::future::join(response1, response2).await;
832        assert!(matches!(response1, Ok(Ok(SourceProjectState::NotModified))));
833        assert!(matches!(response2, Ok(Ok(SourceProjectState::NotModified))));
834
835        // No more messages to upstream expected.
836        assert!(upstream_rx.try_recv().is_err());
837    }
838}