1use std::borrow::Cow;
2use std::collections::HashMap;
3use std::collections::HashSet;
4use std::collections::hash_map::Entry;
5use std::sync::Arc;
6use std::time::Duration;
7
8use futures::stream::FuturesUnordered;
9use futures::{StreamExt, future};
10use itertools::Itertools;
11use relay_base_schema::project::ProjectKey;
12use relay_config::{Config, ConfigSnapshot};
13use relay_dynamic_config::ErrorBoundary;
14use relay_statsd::metric;
15use relay_system::{
16 Addr, BroadcastChannel, BroadcastResponse, BroadcastSender, FromMessage, Interface, Service,
17};
18use serde::{Deserialize, Serialize};
19use tokio::task::JoinHandle;
20use tokio::time::Instant;
21
22use crate::services::projects::project::Revision;
23use crate::services::projects::project::{IncomingProjectState, ProjectState};
24use crate::services::projects::source::{FetchProjectState, SourceProjectState};
25use crate::services::upstream::{
26 Method, RequestPriority, SendQuery, UpstreamQuery, UpstreamRelay, UpstreamRequestError,
27};
28use crate::statsd::{RelayCounters, RelayDistributions, RelayTimers};
29use crate::utils::{RetryBackoff, SleepHandle};
30
31#[derive(Debug, Serialize)]
36#[serde(rename_all = "camelCase")]
37pub struct GetProjectStates {
38 public_keys: Vec<ProjectKey>,
40 revisions: Vec<Revision>,
45 full_config: bool,
49 no_cache: bool,
51}
52
53#[derive(Debug, Deserialize)]
58#[serde(rename_all = "camelCase")]
59pub struct GetProjectStatesResponse {
60 #[serde(default)]
62 configs: HashMap<ProjectKey, ErrorBoundary<Option<IncomingProjectState>>>,
63 #[serde(default)]
65 pending: HashSet<ProjectKey>,
66 #[serde(default)]
71 unchanged: HashSet<ProjectKey>,
72}
73
74impl UpstreamQuery for GetProjectStates {
75 type Response = GetProjectStatesResponse;
76
77 fn method(&self) -> Method {
78 Method::POST
79 }
80
81 fn path(&self) -> Cow<'static, str> {
82 Cow::Borrowed("/api/0/relays/projectconfigs/?version=3")
83 }
84
85 fn priority() -> RequestPriority {
86 RequestPriority::High
87 }
88
89 fn retry() -> bool {
90 false
91 }
92
93 fn route(&self) -> &'static str {
94 "project_configs"
95 }
96}
97
98#[derive(Copy, Clone, Debug, thiserror::Error)]
100pub enum Error {
101 #[error("deadline exceeded while fetching project state")]
103 DeadlineExceeded,
104}
105
106#[derive(Debug)]
108struct ProjectStateChannel {
109 channel: BroadcastChannel<Message>,
111 merged: Vec<BroadcastChannel<Message>>,
113 revision: Revision,
114 deadline: Instant,
115 no_cache: bool,
116 attempts: u64,
117 errors: usize,
119 pending: usize,
121}
122
123impl ProjectStateChannel {
124 pub fn new(
125 sender: BroadcastSender<Message>,
126 revision: Revision,
127 timeout: Duration,
128 no_cache: bool,
129 ) -> Self {
130 let now = Instant::now();
131 Self {
132 no_cache,
133 channel: sender.into_channel(),
134 merged: Vec::new(),
135 revision,
136 deadline: now + timeout,
137 attempts: 0,
138 errors: 0,
139 pending: 0,
140 }
141 }
142
143 pub fn no_cache(&mut self) {
144 self.no_cache = true;
145 }
146
147 pub fn attach(&mut self, sender: BroadcastSender<Message>, revision: Revision) {
154 self.channel.attach(sender);
155 if self.revision != revision {
156 self.revision = Revision::default();
157 }
158 }
159
160 pub fn send(self, state: SourceProjectState) {
161 self.do_send(Ok(state));
162 }
163
164 pub fn error(self, err: Error) {
165 self.do_send(Err(err));
166 }
167
168 pub fn expired(&self, now: Instant) -> bool {
169 now > self.deadline
170 }
171
172 pub fn merge(&mut self, channel: ProjectStateChannel) {
173 let ProjectStateChannel {
174 channel,
175 merged,
176 revision,
177 deadline,
178 no_cache,
179 attempts,
180 errors,
181 pending,
182 } = channel;
183
184 self.merged.push(channel);
185 self.merged.extend(merged);
186 if self.revision != revision {
187 self.revision = Revision::default();
188 }
189 self.deadline = self.deadline.max(deadline);
190 self.no_cache |= no_cache;
191 self.attempts += attempts;
192 self.errors += errors;
193 self.pending += pending;
194 }
195
196 fn do_send(self, message: Message) {
197 for channel in self.merged {
198 channel.send(message.clone());
199 }
200 self.channel.send(message)
201 }
202}
203
204type ProjectStateChannels = HashMap<ProjectKey, ProjectStateChannel>;
206
207type Message = Result<SourceProjectState, Error>;
209
210#[derive(Debug)]
216pub struct UpstreamProjectSource(FetchProjectState, BroadcastSender<Message>);
217
218impl Interface for UpstreamProjectSource {}
219
220impl FromMessage<FetchProjectState> for UpstreamProjectSource {
221 type Response = BroadcastResponse<Message>;
222
223 fn from_message(message: FetchProjectState, sender: BroadcastSender<Message>) -> Self {
224 Self(message, sender)
225 }
226}
227
228struct ChannelsBatch {
230 nocache_channels: Vec<(ProjectKey, ProjectStateChannel)>,
231 cache_channels: Vec<(ProjectKey, ProjectStateChannel)>,
232}
233
234struct UpstreamResponse {
236 channels_batch: ProjectStateChannels,
237 response: Result<GetProjectStatesResponse, UpstreamRequestError>,
238}
239
240#[derive(Debug)]
242pub struct UpstreamProjectSourceService {
243 backoff: RetryBackoff,
244 config: Arc<Config>,
245 upstream_relay: Addr<UpstreamRelay>,
246 state_channels: ProjectStateChannels,
247 fetch_tasks: FuturesUnordered<JoinHandle<Vec<Option<UpstreamResponse>>>>,
248 fetch_handle: SleepHandle,
249 last_failed_fetch: Option<Instant>,
255 failure_interval: Duration,
259}
260
261impl UpstreamProjectSourceService {
262 pub fn new(config: Arc<Config>, upstream_relay: Addr<UpstreamRelay>) -> Self {
264 let current_config = config.current();
265
266 Self {
267 backoff: RetryBackoff::new(current_config.http_max_retry_interval()),
268 state_channels: HashMap::new(),
269 fetch_handle: SleepHandle::idle(),
270 upstream_relay,
271 fetch_tasks: FuturesUnordered::new(),
272 last_failed_fetch: None,
273 failure_interval: current_config.http_project_failure_interval(),
274 config,
275 }
276 }
277
278 fn next_backoff(&mut self) -> Duration {
283 self.config.current().query_batch_interval() + self.backoff.next_backoff()
284 }
285
286 fn prepare_batches(&mut self) -> ChannelsBatch {
289 let now = Instant::now();
290 let config = self.config.current();
291 let batch_size = config.query_batch_size();
292 let num_batches = config.max_concurrent_queries();
293
294 let projects: Vec<_> = (self.state_channels.keys().copied())
301 .take(batch_size * num_batches)
302 .collect();
303
304 let fresh_channels = (projects.iter())
305 .filter_map(|id| Some((*id, self.state_channels.remove(id)?)))
306 .filter_map(|(id, channel)| {
307 if !channel.expired(now) {
308 return Some((id, channel));
309 }
310
311 metric!(
313 distribution(RelayDistributions::ProjectStateAttempts) = channel.attempts,
314 result = "timeout",
315 );
316 metric!(
317 counter(RelayCounters::ProjectUpstreamCompleted) += 1,
318 result = "timeout",
319 );
320 relay_log::warn!(
321 errors = channel.errors,
322 pending = channel.pending,
323 tags.did_error = channel.errors > 0,
324 tags.was_pending = channel.pending > 0,
325 tags.project_key = %id,
326 "error fetching project state {id}: deadline exceeded",
327 );
328 channel.error(Error::DeadlineExceeded);
329
330 None
331 });
332
333 let (nocache_channels, cache_channels): (Vec<_>, Vec<_>) =
336 fresh_channels.partition(|(_id, channel)| channel.no_cache);
337
338 let total_count = cache_channels.len() + nocache_channels.len();
339
340 metric!(
341 distribution(RelayDistributions::ProjectStatePending) =
342 self.state_channels.len() as u64
343 );
344
345 relay_log::debug!(
346 "updating project states for {}/{} projects (attempt {})",
347 total_count,
348 total_count + self.state_channels.len(),
349 self.backoff.attempt(),
350 );
351
352 ChannelsBatch {
353 nocache_channels,
354 cache_channels,
355 }
356 }
357
358 fn merge_channel(&mut self, key: ProjectKey, channel: ProjectStateChannel) {
365 match self.state_channels.entry(key) {
366 Entry::Vacant(e) => {
367 e.insert(channel);
368 }
369 Entry::Occupied(mut e) => {
370 e.get_mut().merge(channel);
371 }
372 }
373 }
374
375 async fn fetch_states(
380 config: ConfigSnapshot,
381 upstream_relay: Addr<UpstreamRelay>,
382 channels: ChannelsBatch,
383 ) -> Vec<Option<UpstreamResponse>> {
384 let request_start = Instant::now();
385 let batch_size = config.query_batch_size();
386 let cache_batches = channels.cache_channels.into_iter().chunks(batch_size);
387 let nocache_batches = channels.nocache_channels.into_iter().chunks(batch_size);
388
389 let mut requests = vec![];
390 #[allow(clippy::useless_conversion)]
394 for channels_batch in cache_batches.into_iter().chain(nocache_batches.into_iter()) {
395 let mut channels_batch: ProjectStateChannels = channels_batch.collect();
396 for channel in channels_batch.values_mut() {
397 channel.attempts += 1;
398 }
399 relay_log::debug!("sending request of size {}", channels_batch.len());
400 metric!(
401 distribution(RelayDistributions::ProjectStateRequestBatchSize) =
402 channels_batch.len() as u64
403 );
404
405 let query = GetProjectStates {
406 public_keys: channels_batch.keys().copied().collect(),
407 revisions: channels_batch
408 .values()
409 .map(|c| c.revision.clone())
410 .collect(),
411 full_config: config.processing_enabled() || config.request_full_project_config(),
412 no_cache: channels_batch.values().any(|c| c.no_cache),
413 };
414
415 metric!(counter(RelayCounters::ProjectStateRequest) += 1);
417
418 let upstream_relay = upstream_relay.clone();
419 requests.push(async move {
420 match upstream_relay.send(SendQuery(query)).await {
421 Ok(response) => Some(UpstreamResponse {
422 channels_batch,
423 response,
424 }),
425 Err(_err) => {
432 relay_log::error!(
433 "upstream relay stopped while project upstream cache task is running"
434 );
435 None
436 }
437 }
438 });
439 }
440
441 let responses = future::join_all(requests).await;
443 metric!(timer(RelayTimers::ProjectStateRequestDuration) = request_start.elapsed());
444 responses
445 }
446
447 fn schedule_fetch(&mut self) {
451 if self.fetch_handle.is_idle() {
452 let wait = self.next_backoff();
453 self.fetch_handle.set(wait);
454 }
455 }
456
457 fn handle_responses(&mut self, responses: Vec<Option<UpstreamResponse>>) {
459 for response in responses.into_iter().flatten() {
461 let UpstreamResponse {
462 channels_batch,
463 response,
464 } = response;
465
466 match response {
467 Ok(mut response) => {
468 self.backoff.reset();
475 self.last_failed_fetch = None;
476
477 metric!(
479 distribution(RelayDistributions::ProjectStateReceived) =
480 response.configs.len() as u64
481 );
482 for (key, mut channel) in channels_batch {
483 if response.pending.contains(&key) {
484 channel.pending += 1;
485 self.merge_channel(key, channel);
486 continue;
487 }
488
489 let mut result = "ok";
490 let state = if response.unchanged.contains(&key) {
491 result = "ok_unchanged";
492 SourceProjectState::NotModified
493 } else {
494 let state = response
495 .configs
496 .remove(&key)
497 .unwrap_or(ErrorBoundary::Ok(None));
498
499 let state = match state {
500 ErrorBoundary::Err(error) => {
501 result = "invalid";
502 let error = &error as &dyn std::error::Error;
503 relay_log::error!(error, "error fetching project state {key}");
504 ProjectState::Pending
505 }
506 ErrorBoundary::Ok(None) => ProjectState::Disabled,
507 ErrorBoundary::Ok(Some(state)) => state.into(),
508 };
509
510 SourceProjectState::New(state)
511 };
512
513 metric!(
514 distribution(RelayDistributions::ProjectStateAttempts) =
515 channel.attempts,
516 result = result,
517 );
518 metric!(
519 counter(RelayCounters::ProjectUpstreamCompleted) += 1,
520 result = result,
521 );
522
523 channel.send(state);
524 }
525 }
526 Err(err) => {
527 self.track_failed_response();
528
529 let attempts = channels_batch
530 .values()
531 .map(|b| b.attempts)
532 .max()
533 .unwrap_or(0);
534 if attempts >= 2 {
538 relay_log::error!(
539 error = &err as &dyn std::error::Error,
540 attempts = attempts,
541 "error fetching project states",
542 );
543 }
544
545 metric!(
546 distribution(RelayDistributions::ProjectStatePending) =
547 self.state_channels.len() as u64
548 );
549 self.state_channels.extend(channels_batch.into_iter().map(
551 |(key, mut channel)| {
552 channel.errors += 1;
553 (key, channel)
554 },
555 ))
556 }
557 }
558 }
559
560 if !self.state_channels.is_empty() {
561 self.schedule_fetch()
562 } else {
563 self.backoff.reset();
575 }
576 }
577
578 fn track_failed_response(&mut self) {
580 match self.last_failed_fetch {
581 None => self.last_failed_fetch = Some(Instant::now()),
582 Some(last_failed) => {
583 let failure_duration = last_failed.elapsed();
584 if failure_duration >= self.failure_interval {
585 relay_log::error!(
586 failure_duration = format!("{} seconds", failure_duration.as_secs()),
587 backoff_attempts = self.backoff.attempt(),
588 "can't fetch project states"
589 );
590 }
591 }
592 }
593 metric!(counter(RelayCounters::ProjectUpstreamFailed) += 1);
594 }
595
596 fn do_fetch(&mut self) {
598 self.fetch_handle.reset();
599
600 if self.state_channels.is_empty() {
601 relay_log::error!("project state schedule fetch request without projects");
602 return;
603 }
604
605 let config = self.config.current();
606 let channels = self.prepare_batches();
607 let upstream_relay = self.upstream_relay.clone();
608
609 let fetch_task = relay_system::spawn!(async move {
610 Self::fetch_states(config, upstream_relay, channels).await
611 });
612 self.fetch_tasks.push(fetch_task);
613 }
614
615 fn handle_message(&mut self, message: UpstreamProjectSource) {
617 let UpstreamProjectSource(
618 FetchProjectState {
619 project_key,
620 current_revision,
621 no_cache,
622 },
623 sender,
624 ) = message;
625
626 let query_timeout = self.config.current().query_timeout();
627
628 match self.state_channels.entry(project_key) {
631 Entry::Vacant(entry) => {
632 entry.insert(ProjectStateChannel::new(
633 sender,
634 current_revision,
635 query_timeout,
636 no_cache,
637 ));
638 }
639 Entry::Occupied(mut entry) => {
640 let channel = entry.get_mut();
641 channel.attach(sender, current_revision);
642 if no_cache {
645 channel.no_cache();
646 }
647 }
648 }
649
650 if !self.backoff.started() {
652 self.backoff.reset();
653 self.schedule_fetch();
654 }
655 }
656}
657
658impl Service for UpstreamProjectSourceService {
659 type Interface = UpstreamProjectSource;
660
661 async fn run(mut self, mut rx: relay_system::Receiver<Self::Interface>) {
662 relay_log::info!("project upstream cache started");
663 let mut expects_input = true;
664 while expects_input || !self.state_channels.is_empty() || !self.fetch_tasks.is_empty() {
665 tokio::select! {
666 biased;
667
668 () = &mut self.fetch_handle => self.do_fetch(),
669 Some(result) = self.fetch_tasks.next() => {
670 let responses = match result {
671 Ok(responses) => responses,
672 Err(error) => {
673 relay_log::error!(
674 error = &error as &dyn std::error::Error,
675 "fetch task panicked or was cancelled"
676 );
677 Vec::new()
678 }
679 };
680 self.handle_responses(responses);
681 }
682 signal = rx.recv(), if expects_input => match signal {
683 Some(message) => self.handle_message(message),
684 None => expects_input = false,
685 }
686 }
687 }
688 relay_log::info!("project upstream cache stopped");
689 }
690}
691
692#[cfg(test)]
693mod tests {
694 use crate::http::Response;
695 use futures::future::poll_immediate;
696
697 use super::*;
698
699 fn to_response(body: &impl serde::Serialize) -> Response {
700 let body = serde_json::to_vec(body).unwrap();
701 let response = http::response::Response::builder()
702 .status(http::StatusCode::OK)
703 .header(http::header::CONTENT_LENGTH, body.len())
704 .body(body)
705 .unwrap();
706
707 Response(response.into())
708 }
709
710 #[tokio::test(start_paused = true)]
711 async fn test_shutdown_with_closed_upstream() {
712 let (upstream, upstream_rx) = Addr::custom();
713 drop(upstream_rx);
714 let service = UpstreamProjectSourceService::new(Arc::new(Config::default()), upstream);
715 let (addr, rx) = relay_system::channel(UpstreamProjectSourceService::name());
716 let response = addr.send(FetchProjectState {
717 project_key: ProjectKey::parse("abd0f232775f45feab79864e580d160b").unwrap(),
718 current_revision: Revision::default(),
719 no_cache: false,
720 });
721 drop(addr);
722
723 tokio::time::timeout(Duration::from_secs(10), service.run(rx))
724 .await
725 .expect("project upstream cache should stop when the upstream service is gone");
726 assert!(response.await.is_err());
727 }
728
729 #[tokio::test(start_paused = true)]
730 async fn test_shutdown_drains_pending_requests() {
731 tokio::time::timeout(Duration::from_secs(10), async {
732 let (upstream, mut upstream_rx) = Addr::custom();
733 let config = Arc::new(Config::default());
734 let service = UpstreamProjectSourceService::new(Arc::clone(&config), upstream);
735 let (addr, rx) = relay_system::channel(UpstreamProjectSourceService::name());
736 let task = relay_system::spawn!(service.run(rx));
737 let project_key = ProjectKey::parse("abd0f232775f45feab79864e580d160b").unwrap();
738 let response = addr.send(FetchProjectState {
739 project_key,
740 current_revision: "123".into(),
741 no_cache: false,
742 });
743
744 let UpstreamRelay::SendRequest(mut request) = upstream_rx.recv().await.unwrap() else {
745 panic!()
746 };
747 drop(addr);
749 request.configure(&config.current());
750 request
751 .respond(Ok(to_response(&serde_json::json!({
752 "pending": [project_key],
753 }))))
754 .await;
755
756 let UpstreamRelay::SendRequest(mut request) = upstream_rx.recv().await.unwrap() else {
757 panic!()
758 };
759 request.configure(&config.current());
760 request
761 .respond(Ok(to_response(&serde_json::json!({
762 "unchanged": [project_key],
763 }))))
764 .await;
765
766 assert!(matches!(
767 response.await,
768 Ok(Ok(SourceProjectState::NotModified))
769 ));
770 task.await.unwrap();
771 })
772 .await
773 .expect("project upstream cache should drain pending requests and stop");
774 }
775
776 #[tokio::test]
777 async fn test_schedule_merge_channels() {
778 let (upstream_addr, mut upstream_rx) = Addr::custom();
779 let config = Arc::new(Config::from_json_value(serde_json::json!({})).unwrap());
780 let project_key = ProjectKey::parse("abd0f232775f45feab79864e580d160b").unwrap();
781
782 macro_rules! next_send_request {
783 () => {{
784 let UpstreamRelay::SendRequest(mut req) = upstream_rx.recv().await.unwrap() else {
785 panic!()
786 };
787 req.configure(&config.current());
788 req
789 }};
790 }
791
792 let service =
793 UpstreamProjectSourceService::new(Arc::clone(&config), upstream_addr).start_detached();
794
795 let mut response1 = service.send(FetchProjectState {
796 project_key,
797 current_revision: "123".into(),
798 no_cache: false,
799 });
800
801 let request1 = next_send_request!();
803
804 let mut response2 = service.send(FetchProjectState {
807 project_key,
808 current_revision: Revision::default(),
809 no_cache: false,
810 });
811
812 request1
815 .respond(Ok(to_response(&serde_json::json!({
816 "pending": [project_key],
817 }))))
818 .await;
819
820 assert!(poll_immediate(&mut response1).await.is_none());
822 assert!(poll_immediate(&mut response2).await.is_none());
823
824 next_send_request!()
826 .respond(Ok(to_response(&serde_json::json!({
827 "unchanged": [project_key],
828 }))))
829 .await;
830
831 let (response1, response2) = futures::future::join(response1, response2).await;
832 assert!(matches!(response1, Ok(Ok(SourceProjectState::NotModified))));
833 assert!(matches!(response2, Ok(Ok(SourceProjectState::NotModified))));
834
835 assert!(upstream_rx.try_recv().is_err());
837 }
838}