1use std::convert::Infallible;
2use std::sync::Arc;
3use std::time::Duration;
4
5use crate::metrics::MetricOutcomes;
6use crate::services::autoscaling::{AutoscalingMetricService, AutoscalingMetrics};
7use crate::services::buffer::{
8 ObservableEnvelopeBuffer, PartitionedEnvelopeBuffer, ProjectKeyPair,
9};
10use crate::services::cogs::{CogsService, CogsServiceRecorder};
11use crate::services::config_reload::ConfigReloadService;
12use crate::services::global_config::{
13 GlobalConfigHandle, GlobalConfigManager, GlobalConfigService,
14};
15use crate::services::health_check::{HealthCheck, HealthCheckService};
16use crate::services::metrics::RouterService;
17#[cfg(feature = "processing")]
18use crate::services::objectstore::{self, Objectstore};
19use crate::services::outcome::{
20 ClientReportOutcomeProducerService, NullOutcomeProducerService, OutcomeProducerService,
21 TrackOutcome,
22};
23use crate::services::processor::{self, EnvelopeProcessor};
24use crate::services::projects::cache::{ProjectCacheHandle, ProjectCacheService};
25use crate::services::projects::source::ProjectSource;
26use crate::services::proxy_processor::ProxyProcessorService;
27use crate::services::relays::{RelayCache, RelayCacheService};
28use crate::services::stats::RelayStats;
29#[cfg(feature = "processing")]
30use crate::services::store::{self, create_kafka_client};
31use crate::services::upload::{self, Upload};
32use crate::services::upstream::{UpstreamRelay, UpstreamRelayService};
33use crate::utils::{MemoryChecker, MemoryStat, ThreadKind};
34#[cfg(feature = "processing")]
35use anyhow::Context;
36use anyhow::Result;
37use axum::extract::FromRequestParts;
38use axum::http::request::Parts;
39use relay_cogs::Cogs;
40use relay_config::{Config, ConfigSnapshot, EmitOutcomes, RelayMode};
41#[cfg(feature = "processing")]
42use relay_config::{RedisConfigRef, RedisConfigsRef};
43#[cfg(feature = "processing")]
44use relay_redis::AsyncRedisClient;
45#[cfg(feature = "processing")]
46use relay_redis::redis::Script;
47#[cfg(feature = "processing")]
48use relay_redis::{RedisClients, RedisError, RedisScripts};
49use relay_system::{Addr, AssertEmpty, Service, ServiceSpawn, ServiceSpawnExt as _, channel};
50use relay_threading::ServicePool;
51
52#[derive(Debug, thiserror::Error)]
54pub enum ServiceError {
55 #[error("could not load the Geoip Db")]
57 GeoIp,
58
59 #[cfg(feature = "processing")]
61 #[error("could not initialize kafka producer: {0}")]
62 Kafka(String),
63
64 #[cfg(feature = "processing")]
66 #[error("could not initialize redis client during startup")]
67 Redis,
68}
69
70#[derive(Clone, Debug)]
71pub struct Registry {
72 pub cogs: Cogs,
73 pub health_check: Addr<HealthCheck>,
74 pub outcome_aggregator: Addr<TrackOutcome>,
75 pub processor: Addr<EnvelopeProcessor>,
76 pub relay_cache: Addr<RelayCache>,
77 pub global_config: Addr<GlobalConfigManager>,
78 pub upstream_relay: Addr<UpstreamRelay>,
79 pub envelope_buffer: Arc<PartitionedEnvelopeBuffer>,
80 pub project_cache_handle: ProjectCacheHandle,
81 pub autoscaling: Option<Addr<AutoscalingMetrics>>,
82 #[cfg(feature = "processing")]
83 pub objectstore: Option<Addr<Objectstore>>,
84 pub upload: Addr<Upload>,
85 pub global_config_handle: GlobalConfigHandle,
86}
87
88pub fn create_runtime(name: &'static str, threads: usize) -> relay_system::Runtime {
90 relay_system::Runtime::builder(name)
91 .worker_threads(threads)
92 .max_blocking_threads(150)
103 .thread_keep_alive(Duration::from_secs(1))
107 .build()
108}
109
110fn create_processor_pool(config: &ConfigSnapshot) -> Result<ServicePool> {
111 let thread_count = match config.cpu_concurrency() {
114 conc @ 0..=2 => conc.max(1),
115 conc @ 3..=4 => conc - 1,
116 conc => conc - 2,
117 };
118 relay_log::info!("starting {thread_count} envelope processing workers");
119
120 let pool = crate::utils::ThreadPoolBuilder::new("processor", tokio::runtime::Handle::current())
121 .num_threads(thread_count)
122 .max_concurrency(config.pool_concurrency())
123 .thread_kind(ThreadKind::Worker)
124 .build()?;
125
126 Ok(pool)
127}
128
129#[cfg(feature = "processing")]
130fn create_store_pool(config: &ConfigSnapshot) -> Result<ServicePool> {
131 let thread_count = config.cpu_concurrency().div_ceil(8);
137 relay_log::info!("starting {thread_count} store workers");
138
139 let pool = crate::utils::ThreadPoolBuilder::new("store", tokio::runtime::Handle::current())
140 .num_threads(thread_count)
141 .max_concurrency(config.pool_concurrency())
142 .build()?;
143
144 Ok(pool)
145}
146
147#[derive(Debug)]
148struct StateInner {
149 config: Arc<Config>,
150 memory_checker: MemoryChecker,
151 registry: Registry,
152 _daemons: Vec<Addr<()>>, _asserter: AssertEmpty<TrackOutcome>,
154}
155
156#[derive(Clone, Debug)]
158pub struct ServiceState {
159 inner: Arc<StateInner>,
160}
161
162impl ServiceState {
163 pub async fn start(
165 handle: &relay_system::Handle,
166 services: &dyn ServiceSpawn,
167 config: Arc<Config>,
168 ) -> Result<Self> {
169 let upstream_relay = services.start(UpstreamRelayService::new(config.clone()));
170 let current_config = config.current();
171
172 #[cfg(feature = "processing")]
173 let redis_clients = current_config
174 .redis()
175 .filter(|_| current_config.processing_enabled())
176 .map(create_redis_clients)
177 .transpose()
178 .context(ServiceError::Redis)?;
179
180 #[cfg(feature = "processing")]
185 if let Some(redis_clients) = &redis_clients {
186 initialize_redis_scripts_for_client(redis_clients)
187 .await
188 .context(ServiceError::Redis)?;
189 }
190
191 let memory_stat = MemoryStat::new(current_config.memory_stat_refresh_frequency_ms());
194
195 let (envelope_processor, envelope_processor_rx) = match current_config.relay_mode() {
198 RelayMode::Proxy => channel(ProxyProcessorService::name()),
199 RelayMode::Managed => channel("envelope_processor"),
200 };
201
202 let (aggregator, aggregator_rx) = channel(RouterService::name());
203
204 let (outcome_metrics_router, outcome_metrics_router_rx) = channel("outcome_metrics_router");
206 let (dummy_outcomes, _asserter) = Addr::never(); let outcome_aggregator = match current_config.emit_outcomes() {
209 EmitOutcomes::None => services.start(NullOutcomeProducerService::new()),
210 _ => match current_config.relay_mode() {
211 RelayMode::Proxy => services.start(ClientReportOutcomeProducerService::new(
212 ¤t_config,
213 envelope_processor.clone(),
214 )),
215 RelayMode::Managed => services.start(OutcomeProducerService::new(
216 Arc::clone(&config),
217 outcome_metrics_router,
218 )),
219 },
220 };
221
222 let (global_config, global_config_rx) =
223 GlobalConfigService::new(config.clone(), upstream_relay.clone());
224 let global_config_handle = global_config.handle();
225 let global_config = services.start(global_config);
229
230 let project_source = ProjectSource::start_in(
231 services,
232 Arc::clone(&config),
233 upstream_relay.clone(),
234 #[cfg(feature = "processing")]
235 redis_clients.clone(),
236 )
237 .await;
238 let project_cache_handle =
239 ProjectCacheService::new(Arc::clone(&config), project_source).start_in(services);
240
241 let cogs = CogsService::new(¤t_config);
242 let cogs = Cogs::new(CogsServiceRecorder::new(
243 ¤t_config,
244 services.start(cogs),
245 ));
246
247 let metric_outcomes = MetricOutcomes::new(outcome_aggregator.clone());
248
249 #[cfg(feature = "processing")]
250 let store_pool = create_store_pool(¤t_config)?;
251
252 #[cfg(feature = "processing")]
253 let mut maybe_kafka_client = None;
254
255 #[cfg(feature = "processing")]
256 let store = current_config
257 .processing_enabled()
258 .then(|| {
259 let kafka_client = create_kafka_client(¤t_config)?;
260 maybe_kafka_client = Some(kafka_client.clone());
261 let service = store::create_service(
262 store_pool.clone(),
263 kafka_client.clone(),
264 config.clone(),
265 global_config_handle.clone(),
266 metric_outcomes.clone(),
267 );
268 Ok::<_, anyhow::Error>(services.start(service))
269 })
270 .transpose()?;
271
272 #[cfg(feature = "processing")]
273 let objectstore = objectstore::create_service(current_config.objectstore(), store.clone())?
274 .map(|s| services.start(s));
275
276 let envelope_buffer = PartitionedEnvelopeBuffer::create(
277 current_config.spool_partitions(),
278 config.clone(),
279 memory_stat.clone(),
280 global_config_rx.clone(),
281 project_cache_handle.clone(),
282 envelope_processor.clone(),
283 outcome_aggregator.clone(),
284 services,
285 );
286
287 let (processor_pool, aggregator_handle, autoscaling) = match current_config.relay_mode() {
288 RelayMode::Proxy => {
289 services.start_with(
290 ProxyProcessorService::new(
291 config.clone(),
292 project_cache_handle.clone(),
293 upstream_relay.clone(),
294 ),
295 envelope_processor_rx,
296 );
297 (None, None, None)
298 }
299 RelayMode::Managed => {
300 let processor_pool = create_processor_pool(¤t_config)?;
301 #[cfg(feature = "processing")]
303 let rate_limiter =
304 processor::create_rate_limiter(¤t_config, redis_clients.clone());
305
306 let bucket_processor = services.start(processor::bucket::create_service(
307 processor_pool.clone(),
308 config.clone(),
309 global_config_handle.clone(),
310 project_cache_handle.clone(),
311 cogs.clone(),
312 #[cfg(feature = "processing")]
313 rate_limiter.clone(),
314 processor::bucket::Addrs {
315 outcome_aggregator: outcome_aggregator.clone(),
316 upstream_relay: upstream_relay.clone(),
317 #[cfg(feature = "processing")]
318 store_forwarder: store.clone(),
319 },
320 ));
321
322 let router = RouterService::new(
323 handle.clone(),
324 current_config.default_aggregator_config().clone(),
325 current_config.secondary_aggregator_configs().clone(),
326 Some(bucket_processor.recipient()),
327 project_cache_handle.clone(),
328 );
329 let router_handle = router.handle();
330 services.start_with(router, aggregator_rx);
331
332 {
357 #[cfg(feature = "processing")]
358 let outcome_store = {
359 if current_config.processing_enabled() {
360 let (addr, rx) = channel("outcome_store");
361 let service = store::create_service(
362 store_pool.clone(),
363 maybe_kafka_client
364 .clone()
365 .expect("kafka client created by store"),
366 config.clone(),
367 global_config_handle.clone(),
368 MetricOutcomes::new(dummy_outcomes.clone()),
369 );
370 services.start_with(service, rx);
371 Some(addr)
372 } else {
373 None
374 }
375 };
376
377 let (outcome_bucket_processor, outcome_bucket_rx) =
378 channel("outcome_bucket_processor");
379 let outcome_bucket_service = processor::bucket::create_service(
380 processor_pool.clone(),
381 config.clone(),
382 global_config_handle.clone(),
383 project_cache_handle.clone(),
384 cogs.clone(),
385 #[cfg(feature = "processing")]
386 rate_limiter.clone(),
387 processor::bucket::Addrs {
388 outcome_aggregator: dummy_outcomes,
389 upstream_relay: upstream_relay.clone(),
390 #[cfg(feature = "processing")]
391 store_forwarder: outcome_store,
392 },
393 );
394 services.start_with(outcome_bucket_service, outcome_bucket_rx);
395
396 let outcome_metrics_router = RouterService::prefixed(
399 handle.clone(),
400 "outcomes_",
401 current_config.default_aggregator_config().clone(),
402 current_config.secondary_aggregator_configs().clone(),
403 Some(outcome_bucket_processor.recipient()),
404 project_cache_handle.clone(),
405 );
406 services.start_with(outcome_metrics_router, outcome_metrics_router_rx);
407 }
408
409 services.start_with(
410 processor::envelope::create_service(
411 processor_pool.clone(),
412 config.clone(),
413 global_config_handle.clone(),
414 project_cache_handle.clone(),
415 cogs.clone(),
416 #[cfg(feature = "processing")]
417 rate_limiter,
418 processor::envelope::Addrs {
419 outcome_aggregator: outcome_aggregator.clone(),
420 upstream_relay: upstream_relay.clone(),
421 #[cfg(feature = "processing")]
422 objectstore: objectstore.clone(),
423 #[cfg(feature = "processing")]
424 store_forwarder: store,
425 aggregator: aggregator.clone(),
426 },
427 metric_outcomes.clone(),
428 ),
429 envelope_processor_rx,
430 );
431
432 let autoscaling = services.start(AutoscalingMetricService::new(
433 memory_stat.clone(),
434 envelope_buffer.clone(),
435 handle.clone(),
436 processor_pool.clone(),
437 ));
438
439 (Some(processor_pool), Some(router_handle), Some(autoscaling))
440 }
441 };
442
443 let health_check = services.start(HealthCheckService::new(
444 config.clone(),
445 MemoryChecker::new(memory_stat.clone(), config.clone()),
446 aggregator_handle,
447 upstream_relay.clone(),
448 envelope_buffer.clone(),
449 ));
450
451 let stats = services.start(RelayStats::new(
452 config.clone(),
453 handle.clone(),
454 upstream_relay.clone(),
455 #[cfg(feature = "processing")]
456 redis_clients.clone(),
457 processor_pool,
458 #[cfg(feature = "processing")]
459 store_pool,
460 ));
461
462 let relay_cache = services.start(RelayCacheService::new(
463 config.clone(),
464 upstream_relay.clone(),
465 ));
466
467 let upload = services.start(upload::create_service(
468 &config,
469 &upstream_relay,
470 #[cfg(feature = "processing")]
471 &objectstore,
472 ));
473
474 let config_reload = services.start(ConfigReloadService::new(
475 config.clone(),
476 #[cfg(feature = "processing")]
477 maybe_kafka_client,
478 ));
479
480 let registry = Registry {
481 cogs,
482 processor: envelope_processor,
483 health_check,
484 outcome_aggregator,
485 relay_cache,
486 global_config,
487 project_cache_handle,
488 upstream_relay,
489 envelope_buffer,
490 autoscaling,
491 #[cfg(feature = "processing")]
492 objectstore,
493 upload,
494 global_config_handle,
495 };
496
497 let state = StateInner {
498 config: config.clone(),
499 memory_checker: MemoryChecker::new(memory_stat, config.clone()),
500 registry,
501 _daemons: vec![stats, config_reload],
502 _asserter,
503 };
504
505 Ok(ServiceState {
506 inner: Arc::new(state),
507 })
508 }
509
510 pub fn config(&self) -> ConfigSnapshot {
512 self.inner.config.current()
513 }
514
515 pub fn cogs(&self) -> &Cogs {
517 &self.inner.registry.cogs
518 }
519
520 pub fn memory_checker(&self) -> &MemoryChecker {
524 &self.inner.memory_checker
525 }
526
527 pub fn autoscaling(&self) -> Option<&Addr<AutoscalingMetrics>> {
528 self.inner.registry.autoscaling.as_ref()
529 }
530
531 pub fn envelope_buffer_is_empty(&self) -> bool {
533 self.inner.registry.envelope_buffer.is_empty()
534 }
535
536 pub fn envelope_buffer(&self, project_key_pair: ProjectKeyPair) -> &ObservableEnvelopeBuffer {
538 self.inner.registry.envelope_buffer.buffer(project_key_pair)
539 }
540
541 pub fn project_cache_handle(&self) -> &ProjectCacheHandle {
543 &self.inner.registry.project_cache_handle
544 }
545
546 pub fn relay_cache(&self) -> &Addr<RelayCache> {
548 &self.inner.registry.relay_cache
549 }
550
551 pub fn health_check(&self) -> &Addr<HealthCheck> {
553 &self.inner.registry.health_check
554 }
555
556 pub fn upstream_relay(&self) -> &Addr<UpstreamRelay> {
558 &self.inner.registry.upstream_relay
559 }
560
561 pub fn processor(&self) -> &Addr<EnvelopeProcessor> {
563 &self.inner.registry.processor
564 }
565
566 pub fn global_config(&self) -> &Addr<GlobalConfigManager> {
568 &self.inner.registry.global_config
569 }
570
571 pub fn outcome_aggregator(&self) -> &Addr<TrackOutcome> {
573 &self.inner.registry.outcome_aggregator
574 }
575
576 #[cfg(feature = "processing")]
577 pub fn objectstore(&self) -> Option<&Addr<Objectstore>> {
579 self.inner.registry.objectstore.as_ref()
580 }
581
582 pub fn upload(&self) -> &Addr<Upload> {
584 &self.inner.registry.upload
585 }
586
587 pub fn global_config_handle(&self) -> &GlobalConfigHandle {
588 &self.inner.registry.global_config_handle
589 }
590}
591
592#[cfg(feature = "processing")]
600pub fn create_redis_clients(configs: RedisConfigsRef<'_>) -> Result<RedisClients, RedisError> {
601 const PROJECT_CONFIG_REDIS_CLIENT: (&str, &str) = ("projectconfig", "projectconfig_secondary");
602 const QUOTA_REDIS_CLIENT: (&str, &str) = ("quotas", "quotas_secondary");
603 const UNIFIED_REDIS_CLIENT: (&str, &str) = ("unified", "unified_secondary");
604
605 match configs {
606 RedisConfigsRef::Unified(unified) => {
607 let client = create_async_redis_client(UNIFIED_REDIS_CLIENT, &unified)?;
608
609 Ok(RedisClients {
610 project_configs: client.clone(),
611 quotas: client,
612 })
613 }
614 RedisConfigsRef::Individual {
615 project_configs,
616 quotas,
617 } => {
618 let project_configs =
619 create_async_redis_client(PROJECT_CONFIG_REDIS_CLIENT, &project_configs)?;
620 let quotas = create_async_redis_client(QUOTA_REDIS_CLIENT, "as)?;
621
622 Ok(RedisClients {
623 project_configs,
624 quotas,
625 })
626 }
627 }
628}
629
630#[cfg(feature = "processing")]
631fn create_async_redis_client(
632 names: (&'static str, &'static str),
633 config: &RedisConfigRef<'_>,
634) -> Result<AsyncRedisClient, RedisError> {
635 let (name, secondary_name) = names;
636
637 match config {
638 RedisConfigRef::Cluster {
639 cluster_nodes,
640 options,
641 } => AsyncRedisClient::cluster(name, cluster_nodes.iter().map(|s| s.as_str()), options),
642 RedisConfigRef::MultiWrite { configs } => {
643 let mut configs = configs.iter();
644 let primary = configs.next().ok_or(RedisError::Configuration)?;
645 let primary = create_async_redis_client(names, primary)?;
646 let secondaries = configs
648 .map(|config| create_async_redis_client((secondary_name, secondary_name), config))
649 .collect::<Result<_, _>>()?;
650
651 Ok(AsyncRedisClient::multi_write(primary, secondaries))
652 }
653 RedisConfigRef::Single { server, options } => {
654 AsyncRedisClient::single(name, server, options)
655 }
656 }
657}
658
659#[cfg(feature = "processing")]
660async fn initialize_redis_scripts_for_client(
661 redis_clients: &RedisClients,
662) -> Result<(), RedisError> {
663 let scripts = RedisScripts::all();
664
665 let RedisClients {
666 project_configs,
667 quotas,
668 } = redis_clients;
669
670 initialize_redis_scripts(project_configs, &scripts).await?;
671 initialize_redis_scripts(quotas, &scripts).await?;
672
673 Ok(())
674}
675
676#[cfg(feature = "processing")]
677async fn initialize_redis_scripts(
678 client: &AsyncRedisClient,
679 scripts: &[&Script],
680) -> Result<(), RedisError> {
681 let mut connection = client.get_connection().await?;
682
683 for script in scripts {
684 script
687 .prepare_invoke()
688 .load_async(&mut connection)
689 .await
690 .map_err(RedisError::Redis)?;
691 }
692
693 Ok(())
694}
695
696impl FromRequestParts<Self> for ServiceState {
697 type Rejection = Infallible;
698
699 async fn from_request_parts(_: &mut Parts, state: &Self) -> Result<Self, Self::Rejection> {
700 Ok(state.clone())
701 }
702}