fully split service/instance metric classes & usages

Author: syphar

src/bin/cratesfyi.rs

@@ -16,7 +16,7 @@ use docs_rs::utils::{
 };
 use docs_rs::{
     start_background_metrics_webserver, start_web_server, BuildQueue, Config, Context, Index,
-    Metrics, PackageKind, RustwideBuilder, Storage,
+    InstanceMetrics, PackageKind, RustwideBuilder, ServiceMetrics, Storage,
 };
 use humantime::Duration;
 use once_cell::sync::OnceCell;
@@ -712,7 +712,8 @@ struct BinContext {
     cdn: OnceCell<Arc<CdnBackend>>,
     config: OnceCell<Arc<Config>>,
     pool: OnceCell<Pool>,
-    metrics: OnceCell<Arc<Metrics>>,
+    service_metrics: OnceCell<Arc<ServiceMetrics>>,
+    instance_metrics: OnceCell<Arc<InstanceMetrics>>,
     index: OnceCell<Arc<Index>>,
     repository_stats_updater: OnceCell<Arc<RepositoryStatsUpdater>>,
     runtime: OnceCell<Arc<Runtime>>,
@@ -726,7 +727,8 @@ impl BinContext {
             cdn: OnceCell::new(),
             config: OnceCell::new(),
             pool: OnceCell::new(),
-            metrics: OnceCell::new(),
+            service_metrics: OnceCell::new(),
+            instance_metrics: OnceCell::new(),
             index: OnceCell::new(),
             repository_stats_updater: OnceCell::new(),
             runtime: OnceCell::new(),
@@ -753,13 +755,13 @@ impl Context for BinContext {
     lazy! {
         fn build_queue(self) -> BuildQueue = BuildQueue::new(
             self.pool()?,
-            self.metrics()?,
+            self.instance_metrics()?,
             self.config()?,
             self.storage()?,
         );
         fn storage(self) -> Storage = Storage::new(
             self.pool()?,
-            self.metrics()?,
+            self.instance_metrics()?,
             self.config()?,
             self.runtime()?,
         )?;
@@ -768,7 +770,8 @@ impl Context for BinContext {
             &self.runtime()?,
         );
         fn config(self) -> Config = Config::from_env()?;
-        fn metrics(self) -> Metrics = Metrics::new()?;
+        fn service_metrics(self) -> ServiceMetrics = ServiceMetrics::new()?;
+        fn instance_metrics(self) -> InstanceMetrics = InstanceMetrics::new()?;
         fn runtime(self) -> Runtime = {
             Builder::new_multi_thread()
                 .enable_all()
@@ -793,7 +796,9 @@ impl Context for BinContext {
     fn pool(&self) -> Result<Pool> {
         Ok(self
             .pool
-            .get_or_try_init::<_, Error>(|| Ok(Pool::new(&*self.config()?, self.metrics()?)?))?
+            .get_or_try_init::<_, Error>(|| {
+                Ok(Pool::new(&*self.config()?, self.instance_metrics()?)?)
+            })?
             .clone())
     }
 }

src/build_queue.rs

@@ -4,7 +4,7 @@ use crate::docbuilder::PackageKind;
 use crate::error::Result;
 use crate::storage::Storage;
 use crate::utils::{get_config, get_crate_priority, report_error, retry, set_config, ConfigName};
-use crate::{Config, Index, Metrics, RustwideBuilder};
+use crate::{Config, Index, InstanceMetrics, RustwideBuilder};
 use anyhow::Context;
 use fn_error_context::context;
 
@@ -28,14 +28,14 @@ pub struct BuildQueue {
     config: Arc<Config>,
     storage: Arc<Storage>,
     pub(crate) db: Pool,
-    metrics: Arc<Metrics>,
+    metrics: Arc<InstanceMetrics>,
     max_attempts: i32,
 }
 
 impl BuildQueue {
     pub fn new(
         db: Pool,
-        metrics: Arc<Metrics>,
+        metrics: Arc<InstanceMetrics>,
         config: Arc<Config>,
         storage: Arc<Storage>,
     ) -> Self {
@@ -204,7 +204,7 @@ impl BuildQueue {
                 to_process.name, to_process.version
             )
         });
-        self.metrics.instance.total_builds.inc();
+        self.metrics.total_builds.inc();
         if let Err(err) =
             cdn::queue_crate_invalidation(&mut transaction, &self.config, &to_process.name)
         {
@@ -225,7 +225,7 @@ impl BuildQueue {
                     .get(0);
 
                 if attempt >= self.max_attempts {
-                    self.metrics.instance.failed_builds.inc();
+                    self.metrics.failed_builds.inc();
                 }
 
                 report_error(&e);
@@ -320,7 +320,7 @@ impl BuildQueue {
                             "{}-{} added into build queue",
                             release.name, release.version
                         );
-                        self.metrics.instance.queued_builds.inc();
+                        self.metrics.queued_builds.inc();
                         crates_added += 1;
                     }
                     Err(err) => report_error(&err),
@@ -600,9 +600,9 @@ mod tests {
             assert!(!called, "there were still items in the queue");
 
             // Ensure metrics were recorded correctly
-            let metrics = env.metrics();
-            assert_eq!(metrics.instance.total_builds.get(), 9);
-            assert_eq!(metrics.instance.failed_builds.get(), 1);
+            let metrics = env.instance_metrics();
+            assert_eq!(metrics.total_builds.get(), 9);
+            assert_eq!(metrics.failed_builds.get(), 1);
 
             // no invalidations were run since we don't have a distribution id configured
             assert!(cdn::queued_or_active_crate_invalidations(&mut *env.db().conn())?.is_empty());

src/cdn.rs

@@ -1,4 +1,4 @@
-use crate::{metrics::duration_to_seconds, utils::report_error, Config, Metrics};
+use crate::{metrics::duration_to_seconds, utils::report_error, Config, InstanceMetrics};
 use anyhow::{anyhow, bail, Context, Error, Result};
 use aws_sdk_cloudfront::{
     config::{retry::RetryConfig, Region},
@@ -303,7 +303,7 @@ impl CdnBackend {
 #[instrument(skip(conn))]
 pub(crate) fn handle_queued_invalidation_requests(
     cdn: &CdnBackend,
-    metrics: &Metrics,
+    metrics: &InstanceMetrics,
     conn: &mut impl postgres::GenericClient,
     distribution_id: &str,
 ) -> Result<()> {
@@ -363,7 +363,6 @@ pub(crate) fn handle_queued_invalidation_requests(
         if let Ok(duration) = (now - row.get::<_, DateTime<Utc>>(0)).to_std() {
             // This can only fail when the duration is negative, which can't happen anyways
             metrics
-                .instance
                 .cdn_invalidation_time
                 .with_label_values(&[distribution_id])
                 .observe(duration_to_seconds(duration));
@@ -401,7 +400,6 @@ pub(crate) fn handle_queued_invalidation_requests(
         if let Ok(duration) = (now - row.get::<_, DateTime<Utc>>("queued")).to_std() {
             // This can only fail when the duration is negative, which can't happen anyways
             metrics
-                .instance
                 .cdn_queue_time
                 .with_label_values(&[distribution_id])
                 .observe(duration_to_seconds(duration));
@@ -709,13 +707,13 @@ mod tests {
             // now handle the queued invalidations
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_static",
             )?;
@@ -743,13 +741,13 @@ mod tests {
             // now handle again
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_static",
             )?;
@@ -808,7 +806,7 @@ mod tests {
             // handle the queued invalidations
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;
@@ -862,7 +860,7 @@ mod tests {
             // handle the queued invalidations
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;
@@ -888,7 +886,7 @@ mod tests {
             // now handle again
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;
@@ -923,7 +921,7 @@ mod tests {
             // run the handler
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;

src/context.rs

@@ -2,7 +2,7 @@ use crate::cdn::CdnBackend;
 use crate::db::Pool;
 use crate::error::Result;
 use crate::repositories::RepositoryStatsUpdater;
-use crate::{BuildQueue, Config, Index, Metrics, Storage};
+use crate::{BuildQueue, Config, Index, InstanceMetrics, ServiceMetrics, Storage};
 use std::sync::Arc;
 use tokio::runtime::Runtime;
 
@@ -12,7 +12,8 @@ pub trait Context {
     fn storage(&self) -> Result<Arc<Storage>>;
     fn cdn(&self) -> Result<Arc<CdnBackend>>;
     fn pool(&self) -> Result<Pool>;
-    fn metrics(&self) -> Result<Arc<Metrics>>;
+    fn service_metrics(&self) -> Result<Arc<ServiceMetrics>>;
+    fn instance_metrics(&self) -> Result<Arc<InstanceMetrics>>;
     fn index(&self) -> Result<Arc<Index>>;
     fn repository_stats_updater(&self) -> Result<Arc<RepositoryStatsUpdater>>;
     fn runtime(&self) -> Result<Arc<Runtime>>;

src/db/pool.rs

@@ -1,4 +1,4 @@
-use crate::metrics::Metrics;
+use crate::metrics::InstanceMetrics;
 use crate::Config;
 use postgres::{Client, NoTls};
 use r2d2_postgres::PostgresConnectionManager;
@@ -15,12 +15,12 @@ pub struct Pool {
     pool: Arc<std::sync::Mutex<Option<r2d2::Pool<PostgresConnectionManager<NoTls>>>>>,
     #[cfg(not(test))]
     pool: r2d2::Pool<PostgresConnectionManager<NoTls>>,
-    metrics: Arc<Metrics>,
+    metrics: Arc<InstanceMetrics>,
     max_size: u32,
 }
 
 impl Pool {
-    pub fn new(config: &Config, metrics: Arc<Metrics>) -> Result<Pool, PoolError> {
+    pub fn new(config: &Config, metrics: Arc<InstanceMetrics>) -> Result<Pool, PoolError> {
         debug!(
             "creating database pool (if this hangs, consider running `docker-compose up -d db s3`)"
         );
@@ -30,13 +30,17 @@ impl Pool {
     #[cfg(test)]
     pub(crate) fn new_with_schema(
         config: &Config,
-        metrics: Arc<Metrics>,
+        metrics: Arc<InstanceMetrics>,
         schema: &str,
     ) -> Result<Pool, PoolError> {
         Self::new_inner(config, metrics, schema)
     }
 
-    fn new_inner(config: &Config, metrics: Arc<Metrics>, schema: &str) -> Result<Pool, PoolError> {
+    fn new_inner(
+        config: &Config,
+        metrics: Arc<InstanceMetrics>,
+        schema: &str,
+    ) -> Result<Pool, PoolError> {
         let url = config
             .database_url
             .parse()
@@ -77,7 +81,7 @@ impl Pool {
         match self.with_pool(|p| p.get()) {
             Ok(conn) => Ok(conn),
             Err(err) => {
-                self.metrics.instance.failed_db_connections.inc();
+                self.metrics.failed_db_connections.inc();
                 Err(PoolError::ClientError(err))
             }
         }

src/docbuilder/rustwide_builder.rs

@@ -13,7 +13,7 @@ use crate::utils::{
 };
 use crate::RUSTDOC_STATIC_STORAGE_PREFIX;
 use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage};
-use crate::{Config, Context, Index, Metrics, Storage};
+use crate::{Config, Context, Index, InstanceMetrics, Storage};
 use anyhow::{anyhow, bail, Context as _, Error};
 use docsrs_metadata::{BuildTargets, Metadata, DEFAULT_TARGETS, HOST_TARGET};
 use failure::Error as FailureError;
@@ -45,7 +45,7 @@ pub struct RustwideBuilder {
     config: Arc<Config>,
     db: Pool,
     storage: Arc<Storage>,
-    metrics: Arc<Metrics>,
+    metrics: Arc<InstanceMetrics>,
     index: Arc<Index>,
     rustc_version: String,
     repository_stats_updater: Arc<RepositoryStatsUpdater>,
@@ -92,7 +92,7 @@ impl RustwideBuilder {
             config,
             db: context.pool()?,
             storage: context.storage()?,
-            metrics: context.metrics()?,
+            metrics: context.instance_metrics()?,
             index: context.index()?,
             rustc_version: String::new(),
             repository_stats_updater: context.repository_stats_updater()?,
@@ -484,11 +484,11 @@ impl RustwideBuilder {
 
                     let has_examples = build.host_source_dir().join("examples").is_dir();
                     if res.result.successful {
-                        self.metrics.instance.successful_builds.inc();
+                        self.metrics.successful_builds.inc();
                     } else if res.cargo_metadata.root().is_library() {
-                        self.metrics.instance.failed_builds.inc();
+                        self.metrics.failed_builds.inc();
                     } else {
-                        self.metrics.instance.non_library_builds.inc();
+                        self.metrics.non_library_builds.inc();
                     }
 
                     let release_data = if !is_local {

src/lib.rs

@@ -8,7 +8,7 @@ pub use self::context::Context;
 pub use self::docbuilder::PackageKind;
 pub use self::docbuilder::RustwideBuilder;
 pub use self::index::Index;
-pub use self::metrics::Metrics;
+pub use self::metrics::{InstanceMetrics, ServiceMetrics};
 pub use self::storage::Storage;
 pub use self::web::{start_background_metrics_webserver, start_web_server};
 

src/metrics/mod.rs

@@ -35,21 +35,6 @@ pub const CDN_INVALIDATION_HISTOGRAM_BUCKETS: &[f64; 11] = &[
     24000.0, // 240
 ];
 
-#[derive(Debug)]
-pub struct Metrics {
-    pub instance: InstanceMetrics,
-    pub service: ServiceMetrics,
-}
-
-impl Metrics {
-    pub fn new() -> Result<Self, prometheus::Error> {
-        Ok(Self {
-            instance: InstanceMetrics::new()?,
-            service: ServiceMetrics::new()?,
-        })
-    }
-}
-
 metrics! {
     pub struct InstanceMetrics {
         /// The number of idle database connections