Skip to content

Commit 0bdc495

Browse files
committed
Merge branch 'master' into docker-compose-services
2 parents ce8ef5b + 288f66d commit 0bdc495

File tree

7 files changed

+178
-77
lines changed

7 files changed

+178
-77
lines changed

src/db/add_package.rs

Lines changed: 12 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
use crate::{
2-
db::types::{BuildStatus, Feature, version::Version},
2+
db::types::{
3+
BuildId, BuildStatus, CrateId, Feature, ReleaseId, dependencies::ReleaseDependencyList,
4+
version::Version,
5+
},
36
docbuilder::DocCoverage,
47
error::Result,
58
registry_api::{CrateData, CrateOwner, ReleaseData},
69
storage::CompressionAlgorithm,
7-
utils::{Dependency, MetadataPackage, rustc_version::parse_rustc_date},
10+
utils::{MetadataPackage, rustc_version::parse_rustc_date},
811
web::crate_details::{latest_release, releases_for_crate},
912
};
1013
use anyhow::{Context, anyhow};
11-
use derive_more::{Deref, Display};
1214
use futures_util::stream::TryStreamExt;
13-
use semver::VersionReq;
14-
use serde::{Deserialize, Serialize};
1515
use serde_json::Value;
1616
use slug::slugify;
1717
use std::{
@@ -22,56 +22,6 @@ use std::{
2222
};
2323
use tracing::{debug, error, info, instrument};
2424

25-
#[derive(Debug, Clone, Copy, Display, PartialEq, Eq, Hash, Serialize, sqlx::Type)]
26-
#[sqlx(transparent)]
27-
pub struct CrateId(pub i32);
28-
29-
#[derive(Debug, Clone, Copy, Display, PartialEq, Eq, Hash, Serialize, sqlx::Type)]
30-
#[sqlx(transparent)]
31-
pub struct ReleaseId(pub i32);
32-
33-
#[derive(Debug, Clone, Copy, Display, PartialEq, Eq, Hash, Serialize, sqlx::Type)]
34-
#[sqlx(transparent)]
35-
pub struct BuildId(pub i32);
36-
37-
type DepOut = (String, String, String, bool);
38-
type DepIn = (String, VersionReq, Option<String>, Option<bool>);
39-
40-
/// A crate dependency in our internal representation for releases.dependencies json.
41-
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Deref)]
42-
#[serde(from = "DepIn", into = "DepOut")]
43-
pub(crate) struct ReleaseDependency(Dependency);
44-
45-
impl ReleaseDependency {
46-
pub(crate) fn into_inner(self) -> Dependency {
47-
self.0
48-
}
49-
}
50-
51-
impl From<DepIn> for ReleaseDependency {
52-
fn from((name, req, kind, optional): DepIn) -> Self {
53-
ReleaseDependency(Dependency {
54-
name,
55-
req,
56-
kind,
57-
optional: optional.unwrap_or(false),
58-
rename: None,
59-
})
60-
}
61-
}
62-
63-
impl From<ReleaseDependency> for DepOut {
64-
fn from(rd: ReleaseDependency) -> Self {
65-
let d = rd.0;
66-
(
67-
d.name,
68-
d.req.to_string(),
69-
d.kind.unwrap_or_else(|| "normal".into()),
70-
d.optional,
71-
)
72-
}
73-
}
74-
7525
/// Adds a package into database.
7626
///
7727
/// Package must be built first.
@@ -98,7 +48,12 @@ pub(crate) async fn finish_release(
9848
source_size: u64,
9949
) -> Result<()> {
10050
debug!("updating release data");
101-
let dependencies = convert_dependencies(metadata_pkg)?;
51+
let dependencies: ReleaseDependencyList = metadata_pkg
52+
.dependencies
53+
.iter()
54+
.cloned()
55+
.map(Into::into)
56+
.collect();
10257
let rustdoc = get_rustdoc(metadata_pkg, source_dir).unwrap_or(None);
10358
let readme = get_readme(metadata_pkg, source_dir).unwrap_or(None);
10459
let features = get_features(metadata_pkg);
@@ -133,7 +88,7 @@ pub(crate) async fn finish_release(
13388
WHERE id = $1"#,
13489
release_id.0,
13590
registry_data.release_time,
136-
dependencies,
91+
serde_json::to_value(&dependencies)?,
13792
metadata_pkg.package_name(),
13893
registry_data.yanked,
13994
has_docs,
@@ -432,16 +387,6 @@ pub(crate) async fn initialize_build(
432387
Ok(build_id)
433388
}
434389

435-
/// Convert dependencies into our own internal JSON representation
436-
fn convert_dependencies(pkg: &MetadataPackage) -> Result<serde_json::Value> {
437-
let dependencies: Vec<_> = pkg
438-
.dependencies
439-
.iter()
440-
.map(|dependency| ReleaseDependency(dependency.clone()))
441-
.collect::<Vec<_>>();
442-
Ok(serde_json::to_value(dependencies)?)
443-
}
444-
445390
/// Reads features and converts them to Vec<Feature> with default being first
446391
fn get_features(pkg: &MetadataPackage) -> Vec<Feature> {
447392
let mut features = Vec::with_capacity(pkg.features.len());

src/db/mod.rs

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,16 @@ use sqlx::migrate::{Migrate, Migrator};
44

55
pub use self::add_package::update_latest_version_id;
66
pub(crate) use self::add_package::{
7-
ReleaseDependency, add_doc_coverage, finish_build, finish_release, initialize_build,
8-
initialize_crate, initialize_release, update_build_with_error,
7+
add_doc_coverage, finish_build, finish_release, initialize_build, initialize_crate,
8+
initialize_release, update_build_with_error,
99
};
1010
pub use self::{
11-
add_package::{
12-
BuildId, CrateId, ReleaseId, update_build_status, update_crate_data_in_database,
13-
},
11+
add_package::{update_build_status, update_crate_data_in_database},
1412
delete::{delete_crate, delete_version},
1513
file::{add_path_into_database, add_path_into_remote_archive},
1614
overrides::Overrides,
1715
pool::{AsyncPoolClient, Pool, PoolError},
16+
types::{BuildId, CrateId, ReleaseId},
1817
};
1918

2019
mod add_package;

src/db/types/dependencies.rs

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
use crate::utils::Dependency;
2+
use derive_more::Deref;
3+
use semver::VersionReq;
4+
use serde::{Deserialize, Serialize};
5+
6+
const DEFAULT_KIND: &str = "normal";
7+
8+
/// A crate dependency in our internal representation for releases.dependencies json.
9+
#[derive(Debug, Clone, PartialEq, Deref)]
10+
pub(crate) struct ReleaseDependency(Dependency);
11+
12+
impl<'de> Deserialize<'de> for ReleaseDependency {
13+
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
14+
where
15+
D: serde::Deserializer<'de>,
16+
{
17+
/// The three possible representations of a dependency in our internal JSON format
18+
/// in the `releases.dependencies` column.
19+
#[derive(Serialize, Deserialize)]
20+
#[serde(untagged)]
21+
enum Repr {
22+
/// just [name, version]``
23+
Basic((String, VersionReq)),
24+
/// [name, version, kind]
25+
WithKind((String, VersionReq, String)),
26+
/// [name, version, kind, optional]
27+
Full((String, VersionReq, String, bool)),
28+
}
29+
30+
let src = Repr::deserialize(deserializer)?;
31+
let (name, req, kind, optional) = match src {
32+
Repr::Basic((name, req)) => (name, req, DEFAULT_KIND.into(), false),
33+
Repr::WithKind((name, req, kind)) => (name, req, kind, false),
34+
Repr::Full((name, req, kind, optional)) => (name, req, kind, optional),
35+
};
36+
37+
Ok(ReleaseDependency(Dependency {
38+
name,
39+
req,
40+
kind: Some(kind),
41+
optional,
42+
rename: None,
43+
}))
44+
}
45+
}
46+
47+
impl Serialize for ReleaseDependency {
48+
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
49+
where
50+
S: serde::Serializer,
51+
{
52+
let dep = &self.0;
53+
let kind = dep.kind.as_deref().unwrap_or(DEFAULT_KIND);
54+
(dep.name.as_str(), &dep.req, kind, dep.optional).serialize(serializer)
55+
}
56+
}
57+
58+
impl From<Dependency> for ReleaseDependency {
59+
fn from(dep: Dependency) -> Self {
60+
ReleaseDependency(dep)
61+
}
62+
}
63+
64+
impl From<ReleaseDependency> for Dependency {
65+
fn from(dep: ReleaseDependency) -> Self {
66+
dep.0
67+
}
68+
}
69+
70+
pub(crate) type ReleaseDependencyList = Vec<ReleaseDependency>;
71+
72+
#[cfg(test)]
73+
mod tests {
74+
use super::*;
75+
use anyhow::Result;
76+
use test_case::test_case;
77+
78+
#[test_case("[]", "[]"; "empty")]
79+
#[test_case(
80+
r#"[["vec_map", "^0.0.1"]]"#,
81+
r#"[["vec_map","^0.0.1","normal",false]]"#;
82+
"2-tuple"
83+
)]
84+
#[test_case(
85+
r#"[["vec_map", "^0.0.1", "normal" ]]"#,
86+
r#"[["vec_map","^0.0.1","normal",false]]"#;
87+
"3-tuple"
88+
)]
89+
#[test_case(
90+
r#"[["rand", "^0.9", "normal", false], ["sdl3", "^0.16", "normal", false]]"#,
91+
r#"[["rand","^0.9","normal",false],["sdl3","^0.16","normal",false]]"#;
92+
"4-tuple"
93+
)]
94+
#[test_case(
95+
r#"[["byteorder", "^0.5", "normal", false],["clippy", "^0", "normal", true]]"#,
96+
r#"[["byteorder","^0.5","normal",false],["clippy","^0","normal",true]]"#;
97+
"with optional"
98+
)]
99+
fn test_parse_release_dependency_json(input: &str, output: &str) -> Result<()> {
100+
let deps: ReleaseDependencyList = serde_json::from_str(input)?;
101+
102+
assert_eq!(serde_json::to_string(&deps)?, output);
103+
Ok(())
104+
}
105+
106+
#[test_case(r#"[["vec_map", "^0.0.1"]]"#, "normal", false)]
107+
#[test_case(r#"[["vec_map", "^0.0.1", "dev" ]]"#, "dev", false)]
108+
#[test_case(r#"[["vec_map", "^0.0.1", "dev", true ]]"#, "dev", true)]
109+
fn test_parse_dependency(
110+
input: &str,
111+
expected_kind: &str,
112+
expected_optional: bool,
113+
) -> Result<()> {
114+
let deps: ReleaseDependencyList = serde_json::from_str(input)?;
115+
let [dep] = deps.as_slice() else {
116+
panic!("expected exactly one dependency");
117+
};
118+
119+
assert_eq!(dep.name, "vec_map");
120+
assert_eq!(dep.req, VersionReq::parse("^0.0.1")?);
121+
assert_eq!(dep.kind.as_deref(), Some(expected_kind));
122+
assert_eq!(dep.optional, expected_optional);
123+
124+
Ok(())
125+
}
126+
}

src/db/types/mod.rs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,21 @@
1+
use derive_more::Display;
12
use serde::{Deserialize, Serialize};
23

4+
pub mod dependencies;
35
pub mod version;
46

7+
#[derive(Debug, Clone, Copy, Display, PartialEq, Eq, Hash, Serialize, sqlx::Type)]
8+
#[sqlx(transparent)]
9+
pub struct CrateId(pub i32);
10+
11+
#[derive(Debug, Clone, Copy, Display, PartialEq, Eq, Hash, Serialize, sqlx::Type)]
12+
#[sqlx(transparent)]
13+
pub struct ReleaseId(pub i32);
14+
15+
#[derive(Debug, Clone, Copy, Display, PartialEq, Eq, Hash, Serialize, sqlx::Type)]
16+
#[sqlx(transparent)]
17+
pub struct BuildId(pub i32);
18+
519
#[derive(Debug, Clone, PartialEq, Eq, Serialize, sqlx::Type)]
620
#[sqlx(type_name = "feature")]
721
pub struct Feature {

src/web/crate_details.rs

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
use crate::{
22
AsyncStorage,
33
db::{
4-
BuildId, CrateId, ReleaseDependency, ReleaseId,
5-
types::{BuildStatus, version::Version},
4+
BuildId, CrateId, ReleaseId,
5+
types::{BuildStatus, dependencies::ReleaseDependencyList, version::Version},
66
},
77
impl_axum_webpage,
88
registry_api::OwnerKind,
@@ -234,12 +234,17 @@ impl CrateDetails {
234234

235235
let parsed_license = krate.license.as_deref().map(super::licenses::parse_license);
236236

237-
let dependencies = krate
237+
let dependencies: Vec<Dependency> = krate
238238
.dependencies
239-
.and_then(|value| serde_json::from_value::<Vec<ReleaseDependency>>(value).ok())
239+
.map(serde_json::from_value::<ReleaseDependencyList>)
240+
.transpose()
241+
// NOTE: we sometimes have invalid semver-requirement strings the database
242+
// (at the time writing, 14 releases out of 2 million).
243+
// We silently ignore those here.
244+
.unwrap_or_default()
240245
.unwrap_or_default()
241246
.into_iter()
242-
.map(|rdep| rdep.into_inner())
247+
.map(Into::into)
243248
.collect();
244249

245250
let mut crate_details = CrateDetails {

src/web/rustdoc.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2674,6 +2674,7 @@ mod test {
26742674
.await
26752675
.get("/testing/0.1.0/testing/")
26762676
.await?
2677+
.error_for_status()?
26772678
.text()
26782679
.await?
26792680
));

templates/core/about/builds.html

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,17 @@ <h4 id="cross-compiling"> <a href="#cross-compiling">Cross-compiling</a> </h4>
6363
This approach is also useful for setting <a href="https://doc.rust-lang.org/cargo/reference/features.html">cargo features</a>.
6464
</p>
6565

66+
<h4 id="testing-builds-ci"> <a href="#testing-builds-ci">Testing documentation builds in CI</a> </h4>
67+
<p>
68+
When you start using docs.rs metadata or <code>#[cfg(docsrs)]</code> in your code, you might run into issues
69+
where your code builds fine locally but fails on docs.rs.
70+
71+
To avoid this, you can add <a href="https://github.com/dtolnay/cargo-docs-rs"><code>cargo docs-rs</code></a> to
72+
your test-suite.
73+
74+
While it doesn't perfectly replicate our build environment, it can catch many common issues.
75+
</p>
76+
6677
<h4 id="testing-builds-locally"> <a href="#testing-builds-locally">Testing documentation builds locally</a> </h4>
6778
{%- set build_subcommand = "{}/blob/master/README.md#build-subcommand"|format(docsrs_repo) -%}
6879
<p>

0 commit comments

Comments
 (0)