1//! Writes the licence notices of every crates.io crate linked into each
2//! shipped artifact, from the crate archives buck fetched: `THIRD-PARTY`
3//! for the extension (`postjevsql.so`) and `THIRD-PARTY-sidecar` for the
4//! sidecar CLI (`postjevsql-sidecar`). Each package installs only its own
5//! file, since each binary links its own set.
6//!
7//! The crate lists are not kept by hand: `:notices` passes, per artifact,
8//! the extracted archives of its target deps (buck's `query_outputs`), so
9//! they follow `third-party/BUCK`. For each crate this reads
10//! `package.name`, `version`, `license` and `license-file` from its
11//! `Cargo.toml`, every top-level LICENSE / LICENCE / COPYING / NOTICE
12//! file, and the files in a REUSE `LICENSES/` directory.
13//!
14//! A crate with no licence expression or no licence text fails the build:
15//! a binary must carry its notices, so a missing one is not skipped. A
16//! crate whose archive ships no text gets it from `texts/<name>-<version>/`,
17//! fetched from its upstream at that version and named in the directory's
18//! `SOURCE`. A text there for a crate linked into no artifact, or that
19//! ships its own, fails the build too, so the directory cannot go stale.
20//! One `texts/` serves every artifact, which is why they are generated in
21//! one run.
22//!
23//! Identical texts are written once per file, naming every crate that
24//! carries them.
25//!
26//! Usage:
27//!   third-party-notices generate <texts-dir> (-- <out-file> <package> <binary> <crate-dir>...)...
28//!   third-party-notices write (<generated-file> <name>)...   (copies each to <name> in the checkout holding the cwd)
29
30use std::collections::{BTreeMap, BTreeSet};
31use std::error::Error;
32use std::fmt::Write as _;
33use std::path::Path;
34
35type Res<T> = Result<T, Box<dyn Error>>;
36
37const USAGE: &str = "usage: third-party-notices generate <texts-dir> (-- <out-file> <package> <binary> <crate-dir>...)... \
38                     | third-party-notices write (<generated-file> <name>)...";
39
40struct Crate {
41    name: String,
42    version: String,
43    license: String,
44    /// File name and text of each licence file.
45    texts: Vec<(String, String)>,
46}
47
48/// One shipped binary and the crates linked into it.
49struct Artifact {
50    out: String,
51    /// The package that installs the file, e.g. `postjevsql-sidecar`.
52    package: String,
53    /// What the crates are linked into, e.g. `postjevsql.so`.
54    binary: String,
55    crates: BTreeMap<(String, String), Crate>,
56}
57
58fn main() {
59    let args: Vec<String> = std::env::args().skip(1).collect();
60    let result = match args.first().map(String::as_str) {
61        Some("generate") if args.len() >= 2 => parse_artifacts(&args[2..])
62            .and_then(|artifacts| generate(Path::new(&args[1]), artifacts)),
63        Some("write") if args.len() >= 3 && args.len() % 2 == 1 => {
64            args[1..].chunks(2).try_for_each(|p| write(Path::new(&p[0]), &p[1]))
65        }
66        _ => Err(USAGE.into()),
67    };
68    if let Err(e) = result {
69        eprintln!("third-party-notices: {e}");
70        std::process::exit(1);
71    }
72}
73
74fn parse_artifacts(args: &[String]) -> Res<Vec<Artifact>> {
75    if args.first().map(String::as_str) != Some("--") {
76        return Err(USAGE.into());
77    }
78    let mut artifacts = Vec::new();
79    for group in args[1..].split(|a| a == "--") {
80        let [out, package, binary, dirs @ ..] = group else {
81            return Err(USAGE.into());
82        };
83        // A crate configured twice (e.g. for two platforms) is one crate.
84        let mut crates = BTreeMap::new();
85        for dir in dirs {
86            let c = read_crate(Path::new(dir))?;
87            crates.insert((c.name.clone(), c.version.clone()), c);
88        }
89        if crates.is_empty() {
90            return Err(format!("{binary}: no crates given").into());
91        }
92        artifacts.push(Artifact {
93            out: out.clone(),
94            package: package.clone(),
95            binary: binary.clone(),
96            crates,
97        });
98    }
99    Ok(artifacts)
100}
101
102fn generate(supplements: &Path, mut artifacts: Vec<Artifact>) -> Res<()> {
103    for entry in std::fs::read_dir(supplements)? {
104        let dir = entry?.path();
105        let id = dir.file_name().unwrap_or_default().to_string_lossy().into_owned();
106        let mut holders: Vec<&mut Crate> = artifacts
107            .iter_mut()
108            .filter_map(|a| a.crates.values_mut().find(|c| format!("{}-{}", c.name, c.version) == id))
109            .collect();
110        if holders.is_empty() {
111            return Err(format!("texts/{id}: not a crate linked into any artifact (remove it)").into());
112        }
113        if holders.iter().any(|c| !c.texts.is_empty()) {
114            return Err(format!("texts/{id}: the crate ships its own licence text (remove it)").into());
115        }
116        if !dir.join("SOURCE").is_file() {
117            return Err(format!("texts/{id}: no SOURCE naming where the text came from").into());
118        }
119        let mut texts = Vec::new();
120        for f in std::fs::read_dir(&dir)? {
121            let f = f?.file_name().to_string_lossy().into_owned();
122            if f != "SOURCE" {
123                let text = std::fs::read_to_string(dir.join(&f))?;
124                texts.push((format!("{f}, from upstream"), normalize(&text)));
125            }
126        }
127        texts.sort();
128        for c in &mut holders {
129            c.texts = texts.clone();
130        }
131    }
132    for a in &artifacts {
133        write_notices(a)?;
134    }
135    Ok(())
136}
137
138fn write_notices(a: &Artifact) -> Res<()> {
139    let missing: Vec<String> = a
140        .crates
141        .values()
142        .filter(|c| c.texts.is_empty())
143        .map(|c| format!("{} {} ({})", c.name, c.version, c.license))
144        .collect();
145    if !missing.is_empty() {
146        return Err(format!("{}: no licence text in: {}", a.binary, missing.join(", ")).into());
147    }
148
149    let mut s = String::new();
150    writeln!(s, "Third-party licence notices for {}\n", a.package)?;
151    writeln!(s, "@generated by tools/third-party-notices from the crates linked into")?;
152    writeln!(s, "{}. Do not edit: change the dependencies and run", a.binary)?;
153    writeln!(s, "  buck2 run //tools/third-party-notices:update")?;
154    writeln!(s, "The drift test (//tools/third-party-notices:drift) fails until you do.\n")?;
155    writeln!(s, "{} itself is MIT OR Apache-2.0 (LICENSE-MIT, LICENSE-APACHE).", a.package)?;
156    writeln!(s, "The crates below are linked into {} under their own terms.", a.binary)?;
157    writeln!(s, "\nCrate, version, licence (SPDX, as the crate declares it):\n")?;
158    for c in a.crates.values() {
159        writeln!(s, "  {} {}  {}", c.name, c.version, c.license)?;
160    }
161
162    // Each distinct text once, with the crates that carry it.
163    let mut by_text: BTreeMap<&str, BTreeSet<String>> = BTreeMap::new();
164    for c in a.crates.values() {
165        for (file, text) in &c.texts {
166            by_text
167                .entry(text.as_str())
168                .or_default()
169                .insert(format!("{} {} ({file})", c.name, c.version));
170        }
171    }
172    let mut blocks: Vec<(String, &str)> = by_text
173        .into_iter()
174        .map(|(text, users)| (users.into_iter().collect::<Vec<_>>().join(", "), text))
175        .collect();
176    blocks.sort();
177    for (users, text) in blocks {
178        writeln!(s, "\n{}\n{users}\n{}\n\n{text}", "=".repeat(72), "=".repeat(72))?;
179    }
180    Ok(std::fs::write(&a.out, s)?)
181}
182
183fn read_crate(dir: &Path) -> Res<Crate> {
184    let manifest: toml::Table = std::fs::read_to_string(dir.join("Cargo.toml"))?.parse()?;
185    let pkg = manifest
186        .get("package")
187        .and_then(|p| p.as_table())
188        .ok_or_else(|| format!("{}: no [package]", dir.display()))?;
189    let field = |k: &str| pkg.get(k).and_then(|v| v.as_str()).map(str::to_owned);
190    let name = field("name").ok_or_else(|| format!("{}: no package.name", dir.display()))?;
191    let version = field("version").ok_or_else(|| format!("{name}: no package.version"))?;
192    let license = field("license").ok_or_else(|| format!("{name} {version}: no package.license"))?;
193
194    let mut files = BTreeSet::new();
195    for entry in std::fs::read_dir(dir)? {
196        let entry = entry?;
197        let file = entry.file_name().to_string_lossy().into_owned();
198        let upper = file.to_ascii_uppercase();
199        if entry.file_type()?.is_file()
200            && ["LICENSE", "LICENCE", "COPYING", "NOTICE"].iter().any(|p| upper.starts_with(p))
201        {
202            files.insert(file);
203        }
204    }
205    let reuse = dir.join("LICENSES");
206    if reuse.is_dir() {
207        for entry in std::fs::read_dir(&reuse)? {
208            files.insert(format!("LICENSES/{}", entry?.file_name().to_string_lossy()));
209        }
210    }
211    if let Some(f) = field("license-file") {
212        files.insert(f);
213    }
214    let texts = files
215        .into_iter()
216        .map(|f| {
217            let text = std::fs::read_to_string(dir.join(&f))
218                .map_err(|e| format!("{name} {version}: {f}: {e}"))?;
219            Ok((f, normalize(&text)))
220        })
221        .collect::<Res<_>>()?;
222    Ok(Crate { name, version, license, texts })
223}
224
225/// Line endings and trailing blanks differ between otherwise equal copies.
226fn normalize(text: &str) -> String {
227    let lines: Vec<&str> = text.lines().map(str::trim_end).collect();
228    lines.join("\n").trim_matches('\n').to_owned()
229}
230
231/// Copies a generated file to `name` in the checkout holding the cwd,
232/// found by its `.buckroot`.
233fn write(generated: &Path, name: &str) -> Res<()> {
234    let cwd = std::env::current_dir()?;
235    let root = cwd
236        .ancestors()
237        .find(|d| d.join(".buckroot").exists())
238        .ok_or("not inside a buck checkout (no .buckroot above the cwd)")?;
239    let dest = root.join(name);
240    std::fs::copy(generated, &dest)?;
241    println!("wrote {}", dest.display());
242    Ok(())
243}