annotate.rsA source file read the way Docco reads one: its comments beside the code
they are about. lex colours a file (comments, strings, keywords,
types, calls); sections cuts it into comments and the code each one
is about, the comment as prose and the code as lines.
The file is lexed whole before it is cut, so a line that only looks like
a comment (// ... inside a string that spans lines, as the JavaScript
kept in a Rust raw string does) stays code.
What a piece of source is, for its colour.
A comment to the end of its line, and one between delimiters.
A name that begins with a capital: a type, in the languages here.
21 Type,
A name that is called, or a macro.
23 Call,
28impl Kind {
The class its span has on the page, if it has a colour.
How a language writes comments and strings, and its reserved words.
What begins a comment that runs to the end of the line.
49 line: &'static [&'static str],
What opens and closes a comment that can span lines.
51 block: Option<(&'static str, &'static str)>,
' quotes a string (not in Rust, where it is a char or a lifetime).
53 single: bool,
Rust's raw strings, lifetimes, attributes and macros.
55 rust: bool,
A string between these may span lines (`, """, '').
57 long: &'static [&'static str],
Keywords are matched whatever their case (SQL).
63const RUST: Language = Language { 64 name: "Rust", 65 line: &["//"], 66 block: Some(("/*", "*/")), 67 single: false, 68 rust: true, 69 long: &[], 70 any_case: false, 71 keywords: &[ 72 "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop", 73 "match", "mod", "move", "mut", "pub", "ref", "return", "self", "Self", "static", "struct", "super", "trait", "true", "type", "unsafe", "use", "where", 74 "while", 75 ], 76}; 77const JS: Language = Language { 78 name: "JavaScript", 79 line: &["//"], 80 block: Some(("/*", "*/")), 81 single: true, 82 rust: false, 83 long: &["`"], 84 any_case: false, 85 keywords: &[ 86 "async", "await", "break", "case", "catch", "class", "const", "continue", "default", "delete", "do", "else", "export", "extends", "false", "finally", "for", 87 "from", "function", "if", "import", "in", "instanceof", "let", "new", "null", "of", "return", "static", "super", "switch", "this", "throw", "true", "try", 88 "typeof", "undefined", "var", "void", "while", "yield", 89 ], 90}; 91const CSS: Language = Language { name: "CSS", line: &[], block: Some(("/*", "*/")), single: true, rust: false, long: &[], any_case: false, keywords: &[] }; 92const TOML: Language = 93 Language { name: "TOML", line: &["#"], block: None, single: true, rust: false, long: &["\"\"\"", "'''"], any_case: false, keywords: &["true", "false"] }; 94const NIX: Language = Language { 95 name: "Nix", 96 line: &["#"], 97 block: Some(("/*", "*/")), 98 single: false, 99 rust: false, 100 long: &["''"], 101 any_case: false, 102 keywords: &["assert", "else", "false", "if", "import", "in", "inherit", "let", "null", "or", "rec", "then", "true", "with"], 103}; 104const SHELL: Language = Language { 105 name: "Shell", 106 line: &["#"], 107 block: None, 108 single: true, 109 rust: false, 110 long: &[], 111 any_case: false, 112 keywords: &["case", "do", "done", "elif", "else", "esac", "exec", "exit", "export", "fi", "for", "function", "if", "in", "local", "return", "then", "while"], 113}; 114const SQL: Language = Language { 115 name: "SQL", 116 line: &["--"], 117 block: Some(("/*", "*/")), 118 single: true, 119 rust: false, 120 long: &[], 121 any_case: true, 122 keywords: &[ 123 "add", "alter", "and", "as", "asc", "by", "case", "column", "create", "default", "delete", "desc", "distinct", "drop", "else", "end", "exists", "from", 124 "group", "having", "if", "in", "index", "insert", "integer", "into", "is", "join", "key", "left", "like", "limit", "not", "null", "on", "or", "order", 125 "primary", "real", "select", "set", "table", "text", "then", "union", "unique", "update", "values", "when", "where", "with", 126 ], 127}; 128const PYTHON: Language = Language { 129 name: "Python", 130 line: &["#"], 131 block: None, 132 single: true, 133 rust: false, 134 long: &["\"\"\"", "'''"], 135 any_case: false, 136 keywords: &[ 137 "and", "as", "assert", "break", "class", "continue", "def", "elif", "else", "except", "False", "finally", "for", "from", "if", "import", "in", "is", 138 "lambda", "None", "not", "or", "pass", "raise", "return", "True", "try", "while", "with", "yield", 139 ], 140}; 141const JSON: Language = 142 Language { name: "JSON", line: &[], block: None, single: false, rust: false, long: &[], any_case: false, keywords: &["true", "false", "null"] };
The language a file is in, by its name. None for one not known here,
which is shown as plain lines.
146pub fn language(path: &str) -> Option<&'static Language> { 147 let name = path.rsplit('/').next().unwrap_or(path); 148 let extension = name.rsplit_once('.').map_or("", |(_, extension)| extension); 149 Some(match (name, extension) { 150 (_, "rs") => &RUST, 151 (_, "js" | "mjs" | "cjs" | "ts") => &JS, 152 (_, "css") => &CSS, 153 (_, "toml") | ("Cargo.lock", _) => &TOML, 154 (_, "nix") => &NIX, 155 (_, "sh" | "bash") | (".envrc", _) => &SHELL, 156 (_, "sql") => &SQL, 157 (_, "py" | "bzl") | ("BUCK" | "PACKAGE" | ".buckconfig", _) => &PYTHON, 158 (_, "yml" | "yaml") | (".gitignore" | ".gitmodules", _) => &TOML, 159 (_, "json") => &JSON, 160 _ => return None, 161 }) 162}
Where the string that begins rest ends, if one begins there.
165fn string(rest: &str, language: &Language) -> Option<usize> { 166 if language.rust { 167 // r"..", r#".."#, br"..", and b"..". 168 let after = rest.strip_prefix("br").or_else(|| rest.strip_prefix('r')); 169 if let Some(after) = after { 170 let hashes = after.len() - after.trim_start_matches('#').len(); 171 if after[hashes..].starts_with('"') { 172 let open = rest.len() - after.len() + hashes + 1; 173 let close = format!("\"{}", "#".repeat(hashes)); 174 return Some(rest[open..].find(&close).map_or(rest.len(), |at| open + at + close.len())); 175 } 176 } 177 if rest.starts_with("b\"") { 178 return quoted(&rest[1..], '"', true).map(|end| end + 1); 179 } 180 } 181 for quote in language.long { 182 if let Some(inside) = rest.strip_prefix(quote) { 183 return Some(inside.find(quote).map_or(rest.len(), |at| at + 2 * quote.len())); 184 } 185 } 186 match rest.chars().next()? { 187 '"' => quoted(rest, '"', language.rust), 188 '\'' if language.single => quoted(rest, '\'', false), 189 _ => None, 190 } 191}
Where a string in quotes ends, minding \. One that does not close on
its line is not a string, unless the language lets strings span lines.
195fn quoted(rest: &str, quote: char, spans: bool) -> Option<usize> { 196 let mut escaped = false; 197 for (at, c) in rest.char_indices().skip(1) { 198 match c { 199 _ if escaped => escaped = false, 200 '\\' => escaped = true, 201 '\n' if !spans => return None, 202 c if c == quote => return Some(at + c.len_utf8()), 203 _ => {} 204 } 205 } 206 spans.then_some(rest.len()) 207}
Rust's ': a char ('x', '\n') is a string, a lifetime ('a) is not.
210fn tick(rest: &str) -> Option<(Kind, usize)> { 211 let mut chars = rest.char_indices().skip(1); 212 let (_, first) = chars.next()?; 213 if first == '\\' { 214 return quoted(rest, '\'', false).map(|end| (Kind::Str, end)); 215 } 216 match chars.next() { 217 Some((at, '\'')) => Some((Kind::Str, at + 1)), 218 _ if first.is_alphabetic() || first == '_' => { 219 let end = rest[1..].find(|c: char| !c.is_alphanumeric() && c != '_').map_or(rest.len(), |at| at + 1); 220 Some((Kind::Meta, end)) 221 } 222 _ => None, 223 } 224}
The file as pieces, each with its kind, in order and with nothing left out: joined, they are the file.
228pub fn lex<'a>(text: &'a str, language: &Language) -> Vec<(Kind, &'a str)> { 229 let mut pieces: Vec<(Kind, &str)> = Vec::new(); 230 let mut plain = 0; 231 let mut at = 0; 232 while at < text.len() { 233 let rest = &text[at..]; 234 let c = rest.chars().next().expect("at is inside the text"); 235 let found: Option<(Kind, usize)> = if language.line.iter().any(|mark| rest.starts_with(mark)) { 236 Some((Kind::Comment, rest.find('\n').unwrap_or(rest.len()))) 237 } else if let Some((open, close)) = language.block.filter(|(open, _)| rest.starts_with(open)) { 238 Some((Kind::Block, rest[open.len()..].find(close).map_or(rest.len(), |end| open.len() + end + close.len()))) 239 } else if let Some(end) = string(rest, language) { 240 Some((Kind::Str, end)) 241 } else if language.rust && c == '\'' { 242 tick(rest) 243 } else if language.rust && (rest.starts_with("#[") || rest.starts_with("#![")) { 244 // To the bracket that closes it, on however many lines. 245 let mut depth = 0; 246 let end = rest.char_indices().find_map(|(at, c)| { 247 match c { 248 '[' => depth += 1, 249 ']' => depth -= 1, 250 _ => {} 251 } 252 (c == ']' && depth == 0).then_some(at + 1) 253 }); 254 Some((Kind::Meta, end.unwrap_or(rest.len()))) 255 } else if c.is_ascii_digit() { 256 let end = rest.char_indices().find(|(at, c)| !(c.is_ascii_alphanumeric() || *c == '_' || (*c == '.' && !rest[*at..].starts_with("..")))).map_or(rest.len(), |(at, _)| at); 257 Some((Kind::Number, end)) 258 } else if c.is_alphabetic() || c == '_' { 259 let end = rest.find(|c: char| !c.is_alphanumeric() && c != '_').unwrap_or(rest.len()); 260 let word = &rest[..end]; 261 let after = &rest[end..]; 262 let keyword = match language.any_case { 263 true => language.keywords.iter().any(|keyword| keyword.eq_ignore_ascii_case(word)), 264 false => language.keywords.contains(&word), 265 }; 266 let kind = if keyword { 267 Kind::Keyword 268 } else if language.rust && after.starts_with('!') && !after.starts_with("!=") { 269 Kind::Call 270 } else if after.starts_with('(') { 271 Kind::Call 272 } else if c.is_uppercase() { 273 Kind::Type 274 } else { 275 Kind::Plain 276 }; 277 Some((kind, end)) 278 } else { 279 None 280 }; 281 match found { 282 Some((Kind::Plain, len)) | Some((_, len)) if len == 0 || found.is_some_and(|(kind, _)| kind == Kind::Plain) => at += len.max(c.len_utf8()), 283 Some((kind, len)) => { 284 if plain < at { 285 pieces.push((Kind::Plain, &text[plain..at])); 286 } 287 pieces.push((kind, &rest[..len])); 288 at += len; 289 plain = at; 290 } 291 None => at += c.len_utf8(), 292 } 293 } 294 if plain < text.len() { 295 pieces.push((Kind::Plain, &text[plain..])); 296 } 297 pieces 298}
One line of a file: its number from 1, and its pieces.
How far in the line begins, in columns of whitespace.
The line only closes something opened above it.
A comment written to document what follows: Rust's /// and //!,
and a /** block.
Nothing on the line but comment.
The file's lines, each with its pieces. A piece that spans lines (a block comment, a long string) is cut at each line's end and keeps its kind.
345pub fn lines<'a>(text: &'a str, language: &Language) -> Vec<Line<'a>> { 346 let mut lines = vec![Line { number: 1, pieces: Vec::new() }]; 347 for (kind, piece) in lex(text, language) { 348 for (index, part) in piece.split('\n').enumerate() { 349 if index > 0 { 350 let number = lines.len() + 1; 351 lines.push(Line { number, pieces: Vec::new() }); 352 } 353 if !part.is_empty() { 354 lines.last_mut().expect("there is a first line").pieces.push((kind, part)); 355 } 356 } 357 } 358 // A file that ends with a newline has no line after it. 359 if text.ends_with('\n') { 360 lines.pop(); 361 } 362 lines 363}
A run of comment, and the code under it.
The comments' words, their marks taken off: markdown, as Rust's doc comments are, and near enough for any other.
What a comment line says, without what marks it as a comment.
375fn said(line: &Line<'_>, language: &Language) -> String { 376 let text: String = line.pieces.iter().map(|(_, text)| *text).collect(); 377 let mut text = text.trim(); 378 if line.pieces.iter().any(|(kind, _)| *kind == Kind::Block) { 379 if let Some((open, close)) = language.block { 380 text = text.strip_prefix(open).unwrap_or(text); 381 text = text.strip_suffix(close).unwrap_or(text).trim_end(); 382 text = text.strip_prefix(['*', '!']).unwrap_or(text); 383 } 384 } else { 385 // `///` and `//!` before `//`. 386 let mark = language.line.iter().find(|mark| text.starts_with(**mark)).copied().unwrap_or_default(); 387 text = &text[mark.len()..]; 388 if language.rust { 389 text = text.strip_prefix(['/', '!']).unwrap_or(text); 390 } 391 } 392 text.strip_prefix(' ').unwrap_or(text).to_owned() 393}
The file as sections, each a comment and the code it is about.
A section begins at a comment that stands for what follows it: one at
the left margin, a doc comment at any depth (Rust's ///), or, in a
language with no doc comments, one after a blank line. A comment in the
middle of a body, straight after a line of code, is a remark on that
code and stays in it.
A comment speaks for the item under it and no further: after a blank line, code that begins no deeper than that item is a section of its own, with nothing beside it. A file with no such comments is one section.
406pub fn sections<'a>(text: &'a str, language: &Language) -> Vec<Section<'a>> { 407 let mut sections: Vec<Section<'a>> = Vec::new(); 408 let mut prose: Vec<String> = Vec::new(); 409 let mut code: Vec<Line<'a>> = Vec::new(); 410 let mut close = |prose: &mut Vec<String>, code: &mut Vec<Line<'a>>| { 411 while code.last().is_some_and(Line::blank) { 412 code.pop(); 413 } 414 while prose.last().is_some_and(|line| line.is_empty()) { 415 prose.pop(); 416 } 417 if !prose.is_empty() || !code.is_empty() { 418 sections.push(Section { prose: prose.join("\n"), code: std::mem::take(code) }); 419 } 420 prose.clear(); 421 }; 422 // Whether the line before was blank (or there was none). 423 let mut gap = true; 424 for line in lines(text, language) { 425 // A script's first line names its interpreter; it is not a comment. 426 let shebang = line.number == 1 && line.pieces.first().is_some_and(|(_, text)| text.starts_with("#!") && !language.rust); 427 let blank = line.blank(); 428 // Still in the comment that opened this section. 429 let continues = code.is_empty() && !prose.is_empty(); 430 let opens = line.indent() == 0 || if language.rust { line.doc() } else { gap }; 431 if line.comment() && !shebang && (continues || opens) { 432 if !code.is_empty() { 433 close(&mut prose, &mut code); 434 } 435 prose.push(said(&line, language)); 436 } else if blank && code.is_empty() { 437 // A gap between comments is a paragraph break; one before the 438 // code is dropped when the section closes. 439 if !prose.is_empty() { 440 prose.push(String::new()); 441 } 442 } else { 443 // The comment's item has ended, and something new begins. 444 let claimed = code.iter().find(|line| !line.blank()).map(Line::indent); 445 if !prose.is_empty() && gap && !blank && !line.closes() && claimed.is_some_and(|claimed| line.indent() <= claimed) { 446 close(&mut prose, &mut code); 447 } 448 code.push(line); 449 } 450 gap = blank; 451 } 452 close(&mut prose, &mut code); 453 sections 454}
456#[cfg(test)] 457mod tests { 458 use super::*; 459 460 fn kinds<'a>(text: &'a str, language: &Language) -> Vec<(Kind, &'a str)> { 461 lex(text, language).into_iter().filter(|(kind, _)| *kind != Kind::Plain).collect() 462 } 463 464 #[test] 465 fn lexing_leaves_nothing_out() { 466 for (path, text) in [ 467 ("a.rs", "//! Top.\nfn main() { let s = r#\"a \"quoted\" // not\"#; println!(\"{}\", 'x'); }\n"), 468 ("a.js", "// hi\nconst a = `x ${y}`; /* b */ let it's = 1;\n"), 469 ("a.toml", "# why\nname = \"lm'jtfy\" # trailing\nurl = \"https://x\"\n"), 470 ("a.css", "/* The look.\n Two lines. */\nnav { color: #f386a1; }\n"), 471 ("a.rs", "let unclosed = \"never ends\nfn é() {}"), 472 ] { 473 let language = language(path).unwrap(); 474 let joined: String = lex(text, language).into_iter().map(|(_, piece)| piece).collect(); 475 assert_eq!(joined, text); 476 let relined: Vec<String> = lines(text, language).iter().map(|line| line.pieces.iter().map(|(_, piece)| *piece).collect()).collect(); 477 assert_eq!(relined, text.lines().collect::<Vec<_>>(), "{path}"); 478 } 479 } 480 481 #[test] 482 fn rust_is_coloured() { 483 let text = "#[derive(Debug)]\npub struct Event<'a> { n: u32 }\nfn go(x: &'a str) -> Option<String> { format!(\"{x}\"); let c = '\\n'; a != b; 1.5..2 }"; 484 let found = kinds(text, &RUST); 485 for wanted in [ 486 (Kind::Meta, "#[derive(Debug)]"), 487 (Kind::Keyword, "pub"), 488 (Kind::Keyword, "struct"), 489 (Kind::Type, "Event"), 490 (Kind::Meta, "'a"), 491 (Kind::Call, "go"), 492 (Kind::Type, "Option"), 493 (Kind::Call, "format"), 494 (Kind::Str, "\"{x}\""), 495 (Kind::Str, "'\\n'"), 496 (Kind::Number, "1.5"), 497 (Kind::Number, "2"), 498 ] { 499 assert!(found.contains(&wanted), "{wanted:?} not in {found:?}"); 500 } 501 // `a != b` is not a macro. 502 assert!(!found.contains(&(Kind::Call, "a"))); 503 } 504 505 #[test] 506 fn a_comment_inside_a_string_is_not_a_comment() { 507 let text = "/// The drawer.\nconst JS: &str = r##\"\n// The page's look.\nlet a = 1;\n\"##;\n// After.\nfn f() {}\n"; 508 let sections = sections(text, &RUST); 509 assert_eq!(sections.len(), 2); 510 assert_eq!(sections[0].prose, "The drawer."); 511 // The line that looks like a comment is code, in the string. 512 assert_eq!(sections[0].code.iter().map(|line| line.number).collect::<Vec<_>>(), [2, 3, 4, 5]); 513 assert_eq!(sections[0].code[1].pieces, [(Kind::Str, "// The page's look.")]); 514 assert_eq!((sections[1].prose.as_str(), sections[1].code[0].number), ("After.", 7)); 515 } 516 517 #[test] 518 fn a_comment_speaks_for_the_item_under_it_and_no_further() { 519 // The shape of jev-client's client.rs, which the owner read on the 520 // site (2026-10-03): the constant's comment had the struct after it 521 // beside it too, and a remark inside a function cut the function. 522 let text = "\ 523use std::fmt; 524 525/// Resends before it counts. 526const MAX: u32 = 2; 527 528pub struct Client { 529 transport: T, 530} 531 532/// A verified response. 533pub struct Answered { 534 pub response: Response, 535 /// The headers as they arrived. 536 pub headers: HeaderMap, 537} 538 539impl Client { 540 /// Sends it. 541 pub fn send(&self) { 542 let a = 1; 543 544 let b = 2; 545 } 546 547 fn headers(&self) { 548 if retries > 0 { 549 // As both vendor SDKs do. 550 headers.insert(); 551 } 552 } 553} 554"; 555 let sections = sections(text, &RUST); 556 // Each section's comment, and its first and last line of code. 557 let found: Vec<(&str, Vec<usize>)> = 558 sections.iter().map(|section| (section.prose.as_str(), vec![section.code[0].number, section.code[section.code.len() - 1].number])).collect(); 559 assert_eq!( 560 found, 561 [ 562 ("", vec![1, 1]), 563 // The constant alone; the struct after it has nothing said of it. 564 ("Resends before it counts.", vec![4, 4]), 565 ("", vec![6, 8]), 566 ("A verified response.", vec![11, 12]), 567 // A field's own doc comment, beside the field. 568 ("The headers as they arrived.", vec![14, 15]), 569 ("", vec![17, 17]), 570 // A blank line inside the method does not end it; the next 571 // method, as deep as this one, does. 572 ("Sends it.", vec![19, 23]), 573 // The remark inside the function stays in the function. 574 ("", vec![25, 31]), 575 ] 576 ); 577 } 578 579 #[test] 580 fn comments_become_prose_and_the_code_under_them_keeps_its_line_numbers() { 581 let text = "//! The module.\n//!\n//! Second paragraph.\n\nuse std::fmt;\n\n/// Adds.\n///\n/// - one\nfn add() {} // trailing\n\n\n// plain\nfn sub() {}\n"; 582 let sections = sections(text, &RUST); 583 assert_eq!(sections.iter().map(|section| section.prose.as_str()).collect::<Vec<_>>(), ["The module.\n\nSecond paragraph.", "Adds.\n\n- one", "plain"]); 584 assert_eq!(sections.iter().map(|section| section.code.iter().map(|line| line.number).collect::<Vec<_>>()).collect::<Vec<_>>(), [vec![5], vec![10], vec![14]]); 585 // In a language with no doc comments, an indented comment opens a 586 // section after a blank line, and is a remark straight after code. 587 let nix = super::sections("{\n a = 1;\n\n # The tool.\n tool = x;\n # why\n other = y;\n}\n", &NIX); 588 assert_eq!(nix.iter().map(|section| (section.prose.as_str(), section.code[0].number)).collect::<Vec<_>>(), [("", 1), ("The tool.", 5)]); 589 assert_eq!(nix[1].code.len(), 4); 590 // A trailing comment stays with its code. 591 assert!(sections[1].code[0].pieces.contains(&(Kind::Comment, "// trailing"))); 592 } 593 594 #[test] 595 fn other_languages_have_their_own_comments() { 596 let toml = sections("# The key is a secret.\n[secrets]\nrequired = [\"A\"] # here\n", &TOML); 597 assert_eq!((toml[0].prose.as_str(), toml[0].code.len()), ("The key is a secret.", 2)); 598 let css = sections("/* The look.\n Square. */\nnav { gap: 0.5rem; }\n", &CSS); 599 assert_eq!(css[0].prose, "The look.\nSquare."); 600 assert_eq!(css[0].code[0].number, 3); 601 let sql = sections("-- Newest first.\nSELECT * FROM events ORDER BY id desc;\n", &SQL); 602 assert_eq!(sql[0].prose, "Newest first."); 603 assert!(sql[0].code[0].pieces.contains(&(Kind::Keyword, "desc"))); 604 let shell = sections("#!/usr/bin/env bash\n# Runs it.\nexec wrangler dev\n", &SHELL); 605 assert_eq!((shell[0].prose.as_str(), shell[0].code[0].number), ("", 1)); 606 assert_eq!(shell[1].prose, "Runs it."); 607 // No whole-line comments: one section, no prose. 608 let json = sections("{\"a\": true}\n", &JSON); 609 assert_eq!((json.len(), json[0].prose.as_str()), (1, "")); 610 } 611 612 #[test] 613 fn a_file_is_known_by_its_name() { 614 for (path, name) in [("src/lib.rs", "Rust"), ("page.js", "JavaScript"), ("wrangler.toml", "TOML"), ("flake.nix", "Nix"), (".envrc", "Shell"), ("x/BUCK", "Python")] { 615 assert_eq!(language(path).map(|language| language.name), Some(name), "{path}"); 616 } 617 assert!(language("README.md").is_none()); 618 assert!(language("emoji.woff2").is_none()); 619 } 620}