Menu
Public repository / 3a1d4dc0df9425a910df82402300a2aeb40318ae
AkurAI-Framework / crates/cli/src/seo.rs
Latest change 671120509707c7fd3eec87700f23f98bd3aea5bb - feat(seo): sitemap.xml + robots.txt + per-route head metadata by Ólafur Búi Ólafsson
//! SEO essentials: `robots.txt`, `sitemap.xml`, and per-route `<head>` metadata.
//!
//! All three are derived from declarative project files, never compiled in:
//! - `backend/routes.json` supplies the URL set for the sitemap, plus optional
//! per-route `title`/`description` for `<head>` metadata.
//! - `backend/page.json` supplies an optional `"site_url"` string. When present,
//! sitemap `<loc>`s and the robots `Sitemap:` line become absolute URLs;
//! when absent they fall back to root-relative paths.
//!
//! Only STATIC routes go in the sitemap — patterns with `:param` or `*wildcard`
//! need data we don't have here, so they are skipped. XML special characters in
//! `<loc>` values are escaped so the output is always well-formed.
use crate::cmd_serve;
use akurai_http::Response;
use akurai_json::Value;
use std::fs;
use std::path::Path;
/// `GET /robots.txt` — allow all crawling and point at the sitemap.
pub fn robots(root: &Path) -> Response {
let site = site_url(root);
Response::new(200).with_body(
"text/plain; charset=utf-8",
build_robots(site.as_deref()).into_bytes(),
)
}
/// `GET /sitemap.xml` — a urlset built from the static routes in `routes.json`.
pub fn sitemap(root: &Path) -> Response {
let site = site_url(root);
let paths = route_paths(root);
Response::new(200).with_body(
"application/xml; charset=utf-8",
build_sitemap(paths.iter().map(String::as_str), site.as_deref()).into_bytes(),
)
}
/// A declared route with its optional `<head>` metadata.
struct RouteEntry {
pattern: String,
title: Option<String>,
description: Option<String>,
}
/// Extra render-context pairs (`meta_title`/`meta_description`) for the route
/// matching `path`, or an empty vec when the route carries no metadata. Routes
/// without these fields contribute nothing, so behaviour is unchanged for them.
pub fn route_meta(root: &Path, path: &str) -> Vec<(String, Value)> {
let mut out = Vec::new();
let Some(routes) = load_routes(root) else {
return out;
};
// Match the request path against the declared patterns via the same router
// semantics used for page routing, so `:param`/`*wildcard` routes resolve.
let mut router = akurai_router::Router::new();
for route in &routes {
router.route(
&route.pattern,
(route.title.clone(), route.description.clone()),
);
}
if let Some(m) = router.match_path(path) {
let (title, desc) = &m.value;
if let Some(t) = title {
out.push(("meta_title".into(), Value::Str(t.clone())));
}
if let Some(d) = desc {
out.push(("meta_description".into(), Value::Str(d.clone())));
}
}
out
}
/// Read the optional `"site_url"` string from `backend/page.json`.
fn site_url(root: &Path) -> Option<String> {
cmd_serve::load_context(root)
.get("site_url")
.and_then(Value::as_str)
.map(|s| s.to_string())
}
/// Parse `backend/routes.json` into [`RouteEntry`] values. Missing, unreadable,
/// or malformed files yield `None` rather than an error.
fn load_routes(root: &Path) -> Option<Vec<RouteEntry>> {
let path = root.parent()?.join("backend/routes.json");
let text = fs::read_to_string(path).ok()?;
let value = akurai_json::parse(&text).ok()?;
let Some(Value::Array(entries)) = value.get("routes") else {
return None;
};
let mut out = Vec::new();
for entry in entries {
if let Some(pattern) = entry.get("path").and_then(Value::as_str) {
out.push(RouteEntry {
pattern: pattern.to_string(),
title: entry.get("title").and_then(Value::as_str).map(String::from),
description: entry
.get("description")
.and_then(Value::as_str)
.map(String::from),
});
}
}
Some(out)
}
/// All declared route patterns (filtering to static ones happens in
/// [`build_sitemap`], the single source of truth for that rule).
fn route_paths(root: &Path) -> Vec<String> {
load_routes(root)
.map(|routes| routes.into_iter().map(|r| r.pattern).collect())
.unwrap_or_default()
}
/// A route is sitemap-eligible only if it is fully static — no `:param` and no
/// `*wildcard` segments, which would need data we don't have at this layer.
fn is_static_route(path: &str) -> bool {
!path.contains(':') && !path.contains('*')
}
/// Build a `robots.txt` body that allows all crawling and points at the
/// sitemap (absolute when `site_url` is set, root-relative otherwise).
fn build_robots(site_url: Option<&str>) -> String {
let sitemap = match site_url.map(|u| u.trim_end_matches('/')) {
Some(base) => format!("{base}/sitemap.xml"),
None => "/sitemap.xml".to_string(),
};
format!("User-agent: *\nAllow: /\n\nSitemap: {sitemap}\n")
}
/// Build a well-formed sitemap urlset from the given paths, skipping dynamic
/// routes and escaping XML special characters in each `<loc>`.
fn build_sitemap<'a>(paths: impl IntoIterator<Item = &'a str>, site_url: Option<&str>) -> String {
let base = site_url.map(|u| u.trim_end_matches('/'));
let mut out = String::from("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
out.push_str("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");
for path in paths {
if !is_static_route(path) {
continue;
}
let loc = match base {
Some(base) => format!("{base}{path}"),
None => path.to_string(),
};
out.push_str(" <url><loc>");
escape_xml(&loc, &mut out);
out.push_str("</loc></url>\n");
}
out.push_str("</urlset>\n");
out
}
/// Escape the XML-significant characters for character data inside a `<loc>`.
fn escape_xml(s: &str, out: &mut String) {
for c in s.chars() {
match c {
'&' => out.push_str("&"),
'<' => out.push_str("<"),
'>' => out.push_str(">"),
c => out.push(c),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sitemap_includes_static_and_excludes_dynamic() {
let xml = build_sitemap(
["/", "/components", "/posts/:id", "/files/*path"],
Some("https://example.com"),
);
assert!(xml.starts_with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"));
assert!(xml.contains("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">"));
assert!(xml.contains("<loc>https://example.com/</loc>"));
assert!(xml.contains("<loc>https://example.com/components</loc>"));
// Dynamic routes must be skipped entirely.
assert!(!xml.contains(":id"));
assert!(!xml.contains("*path"));
assert!(xml.trim_end().ends_with("</urlset>"));
}
#[test]
fn sitemap_uses_relative_loc_without_site_url() {
let xml = build_sitemap(["/", "/components"], None);
assert!(xml.contains("<loc>/</loc>"));
assert!(xml.contains("<loc>/components</loc>"));
}
#[test]
fn sitemap_escapes_xml_special_chars() {
let xml = build_sitemap(["/search?q=a&b<c>d"], None);
assert!(xml.contains("<loc>/search?q=a&b<c>d</loc>"));
// The raw, unescaped form must not leak through.
assert!(!xml.contains("a&b<c>d"));
}
#[test]
fn robots_points_to_absolute_sitemap() {
let txt = build_robots(Some("https://example.com/"));
assert!(txt.contains("User-agent: *"));
assert!(txt.contains("Allow: /"));
assert!(txt.contains("Sitemap: https://example.com/sitemap.xml"));
}
#[test]
fn robots_falls_back_to_relative_sitemap() {
let txt = build_robots(None);
assert!(txt.contains("Sitemap: /sitemap.xml"));
}
#[test]
fn static_route_predicate() {
assert!(is_static_route("/components"));
assert!(!is_static_route("/posts/:id"));
assert!(!is_static_route("/files/*path"));
}
}