Menu
Public repository / master
AkurAI-Framework / crates/cli/src/seo.rs
Latest change 61d696907a4d9613ca203986774efb4c822c75ec - seo: sitemap_from lists a dynamic route's pages from its data file (slug-safe values only) by Ólafur Búi Ólafsson
//! SEO essentials: `robots.txt`, `sitemap.xml`, and per-route `<head>` metadata.
//!
//! All three are derived from declarative project files, never compiled in:
//! - `backend/routes.json` supplies the URL set for the sitemap, plus optional
//! per-route `title`/`description` for `<head>` metadata.
//! - `backend/page.json` supplies an optional `"site_url"` string. When present,
//! sitemap `<loc>`s and the robots `Sitemap:` line become absolute URLs;
//! when absent they fall back to root-relative paths.
//!
//! Only STATIC routes go in the sitemap — patterns with `:param` or `*wildcard`
//! need data we don't have here, so they are skipped. XML special characters in
//! `<loc>` values are escaped so the output is always well-formed.
use crate::cmd_serve;
use akurai_http::Response;
use akurai_json::Value;
use std::path::Path;
/// `GET /robots.txt` — allow all crawling and point at the sitemap.
pub fn robots(root: &Path) -> Response {
let site = site_url(root);
Response::new(200).with_body(
"text/plain; charset=utf-8",
build_robots(site.as_deref()).into_bytes(),
)
}
/// `GET /sitemap.xml` — a urlset built from the static routes in `routes.json`.
/// In path-prefix locale mode each route is listed per locale with
/// `xhtml:link` alternates.
pub fn sitemap(root: &Path, i18n: Option<&crate::i18n::I18nState>) -> Response {
let site = site_url(root);
let paths = route_paths(root);
let loc = match i18n {
Some(s) if crate::routes::locale_prefix(root) => Some((s.locales(), s.default_locale())),
_ => None,
};
Response::new(200).with_body(
"application/xml; charset=utf-8",
build_sitemap_i18n(paths.iter().map(String::as_str), site.as_deref(), loc).into_bytes(),
)
}
/// Split a leading non-default locale segment off `path`. Returns the locale
/// (if the first segment is a known non-default locale) and the remaining path.
pub fn strip_locale_prefix<'a>(
path: &'a str,
locales: &[String],
default: &str,
) -> (Option<String>, &'a str) {
let rest = path.strip_prefix('/').unwrap_or(path);
let (seg, tail) = match rest.split_once('/') {
Some((a, b)) => (a, Some(b)),
None => (rest, None),
};
if !seg.eq_ignore_ascii_case(default) {
if let Some(l) = locales.iter().find(|l| l.eq_ignore_ascii_case(seg)) {
let rem = match tail {
Some(_) => &path[seg.len() + 1..],
None => "/",
};
return (Some(l.clone()), if rem.is_empty() { "/" } else { rem });
}
}
(None, path)
}
/// The URL path for `path` in `locale` (default locale is unprefixed).
pub fn localized_path(path: &str, locale: &str, default: &str) -> String {
if locale == default {
path.to_string()
} else if path == "/" {
format!("/{locale}")
} else {
format!("/{locale}{path}")
}
}
/// `(hreflang, href)` pairs for every locale plus `x-default` (the default
/// locale's URL). `href` is absolute when `site_url` is set.
pub fn alternates(
path: &str,
locales: &[String],
default: &str,
site_url: Option<&str>,
) -> Vec<(String, String)> {
let base = site_url.map(|u| u.trim_end_matches('/')).unwrap_or("");
let mut out: Vec<(String, String)> = locales
.iter()
.map(|l| {
(
l.clone(),
format!("{base}{}", localized_path(path, l, default)),
)
})
.collect();
out.push(("x-default".into(), format!("{base}{path}")));
out
}
/// `<link rel="alternate" hreflang>` tags for the `<head>`.
pub fn hreflang_html(alts: &[(String, String)]) -> String {
let mut out = String::new();
for (lang, href) in alts {
out.push_str("<link rel=\"alternate\" hreflang=\"");
escape_attr(lang, &mut out);
out.push_str("\" href=\"");
escape_attr(href, &mut out);
out.push_str("\" />\n");
}
out
}
/// Render-context extras for path-prefix mode: `hreflang_links` (raw HTML) for
/// the canonical (unprefixed) route path.
pub fn hreflang_extras(
root: &Path,
path: &str,
locales: &[String],
default: &str,
) -> Vec<(String, Value)> {
let alts = alternates(path, locales, default, site_url(root).as_deref());
vec![("hreflang_links".into(), Value::Str(hreflang_html(&alts)))]
}
fn escape_attr(s: &str, out: &mut String) {
for c in s.chars() {
match c {
'&' => out.push_str("&"),
'<' => out.push_str("<"),
'>' => out.push_str(">"),
'"' => out.push_str("""),
c => out.push(c),
}
}
}
/// Extra render-context pairs (`meta_title`/`meta_description`) for the route
/// matching `path`, or an empty vec when the route carries no metadata. Routes
/// without these fields contribute nothing, so behaviour is unchanged for them.
pub fn route_meta(root: &Path, path: &str) -> Vec<(String, Value)> {
let mut out = Vec::new();
let routes = crate::routes::entries(root);
// Match with the same router semantics used for page routing, so
// `:param`/`*wildcard` routes resolve consistently.
let mut router = akurai_router::Router::new();
for route in routes {
router.route(
&route.path,
(route.def.title.clone(), route.def.description.clone()),
);
}
if let Some(m) = router.match_path(path) {
let (title, desc) = &m.value;
if let Some(t) = title {
out.push(("meta_title".into(), Value::Str(t.clone())));
}
if let Some(d) = desc {
out.push(("meta_description".into(), Value::Str(d.clone())));
}
}
out
}
/// Read the optional `"site_url"` string from `backend/page.json`.
fn site_url(root: &Path) -> Option<String> {
cmd_serve::load_context(root)
.get("site_url")
.and_then(Value::as_str)
.map(|s| s.to_string())
}
/// All declared route patterns (filtering to static ones happens in
/// [`build_sitemap`], the single source of truth for that rule).
fn route_paths(root: &Path) -> Vec<String> {
let mut paths: Vec<String> = crate::routes::entries(root)
.into_iter()
.map(|route| route.path)
.collect();
if let Some(text) = root
.parent()
.and_then(|p| std::fs::read_to_string(p.join("backend/routes.json")).ok())
{
let backend = root.parent().map(|p| p.join("backend")).unwrap_or_default();
paths.extend(expanded_paths(&text, &backend));
}
paths
}
/// Concrete URLs for dynamic routes that opt in with `"sitemap_from": "<key>"`:
/// the route's single `:param` is filled with each string of the array at `<key>`
/// in its `data` file (relative to `backend/`). Values that are not plain URL
/// slugs are skipped, so a data file can never inject other paths.
fn expanded_paths(routes_json: &str, backend: &Path) -> Vec<String> {
let Ok(table) = akurai_json::parse(routes_json) else {
return Vec::new();
};
let routes = match table.get("routes") {
Some(Value::Array(items)) => items.clone(),
_ => match table {
Value::Array(items) => items,
_ => return Vec::new(),
},
};
let mut out = Vec::new();
for route in &routes {
let (Some(pattern), Some(key), Some(data)) = (
route.get("path").and_then(Value::as_str),
route.get("sitemap_from").and_then(Value::as_str),
route.get("data").and_then(Value::as_str),
) else {
continue;
};
let params: Vec<&str> = pattern.split('/').filter(|s| s.starts_with(':')).collect();
if params.len() != 1 || pattern.contains('*') {
continue;
}
let Some(values) = std::fs::read_to_string(backend.join(data))
.ok()
.and_then(|t| akurai_json::parse(&t).ok())
else {
continue;
};
if let Some(Value::Array(items)) = values.get(key) {
for slug in items.iter().filter_map(Value::as_str) {
if !slug.is_empty()
&& slug
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')
{
out.push(pattern.replacen(params[0], slug, 1));
}
}
}
}
out
}
/// A route is sitemap-eligible only if it is fully static — no `:param` and no
/// `*wildcard` segments, which would need data we don't have at this layer.
fn is_static_route(path: &str) -> bool {
!path.contains(':') && !path.contains('*')
}
/// Build a `robots.txt` body that allows all crawling and points at the
/// sitemap (absolute when `site_url` is set, root-relative otherwise).
fn build_robots(site_url: Option<&str>) -> String {
let sitemap = match site_url.map(|u| u.trim_end_matches('/')) {
Some(base) => format!("{base}/sitemap.xml"),
None => "/sitemap.xml".to_string(),
};
format!("User-agent: *\nAllow: /\n\nSitemap: {sitemap}\n")
}
/// Build a well-formed sitemap urlset from the given paths, skipping dynamic
/// routes and escaping XML special characters in each `<loc>`.
#[cfg(test)]
fn build_sitemap<'a>(paths: impl IntoIterator<Item = &'a str>, site_url: Option<&str>) -> String {
build_sitemap_i18n(paths, site_url, None)
}
/// Like [`build_sitemap`]; with `(locales, default)` every route is emitted once
/// per locale, each entry carrying `xhtml:link` alternates (incl. `x-default`).
fn build_sitemap_i18n<'a>(
paths: impl IntoIterator<Item = &'a str>,
site_url: Option<&str>,
locales: Option<(&[String], &str)>,
) -> String {
let base = site_url.map(|u| u.trim_end_matches('/'));
let mut out = String::from("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
out.push_str("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"");
if locales.is_some() {
out.push_str(" xmlns:xhtml=\"http://www.w3.org/1999/xhtml\"");
}
out.push_str(">\n");
for path in paths {
if !is_static_route(path) {
continue;
}
if let Some((locs, default)) = locales {
let alts = alternates(path, locs, default, site_url);
for l in locs {
let mut loc = String::new();
let href = alts.iter().find(|(k, _)| k == l).map(|(_, h)| h.as_str());
escape_xml(href.unwrap_or(path), &mut loc);
out.push_str(&format!(" <url><loc>{loc}</loc>"));
for (lang, href) in &alts {
out.push_str("<xhtml:link rel=\"alternate\" hreflang=\"");
escape_attr(lang, &mut out);
out.push_str("\" href=\"");
escape_attr(href, &mut out);
out.push_str("\"/>");
}
out.push_str("</url>\n");
}
continue;
}
let loc = match base {
Some(base) => format!("{base}{path}"),
None => path.to_string(),
};
out.push_str(" <url><loc>");
escape_xml(&loc, &mut out);
out.push_str("</loc></url>\n");
}
out.push_str("</urlset>\n");
out
}
/// Escape the XML-significant characters for character data inside a `<loc>`.
fn escape_xml(s: &str, out: &mut String) {
for c in s.chars() {
match c {
'&' => out.push_str("&"),
'<' => out.push_str("<"),
'>' => out.push_str(">"),
c => out.push(c),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sitemap_from_expands_dynamic_routes_from_data() {
let dir = std::env::temp_dir().join(format!("akurai-seo-{}", std::process::id()));
std::fs::create_dir_all(dir.join("data")).unwrap();
std::fs::write(
dir.join("data/learn.json"),
r#"{"slugs_is":["hvad-er-rag","ordasafn-api","../evil","a b"]}"#,
)
.unwrap();
let routes = r#"{"routes":[
{"path":"/learn/:slug","template":"post","data":"data/learn.json","sitemap_from":"slugs_is"},
{"path":"/news/:slug","template":"post","data":"data/news.json"},
{"path":"/missing/:slug","template":"post","data":"data/none.json","sitemap_from":"slugs"}]}"#;
let paths = expanded_paths(routes, &dir);
std::fs::remove_dir_all(&dir).ok();
assert_eq!(paths, vec!["/learn/hvad-er-rag", "/learn/ordasafn-api"]);
}
#[test]
fn sitemap_includes_static_and_excludes_dynamic() {
let xml = build_sitemap(
["/", "/components", "/posts/:id", "/files/*path"],
Some("https://example.com"),
);
assert!(xml.starts_with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"));
assert!(xml.contains("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">"));
assert!(xml.contains("<loc>https://example.com/</loc>"));
assert!(xml.contains("<loc>https://example.com/components</loc>"));
// Dynamic routes must be skipped entirely.
assert!(!xml.contains(":id"));
assert!(!xml.contains("*path"));
assert!(xml.trim_end().ends_with("</urlset>"));
}
#[test]
fn sitemap_uses_relative_loc_without_site_url() {
let xml = build_sitemap(["/", "/components"], None);
assert!(xml.contains("<loc>/</loc>"));
assert!(xml.contains("<loc>/components</loc>"));
}
#[test]
fn sitemap_escapes_xml_special_chars() {
let xml = build_sitemap(["/search?q=a&b<c>d"], None);
assert!(xml.contains("<loc>/search?q=a&b<c>d</loc>"));
// The raw, unescaped form must not leak through.
assert!(!xml.contains("a&b<c>d"));
}
#[test]
fn strip_prefix_cases() {
let l = vec!["en".to_string(), "is".to_string()];
assert_eq!(
strip_locale_prefix("/en", &l, "is"),
(Some("en".into()), "/")
);
assert_eq!(
strip_locale_prefix("/en/docs", &l, "is"),
(Some("en".into()), "/docs")
);
assert_eq!(
strip_locale_prefix("/is/docs", &l, "is"),
(None, "/is/docs")
);
assert_eq!(
strip_locale_prefix("/english", &l, "is"),
(None, "/english")
);
assert_eq!(strip_locale_prefix("/", &l, "is"), (None, "/"));
}
#[test]
fn localized_paths() {
assert_eq!(localized_path("/", "is", "is"), "/");
assert_eq!(localized_path("/", "en", "is"), "/en");
assert_eq!(localized_path("/a", "en", "is"), "/en/a");
}
#[test]
fn hreflang_tags_include_x_default() {
let l = vec!["en".to_string(), "is".to_string()];
let html = hreflang_html(&alternates("/a", &l, "is", Some("https://x.is/")));
assert!(html.contains("hreflang=\"en\" href=\"https://x.is/en/a\""));
assert!(html.contains("hreflang=\"is\" href=\"https://x.is/a\""));
assert!(html.contains("hreflang=\"x-default\" href=\"https://x.is/a\""));
}
#[test]
fn sitemap_has_xhtml_alternates() {
let l = vec!["en".to_string(), "is".to_string()];
let xml = build_sitemap_i18n(["/", "/p/:id"], Some("https://x.is"), Some((&l, "is")));
assert!(xml.contains("xmlns:xhtml=\"http://www.w3.org/1999/xhtml\""));
assert!(xml.contains("<loc>https://x.is/en</loc>"));
assert!(xml.contains("<loc>https://x.is/</loc>"));
assert!(xml
.contains("<xhtml:link rel=\"alternate\" hreflang=\"en\" href=\"https://x.is/en\"/>"));
assert!(!xml.contains(":id"));
}
#[test]
fn robots_points_to_absolute_sitemap() {
let txt = build_robots(Some("https://example.com/"));
assert!(txt.contains("User-agent: *"));
assert!(txt.contains("Allow: /"));
assert!(txt.contains("Sitemap: https://example.com/sitemap.xml"));
}
#[test]
fn robots_falls_back_to_relative_sitemap() {
let txt = build_robots(None);
assert!(txt.contains("Sitemap: /sitemap.xml"));
}
#[test]
fn static_route_predicate() {
assert!(is_static_route("/components"));
assert!(!is_static_route("/posts/:id"));
assert!(!is_static_route("/files/*path"));
}
}