Menu
AkurAI-Build
publicLatest change 05d42309ceb65d74053bad78370d08aba9614b62 - artifact: fix hot-path revalidation, .br labeling, and limit skew by Olafur Bui
use std::{
collections::BTreeMap,
fmt,
fs::{self, File, OpenOptions},
io::{Read, Write},
path::{Component, Path},
};
use anyhow::{Context, Result, ensure};
use serde::{Deserialize, Deserializer, Serialize, de::MapAccess};
use sha2::{Digest, Sha256};
const SCHEMA: &str = "bunfork-static-v1";
const MAX_FILES: usize = 50_000;
const MAX_FILE_BYTES: u64 = 256 * 1024 * 1024;
/// One bound shared by admission and in-memory serving: anything that
/// admits must also preload, so an admitted artifact can never fail to boot.
pub const MAX_TOTAL_BYTES: u64 = 512 * 1024 * 1024;
const MAX_MANIFEST_BYTES: u64 = 32 * 1024 * 1024;
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct Manifest {
pub schema: String,
pub root: String,
pub mode: Mode,
pub base: String,
pub fallback: Option<String>,
pub trailing_slash: TrailingSlash,
pub precompressed: bool,
pub immutable: bool,
#[serde(deserialize_with = "deserialize_files")]
pub files: BTreeMap<String, FileRecord>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct FileRecord {
pub sha256: String,
pub bytes: u64,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Mode {
Mpa,
Spa,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TrailingSlash {
Ignore,
Redirect,
Directory,
}
#[derive(Clone, Debug)]
pub struct AdmitOptions {
pub mode: Mode,
pub base: String,
pub fallback: Option<String>,
pub trailing_slash: TrailingSlash,
}
impl Default for AdmitOptions {
fn default() -> Self {
Self {
mode: Mode::Mpa,
base: "/".to_owned(),
fallback: None,
trailing_slash: TrailingSlash::Directory,
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum CacheClass {
Document,
Asset,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum Resolution {
File {
relative_path: String,
status: u16,
cache: CacheClass,
},
Fallback {
relative_path: String,
status: u16,
cache: CacheClass,
},
Redirect {
location: String,
status: u16,
},
NotFound {
status: u16,
},
}
pub fn admit(root: &Path, manifest_path: &Path, options: AdmitOptions) -> Result<Manifest> {
validate_root(root)?;
ensure_manifest_outside_root(root, manifest_path)?;
let files = scan(root, None, true)?;
let manifest = Manifest {
schema: SCHEMA.to_owned(),
root: ".".to_owned(),
mode: options.mode,
base: normalize_base(&options.base)?,
fallback: options.fallback,
trailing_slash: options.trailing_slash,
precompressed: true,
immutable: true,
files,
};
validate_manifest(&manifest)?;
write_manifest(manifest_path, &manifest)?;
verify(root, &manifest)?;
Ok(manifest)
}
pub fn read_manifest(path: &Path) -> Result<Manifest> {
let metadata = fs::symlink_metadata(path)
.with_context(|| format!("inspect static manifest {}", path.display()))?;
ensure!(
metadata.is_file() && !metadata.file_type().is_symlink(),
"static manifest must be a regular file"
);
ensure!(
metadata.len() <= MAX_MANIFEST_BYTES,
"static manifest is too large"
);
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
ensure!(
metadata.nlink() == 1,
"static manifest cannot be hard-linked"
);
}
let mut file = open_nofollow(path)?;
let mut bytes = Vec::with_capacity(usize::try_from(metadata.len())?);
file.read_to_end(&mut bytes)?;
let manifest: Manifest = serde_json::from_slice(&bytes).context("parse static manifest")?;
validate_manifest(&manifest)?;
Ok(manifest)
}
pub fn verify(root: &Path, manifest: &Manifest) -> Result<()> {
validate_root(root)?;
validate_manifest(manifest)?;
let actual = scan(root, None, true)?;
ensure!(
actual == manifest.files,
"static artifact does not match its manifest"
);
Ok(())
}
/// Inventory a Bunfork-owned bundle while excluding its self-describing manifest.
pub fn inventory_excluding(
root: &Path,
excluded_relative_path: &str,
) -> Result<BTreeMap<String, FileRecord>> {
validate_root(root)?;
validate_relative_path(excluded_relative_path)?;
scan(root, Some(excluded_relative_path), true)
}
/// Inventory an immutable file tree using the same admission rules and bounds.
pub fn inventory(root: &Path) -> Result<BTreeMap<String, FileRecord>> {
validate_root(root)?;
scan(root, None, false)
}
/// Resolve a request path against an already-validated manifest.
///
/// Callers must obtain the manifest from `read_manifest`, `admit`, or
/// `verify`, all of which enforce every schema invariant. Revalidating
/// here would rescan every file record on each request.
pub fn resolve(manifest: &Manifest, request_path: &str) -> Result<Resolution> {
let decoded = decode_request_path(request_path)?;
let base = manifest.base.as_str();
let relative = if base == "/" {
decoded.trim_start_matches('/').to_owned()
} else if decoded == base.trim_end_matches('/') {
return Ok(Resolution::Redirect {
location: base.to_owned(),
status: 308,
});
} else if let Some(relative) = decoded.strip_prefix(base) {
relative.to_owned()
} else {
return Ok(Resolution::NotFound { status: 404 });
};
if relative.is_empty() {
return file_resolution(manifest, "index.html", 200)
.map(|value| value.unwrap_or(Resolution::NotFound { status: 404 }));
}
if manifest.files.contains_key(&relative) {
return Ok(file_resolution(manifest, &relative, 200)?
.expect("file existence checked before resolution"));
}
if relative.ends_with('/') {
let index = format!("{relative}index.html");
if let Some(resolution) = file_resolution(manifest, &index, 200)? {
return Ok(resolution);
}
} else {
let html = format!("{relative}.html");
if manifest.files.contains_key(&html) {
return Ok(file_resolution(manifest, &html, 200)?
.expect("file existence checked before resolution"));
}
let index = format!("{relative}/index.html");
if manifest.files.contains_key(&index) {
return match manifest.trailing_slash {
TrailingSlash::Ignore => Ok(file_resolution(manifest, &index, 200)?
.expect("file existence checked before resolution")),
TrailingSlash::Redirect | TrailingSlash::Directory => Ok(Resolution::Redirect {
location: format!("{}{relative}/", manifest.base),
status: 308,
}),
};
}
}
if manifest.mode == Mode::Spa
&& is_navigation_path(&relative)
&& let Some(fallback) = manifest.fallback.as_deref()
{
return Ok(Resolution::Fallback {
relative_path: fallback.to_owned(),
status: if fallback == "404.html" { 404 } else { 200 },
cache: CacheClass::Document,
});
}
if manifest.mode == Mode::Mpa && manifest.files.contains_key("404.html") {
return Ok(Resolution::File {
relative_path: "404.html".to_owned(),
status: 404,
cache: CacheClass::Document,
});
}
Ok(Resolution::NotFound { status: 404 })
}
fn validate_manifest(manifest: &Manifest) -> Result<()> {
ensure!(
manifest.schema == SCHEMA,
"unsupported static manifest schema"
);
ensure!(manifest.root == ".", "static manifest root must be '.'");
ensure!(manifest.precompressed, "precompressed must be true in v1");
ensure!(manifest.immutable, "immutable must be true in v1");
ensure!(
manifest.base == normalize_base(&manifest.base)?,
"invalid base"
);
ensure!(!manifest.files.is_empty(), "static artifact has no files");
ensure!(
manifest.files.len() <= MAX_FILES,
"static artifact has too many files"
);
let mut total = 0_u64;
for (path, record) in &manifest.files {
validate_relative_path(path)?;
validate_digest(&record.sha256)?;
ensure!(
record.bytes <= MAX_FILE_BYTES,
"artifact file is too large: {path}"
);
total = total
.checked_add(record.bytes)
.context("artifact size overflow")?;
}
ensure!(
total <= MAX_TOTAL_BYTES,
"static artifact exceeds the 512 MiB total limit"
);
match manifest.mode {
Mode::Mpa => ensure!(manifest.fallback.is_none(), "MPA mode forbids a fallback"),
Mode::Spa => {
let fallback = manifest
.fallback
.as_deref()
.context("SPA mode requires a fallback")?;
validate_relative_path(fallback)?;
ensure!(fallback.ends_with(".html"), "SPA fallback must be HTML");
ensure!(
manifest.files.contains_key(fallback),
"SPA fallback is missing"
);
}
}
Ok(())
}
fn validate_root(root: &Path) -> Result<()> {
let metadata = fs::symlink_metadata(root)
.with_context(|| format!("inspect static artifact root {}", root.display()))?;
ensure!(
metadata.is_dir() && !metadata.file_type().is_symlink(),
"static artifact root must be a real directory"
);
Ok(())
}
fn ensure_manifest_outside_root(root: &Path, manifest: &Path) -> Result<()> {
let root = fs::canonicalize(root)?;
let parent = manifest
.parent()
.filter(|parent| !parent.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."));
let parent = fs::canonicalize(parent)
.with_context(|| format!("resolve manifest parent {}", parent.display()))?;
let absolute = parent.join(
manifest
.file_name()
.context("manifest path has no file name")?,
);
ensure!(
!absolute.starts_with(root),
"static manifest must be outside the artifact root"
);
Ok(())
}
fn normalize_base(base: &str) -> Result<String> {
ensure!(base.starts_with('/'), "base must start with '/'");
ensure!(
!base.contains(['?', '#', '\\', '\0']),
"base contains unsafe characters"
);
ensure!(!base.contains("//"), "base contains an empty segment");
for segment in base
.trim_matches('/')
.split('/')
.filter(|value| !value.is_empty())
{
ensure!(
segment != "." && segment != "..",
"base contains a dot segment"
);
ensure!(!segment.starts_with('.'), "base contains a hidden segment");
}
if base == "/" {
Ok("/".to_owned())
} else {
Ok(format!("{}/", base.trim_end_matches('/')))
}
}
fn scan(
root: &Path,
excluded_relative_path: Option<&str>,
require_files: bool,
) -> Result<BTreeMap<String, FileRecord>> {
let mut files = BTreeMap::new();
let mut stack = vec![root.to_path_buf()];
let mut total = 0_u64;
while let Some(directory) = stack.pop() {
let mut entries = fs::read_dir(&directory)
.with_context(|| format!("scan artifact directory {}", directory.display()))?
.collect::<std::io::Result<Vec<_>>>()?;
entries.sort_by_key(|entry| entry.file_name());
for entry in entries.into_iter().rev() {
let path = entry.path();
let file_type = entry.file_type()?;
ensure!(!file_type.is_symlink(), "artifact cannot contain symlinks");
let relative = path
.strip_prefix(root)
.context("artifact path escaped root")?;
let relative = relative_path(relative)?;
if excluded_relative_path == Some(relative.as_str()) {
continue;
}
validate_relative_path(&relative)?;
if file_type.is_dir() {
stack.push(path);
continue;
}
ensure!(
file_type.is_file(),
"artifact contains a special file: {relative}"
);
ensure!(
files.len() < MAX_FILES,
"static artifact has too many files"
);
let record = hash_file(&path, &relative)?;
total = total
.checked_add(record.bytes)
.context("artifact size overflow")?;
ensure!(
total <= MAX_TOTAL_BYTES,
"static artifact exceeds the 512 MiB total limit"
);
ensure!(
files.insert(relative, record).is_none(),
"duplicate artifact path"
);
}
}
if require_files {
ensure!(!files.is_empty(), "static artifact has no files");
}
Ok(files)
}
fn hash_file(path: &Path, label: &str) -> Result<FileRecord> {
let mut file = open_nofollow(path)?;
let metadata = file.metadata()?;
ensure!(
metadata.is_file(),
"artifact entry is not a regular file: {label}"
);
ensure!(
metadata.len() <= MAX_FILE_BYTES,
"artifact file is too large: {label}"
);
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
ensure!(
metadata.nlink() == 1,
"artifact files cannot be hard-linked"
);
}
let mut digest = Sha256::new();
let mut bytes = 0_u64;
let mut buffer = [0_u8; 64 * 1024];
loop {
let read = file.read(&mut buffer)?;
if read == 0 {
break;
}
digest.update(&buffer[..read]);
bytes = bytes
.checked_add(u64::try_from(read)?)
.context("file size overflow")?;
}
ensure!(
bytes == metadata.len(),
"artifact file changed while hashing: {label}"
);
ensure_path_identity(path, &file, label)?;
Ok(FileRecord {
sha256: format!("{:x}", digest.finalize()),
bytes,
})
}
fn open_nofollow(path: &Path) -> Result<File> {
let mut options = OpenOptions::new();
options.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC);
}
options
.open(path)
.with_context(|| format!("open artifact file {}", path.display()))
}
fn ensure_path_identity(path: &Path, file: &File, label: &str) -> Result<()> {
let path_metadata = fs::symlink_metadata(path)?;
ensure!(
path_metadata.is_file() && !path_metadata.file_type().is_symlink(),
"artifact file changed while hashing: {label}"
);
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let file_metadata = file.metadata()?;
ensure!(
path_metadata.dev() == file_metadata.dev()
&& path_metadata.ino() == file_metadata.ino(),
"artifact file changed while hashing: {label}"
);
}
Ok(())
}
fn validate_relative_path(path: &str) -> Result<()> {
ensure!(!path.is_empty(), "artifact path cannot be empty");
ensure!(
!path.starts_with('/') && !path.contains('\\') && !path.contains('\0'),
"unsafe artifact path"
);
for component in Path::new(path).components() {
let Component::Normal(value) = component else {
anyhow::bail!("unsafe artifact path: {path}");
};
let value = value.to_str().context("artifact paths must be UTF-8")?;
ensure!(
!value.starts_with('.'),
"artifact path contains a hidden segment: {path}"
);
let lower = value.to_ascii_lowercase();
ensure!(
!matches!(lower.as_str(), "data" | "backups")
&& !lower.ends_with(".db")
&& !lower.ends_with(".sqlite")
&& !lower.ends_with(".sqlite3")
&& !lower.ends_with(".backup")
&& !lower.ends_with(".key")
&& !lower.ends_with(".token")
&& !lower.ends_with(".pem"),
"artifact path looks sensitive: {path}"
);
}
Ok(())
}
fn relative_path(path: &Path) -> Result<String> {
let mut parts = Vec::new();
for component in path.components() {
let Component::Normal(value) = component else {
anyhow::bail!("unsafe artifact path");
};
parts.push(value.to_str().context("artifact paths must be UTF-8")?);
}
Ok(parts.join("/"))
}
fn validate_digest(value: &str) -> Result<()> {
ensure!(
value.len() == 64
&& value
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()),
"invalid lowercase SHA-256 digest"
);
Ok(())
}
fn write_manifest(path: &Path, manifest: &Manifest) -> Result<()> {
let parent = path
.parent()
.filter(|value| !value.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."));
ensure!(parent.is_dir(), "manifest parent does not exist");
let temporary = parent.join(format!(".bunfork-manifest-{}.tmp", std::process::id()));
let mut options = OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o644);
}
let mut file = options
.open(&temporary)
.context("create temporary manifest")?;
let result = (|| -> Result<()> {
let mut bytes = serde_json::to_vec_pretty(manifest)?;
bytes.push(b'\n');
file.write_all(&bytes)?;
file.sync_all()?;
fs::rename(&temporary, path).context("publish static manifest")?;
sync_directory(parent)
})();
if result.is_err() {
let _ = fs::remove_file(&temporary);
}
result
}
fn sync_directory(path: &Path) -> Result<()> {
#[cfg(unix)]
File::open(path)?.sync_all()?;
Ok(())
}
fn file_resolution(manifest: &Manifest, path: &str, status: u16) -> Result<Option<Resolution>> {
if !manifest.files.contains_key(path) {
return Ok(None);
}
Ok(Some(Resolution::File {
relative_path: path.to_owned(),
status,
cache: cache_class(path),
}))
}
fn cache_class(path: &str) -> CacheClass {
if path.ends_with(".html") {
CacheClass::Document
} else {
CacheClass::Asset
}
}
fn decode_request_path(path: &str) -> Result<String> {
ensure!(path.starts_with('/'), "request path must start with '/'");
ensure!(!path.contains(['\\', '\0']), "unsafe request path");
let mut decoded = String::new();
for (index, raw) in path.split('/').enumerate() {
if index > 0 {
decoded.push('/');
}
validate_percent_encoding(raw)?;
let segment = percent_encoding::percent_decode_str(raw)
.decode_utf8()
.context("request path is not valid UTF-8")?;
ensure!(
segment != "."
&& segment != ".."
&& !segment.contains(['/', '\\', '\0'])
&& !segment.starts_with('.'),
"unsafe request path segment"
);
decoded.push_str(&segment);
}
Ok(decoded)
}
fn validate_percent_encoding(value: &str) -> Result<()> {
let bytes = value.as_bytes();
for index in 0..bytes.len() {
if bytes[index] == b'%' {
ensure!(
index + 2 < bytes.len()
&& bytes[index + 1].is_ascii_hexdigit()
&& bytes[index + 2].is_ascii_hexdigit(),
"malformed percent encoding"
);
}
}
Ok(())
}
fn is_navigation_path(path: &str) -> bool {
![
"api/",
"assets/",
"static/",
"_bunfork/",
"_next/",
"_nuxt/",
"_app/",
]
.iter()
.any(|prefix| path.starts_with(prefix))
&& !path
.rsplit('/')
.next()
.is_some_and(|segment| segment.contains('.'))
}
fn deserialize_files<'de, D>(
deserializer: D,
) -> std::result::Result<BTreeMap<String, FileRecord>, D::Error>
where
D: Deserializer<'de>,
{
struct FilesVisitor;
impl<'de> serde::de::Visitor<'de> for FilesVisitor {
type Value = BTreeMap<String, FileRecord>;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("a map of unique artifact paths")
}
fn visit_map<A>(self, mut map: A) -> std::result::Result<Self::Value, A::Error>
where
A: MapAccess<'de>,
{
let mut files = BTreeMap::new();
while let Some((path, record)) = map.next_entry()? {
if files.insert(path, record).is_some() {
return Err(serde::de::Error::custom("duplicate artifact path"));
}
}
Ok(files)
}
}
deserializer.deserialize_map(FilesVisitor)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn admission_is_deterministic_and_detects_mutation() -> Result<()> {
let directory = tempfile::tempdir()?;
let site = directory.path().join("site");
fs::create_dir(&site)?;
fs::write(site.join("index.html"), "hello")?;
let manifest_path = directory.path().join("manifest.json");
let first = admit(&site, &manifest_path, AdmitOptions::default())?;
let first_bytes = fs::read(&manifest_path)?;
let second = admit(&site, &manifest_path, AdmitOptions::default())?;
assert_eq!(first, second);
assert_eq!(first_bytes, fs::read(&manifest_path)?);
fs::write(site.join("index.html"), "changed")?;
assert!(verify(&site, &first).is_err());
Ok(())
}
#[test]
fn resolver_handles_mpa_spa_base_and_traversal() -> Result<()> {
let files = ["index.html", "guide/index.html", "404.html"]
.into_iter()
.map(|path| {
(
path.to_owned(),
FileRecord {
sha256: "0".repeat(64),
bytes: 1,
},
)
})
.collect();
let manifest = Manifest {
schema: SCHEMA.to_owned(),
root: ".".to_owned(),
mode: Mode::Mpa,
base: "/docs/".to_owned(),
fallback: None,
trailing_slash: TrailingSlash::Directory,
precompressed: true,
immutable: true,
files,
};
assert!(matches!(
resolve(&manifest, "/docs/")?,
Resolution::File { .. }
));
assert!(matches!(
resolve(&manifest, "/docs/guide")?,
Resolution::Redirect { .. }
));
assert!(matches!(
resolve(&manifest, "/")?,
Resolution::NotFound { .. }
));
assert!(resolve(&manifest, "/docs/%2e%2e/secret").is_err());
Ok(())
}
#[cfg(unix)]
#[test]
fn admission_rejects_links_and_hidden_files() -> Result<()> {
use std::os::unix::fs::symlink;
let directory = tempfile::tempdir()?;
let site = directory.path().join("site");
fs::create_dir(&site)?;
fs::write(site.join("index.html"), "hello")?;
symlink(site.join("index.html"), site.join("leak.html"))?;
assert!(
admit(
&site,
&directory.path().join("manifest.json"),
AdmitOptions::default()
)
.is_err()
);
fs::remove_file(site.join("leak.html"))?;
fs::write(site.join(".env"), "SECRET=x")?;
assert!(
admit(
&site,
&directory.path().join("manifest.json"),
AdmitOptions::default()
)
.is_err()
);
Ok(())
}
}