Menu
Public repository / b9bf8c65e7cff19e76ae78f73b05d66f6a1b4729
AkurAI-Build / src/releases.rs
Latest change b9e3df61e6ca9f1a0e193b69ba7657b9ec4d3cc9 - OLI-273: tags and releases with attached artifacts by Óli
//! Tags and releases (OLI-273): tags come from the hosted repository, releases
//! are rows with notes and attached assets (copies of run artifacts under the
//! data root), served per repository.
use anyhow::{Context, Result, bail, ensure};
use rusqlite::{OptionalExtension, params};
use serde::Serialize;
use sha2::{Digest, Sha256};
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use crate::db::Database;
use crate::git_process;
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
pub struct Tag {
pub name: String,
/// The commit the tag points at (peeled for annotated tags).
pub sha: String,
pub annotated: bool,
pub subject: String,
pub created_at: i64,
}
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
pub struct Release {
pub id: i64,
pub repository: String,
pub tag: String,
pub name: String,
pub notes: String,
pub author: String,
pub created_at: i64,
pub assets: Vec<Asset>,
}
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
pub struct Asset {
pub id: i64,
pub name: String,
pub sha256: String,
pub bytes: i64,
pub created_at: i64,
}
pub struct NewRelease<'a> {
pub tag: &'a str,
pub name: Option<&'a str>,
pub notes: &'a str,
pub author: &'a str,
}
const MAX_NOTES: usize = 64 * 1024;
/// Tags of a bare repository, newest first.
pub fn tags(repository: &Path) -> Result<Vec<Tag>> {
let out = git_process::output(
repository,
[
"for-each-ref",
"--sort=-creatordate",
"--format=%(refname:short)%09%(objectname)%09%(*objectname)%09%(objecttype)%09%(creatordate:unix)%09%(contents:subject)",
"refs/tags",
],
)?;
Ok(out
.lines()
.filter_map(|line| {
let mut parts = line.splitn(6, '\t');
let name = parts.next()?.to_string();
let object = parts.next()?.to_string();
let peeled = parts.next()?.to_string();
let kind = parts.next()?;
let created_at = parts.next()?.trim().parse().unwrap_or(0);
let subject = parts.next().unwrap_or("").to_string();
let annotated = kind == "tag";
Some(Tag {
name,
sha: if annotated && !peeled.is_empty() {
peeled
} else {
object
},
annotated,
subject,
created_at,
})
})
.collect())
}
pub fn validate_tag(tag: &str) -> Result<()> {
ensure!(
!tag.is_empty()
&& tag.len() <= 200
&& !tag.starts_with('-')
&& !tag.contains("..")
&& tag
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-' | '/' | '+')),
"invalid tag name {tag:?}"
);
Ok(())
}
fn validate_asset_name(name: &str) -> Result<()> {
ensure!(
!name.is_empty()
&& name.len() <= 200
&& !name.starts_with('.')
&& name
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-' | '+')),
"invalid asset name {name:?}"
);
Ok(())
}
fn repository_id(connection: &rusqlite::Connection, name: &str) -> Result<i64> {
connection
.query_row(
"SELECT id FROM repositories WHERE name = ?1",
[name],
|row| row.get(0),
)
.optional()?
.with_context(|| format!("repository not found: {name}"))
}
fn assets_of(connection: &rusqlite::Connection, release_id: i64) -> Result<Vec<Asset>> {
let mut statement = connection.prepare(
"SELECT id, name, sha256, bytes, created_at FROM release_assets WHERE release_id = ?1 ORDER BY name",
)?;
let rows = statement.query_map([release_id], |row| {
Ok(Asset {
id: row.get(0)?,
name: row.get(1)?,
sha256: row.get(2)?,
bytes: row.get(3)?,
created_at: row.get(4)?,
})
})?;
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
}
const RELEASE_SELECT: &str = "SELECT x.id, r.name, x.tag, x.name, x.notes, x.author, x.created_at
FROM releases x JOIN repositories r ON r.id = x.repository_id";
fn map_release(row: &rusqlite::Row<'_>) -> rusqlite::Result<Release> {
Ok(Release {
id: row.get(0)?,
repository: row.get(1)?,
tag: row.get(2)?,
name: row.get(3)?,
notes: row.get(4)?,
author: row.get(5)?,
created_at: row.get(6)?,
assets: Vec::new(),
})
}
fn with_assets(connection: &rusqlite::Connection, mut release: Release) -> Result<Release> {
release.assets = assets_of(connection, release.id)?;
Ok(release)
}
pub fn create(database: &Database, repository: &str, new: &NewRelease<'_>) -> Result<Release> {
validate_tag(new.tag)?;
ensure!(new.notes.len() <= MAX_NOTES, "notes are too long");
ensure!(
!new.author.trim().is_empty() && new.author.len() <= 256,
"invalid author"
);
let name = new
.name
.map(str::trim)
.filter(|n| !n.is_empty())
.unwrap_or(new.tag);
ensure!(name.len() <= 200, "name is too long");
let connection = database.connection()?;
let id = repository_id(&connection, repository)?;
let inserted = connection.execute(
"INSERT OR IGNORE INTO releases (repository_id, tag, name, notes, author) VALUES (?1, ?2, ?3, ?4, ?5)",
params![id, new.tag, name, new.notes, new.author.trim()],
)?;
if inserted == 0 {
bail!("release {} already exists", new.tag);
}
let release = connection.query_row(
&format!("{RELEASE_SELECT} WHERE x.id = ?1"),
[connection.last_insert_rowid()],
map_release,
)?;
with_assets(&connection, release)
}
pub fn list(database: &Database, repository: &str) -> Result<Vec<Release>> {
let connection = database.connection()?;
let id = repository_id(&connection, repository)?;
let mut statement = connection.prepare(&format!(
"{RELEASE_SELECT} WHERE x.repository_id = ?1 ORDER BY x.created_at DESC, x.id DESC"
))?;
let releases: Vec<Release> = statement
.query_map([id], map_release)?
.collect::<std::result::Result<Vec<_>, _>>()?;
releases
.into_iter()
.map(|r| with_assets(&connection, r))
.collect()
}
pub fn get(database: &Database, repository: &str, tag: &str) -> Result<Option<Release>> {
let connection = database.connection()?;
let id = repository_id(&connection, repository)?;
let release = connection
.query_row(
&format!("{RELEASE_SELECT} WHERE x.repository_id = ?1 AND x.tag = ?2"),
params![id, tag],
map_release,
)
.optional()?;
release.map(|r| with_assets(&connection, r)).transpose()
}
pub fn remove(database: &Database, repository: &str, tag: &str) -> Result<Option<Vec<PathBuf>>> {
let connection = database.connection()?;
let id = repository_id(&connection, repository)?;
let release_id: Option<i64> = connection
.query_row(
"SELECT id FROM releases WHERE repository_id = ?1 AND tag = ?2",
params![id, tag],
|row| row.get(0),
)
.optional()?;
let Some(release_id) = release_id else {
return Ok(None);
};
let mut statement =
connection.prepare("SELECT path FROM release_assets WHERE release_id = ?1")?;
let paths: Vec<PathBuf> = statement
.query_map([release_id], |row| row.get::<_, String>(0))?
.collect::<std::result::Result<Vec<_>, _>>()?
.into_iter()
.map(PathBuf::from)
.collect();
connection.execute("DELETE FROM releases WHERE id = ?1", [release_id])?;
Ok(Some(paths))
}
fn hash_file(path: &Path) -> Result<(String, u64)> {
let mut file = fs::File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = [0u8; 64 * 1024];
let mut total = 0u64;
loop {
let read = file.read(&mut buffer)?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
total += read as u64;
}
let digest = hasher.finalize();
Ok((digest.iter().map(|b| format!("{b:02x}")).collect(), total))
}
/// Where release assets live: `<assets_root>/<repository>/<tag>/<name>`.
pub fn asset_dir(assets_root: &Path, repository: &str, tag: &str) -> PathBuf {
assets_root.join(repository).join(tag)
}
/// Copy a file (a run artifact) into the release and record it.
pub fn attach(
database: &Database,
assets_root: &Path,
repository: &str,
tag: &str,
name: &str,
source: &Path,
) -> Result<Asset> {
validate_asset_name(name)?;
let release =
get(database, repository, tag)?.with_context(|| format!("release {tag} not found"))?;
ensure!(
!release.assets.iter().any(|a| a.name == name),
"asset {name} is already attached to {tag}"
);
let dir = asset_dir(assets_root, repository, tag);
fs::create_dir_all(&dir)?;
let target = dir.join(name);
let temp = dir.join(format!("{name}.tmp"));
fs::copy(source, &temp)?;
fs::rename(&temp, &target)?;
let (sha256, bytes) = hash_file(&target)?;
let connection = database.connection()?;
connection.execute(
"INSERT INTO release_assets (release_id, name, path, sha256, bytes) VALUES (?1, ?2, ?3, ?4, ?5)",
params![release.id, name, target.to_string_lossy(), sha256, bytes as i64],
)?;
let id = connection.last_insert_rowid();
Ok(Asset {
id,
name: name.into(),
sha256,
bytes: bytes as i64,
created_at: connection.query_row(
"SELECT created_at FROM release_assets WHERE id = ?1",
[id],
|row| row.get(0),
)?,
})
}
/// The stored file of an asset, checked to still be under the assets root.
pub fn asset_path(
database: &Database,
assets_root: &Path,
repository: &str,
tag: &str,
name: &str,
) -> Result<Option<(Asset, PathBuf)>> {
let Some(release) = get(database, repository, tag)? else {
return Ok(None);
};
let Some(asset) = release.assets.into_iter().find(|a| a.name == name) else {
return Ok(None);
};
let connection = database.connection()?;
let path: String = connection.query_row(
"SELECT path FROM release_assets WHERE id = ?1",
[asset.id],
|row| row.get(0),
)?;
let path = PathBuf::from(path).canonicalize()?;
let root = assets_root.canonicalize()?;
ensure!(
path.starts_with(&root) && path.is_file(),
"asset path is invalid"
);
Ok(Some((asset, path)))
}
#[cfg(test)]
mod tests {
use super::*;
use std::process::Command;
const KEY: &str = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
fn database() -> Database {
let db = Database::memory(KEY).expect("db");
db.add_repository("demo", "https://example.com/demo.git", "main")
.expect("repo");
db
}
fn git_at(dir: &Path, date: &str, args: &[&str]) {
let status = Command::new("git")
.arg("-C")
.arg(dir)
.env("GIT_AUTHOR_DATE", date)
.env("GIT_COMMITTER_DATE", date)
.args(args)
.status()
.expect("git");
assert!(status.success(), "git {args:?}");
}
fn git(dir: &Path, args: &[&str]) {
git_at(dir, "2020-01-01T00:00:00+0000", args);
}
#[test]
fn tags_are_listed_newest_first_with_annotation_and_peeled_commit() {
let temp = tempfile::tempdir().expect("tempdir");
let work = temp.path().join("work");
fs::create_dir_all(&work).expect("work");
git(&work, &["init", "-q", "-b", "main"]);
git(&work, &["config", "user.name", "Ada"]);
git(&work, &["config", "user.email", "ada@example.invalid"]);
git(&work, &["commit", "-q", "--allow-empty", "-m", "one"]);
git(&work, &["tag", "v0.1.0"]);
git_at(
&work,
"2021-01-01T00:00:00+0000",
&["commit", "-q", "--allow-empty", "-m", "two"],
);
git_at(
&work,
"2021-06-01T00:00:00+0000",
&[
"-c",
"user.name=Ada",
"-c",
"user.email=a@b.c",
"tag",
"-a",
"v0.2.0",
"-m",
"Second release",
],
);
let listed = tags(&work).expect("tags");
assert_eq!(listed.len(), 2);
assert_eq!(listed[0].name, "v0.2.0");
assert!(listed[0].annotated);
assert_eq!(listed[0].subject, "Second release");
assert_eq!(listed[0].sha.len(), 40);
let head = git_process::output(&work, ["rev-parse", "HEAD"]).expect("head");
assert_eq!(
listed[0].sha,
head.trim(),
"annotated tags are peeled to the commit"
);
assert!(!listed[1].annotated);
assert_eq!(listed[1].name, "v0.1.0");
}
#[test]
fn releases_are_created_once_per_tag_listed_fetched_and_removed_with_their_assets() {
let db = database();
let temp = tempfile::tempdir().expect("tempdir");
let assets_root = temp.path().join("assets");
let release = create(
&db,
"demo",
&NewRelease {
tag: "v1.0.0",
name: None,
notes: "First.",
author: "ada",
},
)
.expect("create");
assert_eq!(
(release.name.as_str(), release.notes.as_str()),
("v1.0.0", "First.")
);
assert!(
create(
&db,
"demo",
&NewRelease {
tag: "v1.0.0",
name: None,
notes: "",
author: "ada"
}
)
.is_err()
);
assert!(
create(
&db,
"demo",
&NewRelease {
tag: "../x",
name: None,
notes: "",
author: "ada"
}
)
.is_err()
);
let source = temp.path().join("binary");
fs::write(&source, b"hello").expect("write");
let asset = attach(
&db,
&assets_root,
"demo",
"v1.0.0",
"akurai-linux-x86_64",
&source,
)
.expect("attach");
assert_eq!(asset.bytes, 5);
assert_eq!(
asset.sha256,
"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
);
assert!(
attach(
&db,
&assets_root,
"demo",
"v1.0.0",
"akurai-linux-x86_64",
&source
)
.is_err(),
"names are unique per release"
);
assert!(attach(&db, &assets_root, "demo", "v1.0.0", "../evil", &source).is_err());
let fetched = get(&db, "demo", "v1.0.0").expect("get").expect("release");
assert_eq!(fetched.assets.len(), 1);
let (_, path) = asset_path(&db, &assets_root, "demo", "v1.0.0", "akurai-linux-x86_64")
.expect("ok")
.expect("asset");
assert_eq!(fs::read(&path).expect("read"), b"hello");
assert!(
asset_path(&db, &assets_root, "demo", "v1.0.0", "nope")
.expect("ok")
.is_none()
);
assert_eq!(list(&db, "demo").expect("list").len(), 1);
let removed = remove(&db, "demo", "v1.0.0")
.expect("remove")
.expect("existed");
assert_eq!(removed.len(), 1);
assert!(get(&db, "demo", "v1.0.0").expect("get").is_none());
assert!(remove(&db, "demo", "v1.0.0").expect("remove").is_none());
}
}