AkurAI-Build / src/forks.rs

Latest change e8aa34a9be50d7d7ea72f66991ff82dfca7d23e1 - OLI-276: Git over SSH, forks and mirrors, code search through akurai-index, repository insights by Ólafur Búi Ólafsson

//! Forks and mirrors (OLI-276). A fork is a hosted copy of another hosted
//! repository that remembers its upstream and can fast-forward its default
//! branch from it. A mirror is a hosted copy of an external Git URL that pulls
//! every branch and tag, on demand and on a timer.

use std::{
    fs,
    path::{Path, PathBuf},
    process::Command,
};

use anyhow::{Context, Result, bail, ensure};
use rusqlite::params;
use serde::Serialize;

use crate::{
    config,
    db::{Database, Repository},
    hosted_git, protection,
};

#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
pub struct SyncOutcome {
    pub repository: String,
    pub upstream: String,
    /// The default branch moved (fork) or at least one ref changed (mirror).
    pub updated: bool,
    pub reason: String,
}

fn run(command: &mut Command, operation: &str) -> Result<()> {
    let output = command
        .env("GIT_TERMINAL_PROMPT", "0")
        .output()
        .with_context(|| format!("failed to start {operation}"))?;
    ensure!(
        output.status.success(),
        "{operation} failed: {}",
        String::from_utf8_lossy(&output.stderr).trim()
    );
    Ok(())
}

fn git_output(repository: &Path, arguments: &[&str]) -> Result<String> {
    let output = Command::new("git")
        .arg("-C")
        .arg(repository)
        .args(arguments)
        .env("GIT_TERMINAL_PROMPT", "0")
        .output()
        .context("failed to start git")?;
    ensure!(
        output.status.success(),
        "git {} failed: {}",
        arguments.join(" "),
        String::from_utf8_lossy(&output.stderr).trim()
    );
    Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned())
}

/// A bare copy at `hosted/<name>.git` with pushes and hooks enabled.
fn clone_bare(root: &Path, name: &str, source: &str, mirror: bool) -> Result<PathBuf> {
    let destination = hosted_git::repository_path(root, name)?;
    ensure!(
        !destination.exists(),
        "hosted repository already exists: {name}"
    );
    if let Some(parent) = destination.parent() {
        fs::create_dir_all(parent)?;
    }
    let mode = if mirror { "--mirror" } else { "--bare" };
    let cloned = run(
        Command::new("git")
            .args(["clone", mode, "--", source])
            .arg(&destination),
        "git clone",
    )
    .and_then(|()| {
        run(
            Command::new("git").arg("-C").arg(&destination).args([
                "config",
                "http.receivepack",
                "true",
            ]),
            "enable Git Smart HTTP writes",
        )
    })
    .and_then(|()| hosted_git::install_hooks(&destination).map(|_| ()));
    if let Err(error) = cloned {
        let _ = fs::remove_dir_all(&destination);
        return Err(error);
    }
    Ok(destination)
}

fn register(
    database: &Database,
    name: &str,
    path: &Path,
    branch: &str,
    fork_of: Option<&str>,
    mirror_of: Option<&str>,
) -> Result<Repository> {
    database.add_repository(
        name,
        path.to_str()
            .context("hosted repository path is not UTF-8")?,
        branch,
    )?;
    database.connection()?.execute(
        "UPDATE repositories SET fork_of = ?2, mirror_of = ?3 WHERE name = ?1",
        params![name, fork_of, mirror_of],
    )?;
    protection::set_rule(database, name, &protection::Rule::default_for(branch))?;
    database.repository(name)
}

/// Fork hosted `source` as hosted `name`.
pub fn fork(database: &Database, root: &Path, source: &str, name: &str) -> Result<Repository> {
    config::validate_repo_name(name)?;
    let upstream = database.repository(source)?;
    let upstream_path = hosted_git::repository_path(root, &upstream.name)?;
    ensure!(
        upstream_path.is_dir(),
        "{source} is not hosted on this forge"
    );
    if database.repository(name).is_ok() {
        bail!("a repository named {name} already exists");
    }
    let path = clone_bare(
        root,
        name,
        upstream_path.to_str().context("path is not UTF-8")?,
        false,
    )?;
    match register(
        database,
        name,
        &path,
        &upstream.default_branch,
        Some(&upstream.name),
        None,
    ) {
        Ok(repository) => Ok(repository),
        Err(error) => {
            let _ = fs::remove_dir_all(&path);
            Err(error)
        }
    }
}

/// Host a mirror of the external `url` as `name`.
pub fn mirror(
    database: &Database,
    root: &Path,
    name: &str,
    url: &str,
    branch: &str,
) -> Result<Repository> {
    config::validate_repository(name, url, branch)?;
    if database.repository(name).is_ok() {
        bail!("a repository named {name} already exists");
    }
    let path = clone_bare(root, name, url, true)?;
    let registered = git_output(
        &path,
        &["rev-parse", "--verify", &format!("refs/heads/{branch}")],
    )
    .map_err(|_| anyhow::anyhow!("{url} has no branch {branch}"))
    .and_then(|_| register(database, name, &path, branch, None, Some(url)));
    match registered {
        Ok(repository) => Ok(repository),
        Err(error) => {
            let _ = fs::remove_dir_all(&path);
            Err(error)
        }
    }
}

/// Hosted repositories forked from `source`.
pub fn forks_of(database: &Database, source: &str) -> Result<Vec<Repository>> {
    Ok(database
        .repositories()?
        .into_iter()
        .filter(|r| r.fork_of.as_deref() == Some(source))
        .collect())
}

/// Pull from the upstream: a fork fast-forwards its default branch, a mirror
/// takes every branch and tag (deleting what the upstream deleted).
pub fn sync(database: &Database, root: &Path, name: &str, now: i64) -> Result<SyncOutcome> {
    let repository = database.repository(name)?;
    let path = hosted_git::repository_path(root, &repository.name)?;
    ensure!(path.is_dir(), "{name} is not hosted on this forge");
    let outcome = match (&repository.fork_of, &repository.mirror_of) {
        (Some(upstream), _) => {
            let upstream_path = hosted_git::repository_path(root, upstream)?;
            ensure!(
                upstream_path.is_dir(),
                "upstream {upstream} is no longer hosted"
            );
            sync_fork(
                &path,
                &upstream_path,
                &repository.default_branch,
                &repository.name,
                upstream,
            )?
        }
        (None, Some(url)) => sync_mirror(&path, url, &repository.name)?,
        (None, None) => bail!("{name} is neither a fork nor a mirror"),
    };
    database.connection()?.execute(
        "UPDATE repositories SET synced_at = ?2 WHERE name = ?1",
        params![name, now],
    )?;
    Ok(outcome)
}

fn sync_fork(
    path: &Path,
    upstream: &Path,
    branch: &str,
    name: &str,
    upstream_name: &str,
) -> Result<SyncOutcome> {
    let reference = format!("refs/heads/{branch}");
    run(
        Command::new("git")
            .arg("-C")
            .arg(path)
            .args(["fetch", "--quiet", "--"])
            .arg(upstream)
            .arg(format!("{reference}:refs/upstream/{branch}")),
        "fetch from the upstream",
    )?;
    let theirs = git_output(path, &["rev-parse", &format!("refs/upstream/{branch}")])?;
    let ours = git_output(path, &["rev-parse", "--verify", "--quiet", &reference]).ok();
    let outcome = |updated: bool, reason: &str| SyncOutcome {
        repository: name.to_owned(),
        upstream: upstream_name.to_owned(),
        updated,
        reason: reason.to_owned(),
    };
    match ours {
        Some(ours) if ours == theirs => Ok(outcome(false, "already up to date")),
        Some(ours) => {
            let ancestor = Command::new("git")
                .arg("-C")
                .arg(path)
                .args(["merge-base", "--is-ancestor", &ours, &theirs])
                .status()?
                .success();
            if !ancestor {
                return Ok(outcome(
                    false,
                    "diverged: the fork has commits the upstream does not; merge or rebase first",
                ));
            }
            run(
                Command::new("git").arg("-C").arg(path).args([
                    "update-ref",
                    &reference,
                    &theirs,
                    &ours,
                ]),
                "fast-forward the default branch",
            )?;
            Ok(outcome(true, "fast-forwarded"))
        }
        None => {
            run(
                Command::new("git")
                    .arg("-C")
                    .arg(path)
                    .args(["update-ref", &reference, &theirs]),
                "create the default branch",
            )?;
            Ok(outcome(true, "created"))
        }
    }
}

fn sync_mirror(path: &Path, url: &str, name: &str) -> Result<SyncOutcome> {
    let before = git_output(path, &["for-each-ref", "--format=%(refname) %(objectname)"])?;
    run(
        Command::new("git").arg("-C").arg(path).args([
            "fetch",
            "--quiet",
            "--prune",
            "--",
            url,
            "+refs/heads/*:refs/heads/*",
            "+refs/tags/*:refs/tags/*",
        ]),
        "fetch from the mirror source",
    )?;
    let after = git_output(path, &["for-each-ref", "--format=%(refname) %(objectname)"])?;
    let updated = before != after;
    Ok(SyncOutcome {
        repository: name.to_owned(),
        upstream: url.to_owned(),
        updated,
        reason: if updated {
            "refs updated"
        } else {
            "already up to date"
        }
        .to_owned(),
    })
}

/// Mirrors that have not synced for `max_age` seconds; used by the timer.
pub fn due_mirrors(database: &Database, now: i64, max_age: i64) -> Result<Vec<String>> {
    Ok(database
        .repositories()?
        .into_iter()
        .filter(|r| r.mirror_of.is_some() && r.synced_at.is_none_or(|at| at + max_age <= now))
        .map(|r| r.name)
        .collect())
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::tempdir;

    const KEY: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";

    fn git(repository: &Path, arguments: &[&str]) {
        run(
            Command::new("git")
                .arg("-C")
                .arg(repository)
                .args(arguments),
            &format!("git {}", arguments.join(" ")),
        )
        .expect("git");
    }

    fn commit(repository: &Path, file: &str, text: &str) -> String {
        fs::write(repository.join(file), text).expect("write");
        git(repository, &["add", "."]);
        git(repository, &["commit", "-q", "-m", text]);
        git_output(repository, &["rev-parse", "HEAD"]).expect("sha")
    }

    /// A working repository with one commit, hosted on the forge as `name`.
    fn hosted(db: &Database, root: &Path, name: &str) -> (PathBuf, PathBuf) {
        let work = root.join(format!("work-{name}"));
        fs::create_dir_all(&work).expect("mkdir");
        git(&work, &["init", "-q", "-b", "main"]);
        git(&work, &["config", "user.name", "Test"]);
        git(&work, &["config", "user.email", "t@example"]);
        commit(&work, "README.md", "first");
        let hosted = hosted_git::host(root, name, &work).expect("host");
        db.add_repository(name, hosted.to_str().expect("utf8"), "main")
            .expect("add");
        (work, hosted)
    }

    #[test]
    fn a_fork_copies_the_upstream_and_fast_forwards_until_it_diverges() {
        let temp = tempdir().expect("tempdir");
        let root = temp.path();
        let db = Database::memory(KEY).expect("db");
        let (work, _) = hosted(&db, root, "upstream");

        let fork = fork(&db, root, "upstream", "my-fork").expect("fork");
        assert_eq!(fork.fork_of.as_deref(), Some("upstream"));
        assert_eq!(fork.default_branch, "main");
        let fork_path = hosted_git::repository_path(root, "my-fork").expect("path");
        assert!(
            fork_path.join("hooks/pre-receive").exists(),
            "hooks installed"
        );
        assert_eq!(forks_of(&db, "upstream").expect("forks").len(), 1);
        assert!(
            super::fork(&db, root, "upstream", "my-fork").is_err(),
            "no duplicate"
        );
        assert!(super::fork(&db, root, "missing", "x").is_err());

        let first = sync(&db, root, "my-fork", 100).expect("sync");
        assert!(!first.updated, "{first:?}");

        let second = commit(&work, "b.txt", "second");
        hosted_git::sync(root, "upstream", &work).expect("publish");
        let moved = sync(&db, root, "my-fork", 200).expect("sync");
        assert!(moved.updated, "{moved:?}");
        assert_eq!(
            git_output(&fork_path, &["rev-parse", "main"]).expect("sha"),
            second
        );
        assert_eq!(db.repository("my-fork").expect("repo").synced_at, Some(200));

        // A commit of its own on the fork: no silent overwrite.
        let clone = root.join("clone");
        git(
            root,
            &["clone", "-q", fork_path.to_str().expect("utf8"), "clone"],
        );
        git(&clone, &["config", "user.name", "Test"]);
        git(&clone, &["config", "user.email", "t@example"]);
        commit(&clone, "mine.txt", "own work");
        git(&clone, &["push", "-q", "origin", "main"]);
        commit(&work, "c.txt", "third");
        hosted_git::sync(root, "upstream", &work).expect("publish");
        let diverged = sync(&db, root, "my-fork", 300).expect("sync");
        assert!(!diverged.updated);
        assert!(diverged.reason.contains("diverged"), "{diverged:?}");
    }

    #[test]
    fn a_mirror_pulls_every_branch_and_tag_and_is_due_on_a_timer() {
        let temp = tempdir().expect("tempdir");
        let root = temp.path();
        let db = Database::memory(KEY).expect("db");
        let source = root.join("source");
        fs::create_dir_all(&source).expect("mkdir");
        git(&source, &["init", "-q", "-b", "main"]);
        git(&source, &["config", "user.name", "Test"]);
        git(&source, &["config", "user.email", "t@example"]);
        commit(&source, "README.md", "first");

        assert!(
            mirror(&db, root, "m", source.to_str().expect("utf8"), "trunk").is_err(),
            "no such branch"
        );
        assert!(
            !hosted_git::repository_path(root, "m")
                .expect("path")
                .exists(),
            "cleaned up"
        );
        let mirrored =
            mirror(&db, root, "m", source.to_str().expect("utf8"), "main").expect("mirror");
        assert_eq!(mirrored.mirror_of.as_deref(), source.to_str());
        assert_eq!(
            due_mirrors(&db, 1_000, 600).expect("due"),
            vec!["m".to_string()]
        );

        git(&source, &["checkout", "-q", "-b", "feature"]);
        commit(&source, "f.txt", "feature work");
        git(&source, &["tag", "v1"]);
        let outcome = sync(&db, root, "m", 1_000).expect("sync");
        assert!(outcome.updated, "{outcome:?}");
        let path = hosted_git::repository_path(root, "m").expect("path");
        assert!(git_output(&path, &["rev-parse", "refs/heads/feature"]).is_ok());
        assert!(git_output(&path, &["rev-parse", "refs/tags/v1"]).is_ok());
        assert!(due_mirrors(&db, 1_100, 600).expect("due").is_empty());
        assert_eq!(
            due_mirrors(&db, 1_700, 600).expect("due"),
            vec!["m".to_string()]
        );

        let again = sync(&db, root, "m", 1_700).expect("sync");
        assert!(!again.updated);
        git(&source, &["checkout", "-q", "main"]);
        git(&source, &["branch", "-D", "feature"]);
        let pruned = sync(&db, root, "m", 1_800).expect("sync");
        assert!(pruned.updated);
        assert!(
            git_output(&path, &["rev-parse", "--verify", "refs/heads/feature"]).is_err(),
            "pruned"
        );
    }
}