diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6be46bc0..0c3a882e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,14 +20,9 @@ jobs: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - - run: | - nix build .#program-docs - mkdir docs - cp -r result docs/programs - nix build .#nixos-manual - cp -r result docs/nixos-modules + - run: nix build .#manual - uses: actions/upload-pages-artifact@v3 with: - path: docs + path: result - if: github.ref == 'refs/heads/main' uses: actions/deploy-pages@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index a10e788c..9e67707a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## [7.3.0](https://github.com/molybdenumsoftware/pr-tracker/compare/v7.2.0...v7.3.0) (2025-08-07) + + +### Features + +* **fetcher:** some more req/res logging ([680373f](https://github.com/molybdenumsoftware/pr-tracker/commit/680373fb1d7aa05e2a54ad209960c81524915772)) + + +### Bug Fixes + +* address race condition with migrations ([2160ff9](https://github.com/molybdenumsoftware/pr-tracker/commit/2160ff90ced18decf7b00f2f8d3c94c28a3e666c)) + ## [7.2.0](https://github.com/molybdenumsoftware/pr-tracker/compare/v7.1.0...v7.2.0) (2025-05-20) diff --git a/Cargo.lock b/Cargo.lock index 0e1b9029..c00885f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -527,7 +527,7 @@ dependencies = [ [[package]] name = "db-context" -version = "7.2.0" +version = "7.3.0" dependencies = [ "camino", "fragile-child", @@ -763,7 +763,7 @@ dependencies = [ [[package]] name = "fragile-child" -version = "7.2.0" +version = "7.3.0" [[package]] name = "futures" @@ -2924,7 +2924,7 @@ dependencies = [ [[package]] name = "pr-tracker-api" -version = "7.2.0" +version = "7.3.0" dependencies = [ "anyhow", "camino", @@ -2934,10 +2934,10 @@ dependencies = [ "getset", "poem", "poem-openapi", - "pr-tracker-api-config", "pr-tracker-store", "sd-notify", "serde", + "serde_with", "sqlx", "tempfile", "thiserror 1.0.69", @@ -2946,20 +2946,9 @@ dependencies = [ "util", ] -[[package]] -name = "pr-tracker-api-config" -version = "7.2.0" -dependencies = [ - "camino", - "confique", - "serde", - "serde_with", - "tracing-subscriber", -] - [[package]] name = "pr-tracker-fetcher" -version = "7.2.0" +version = "7.3.0" dependencies = [ "anyhow", "camino", @@ -2973,7 +2962,6 @@ dependencies = [ "itertools", "log", "once_cell", - "pr-tracker-fetcher-config", "pr-tracker-store", "reqwest", "serde", @@ -2985,18 +2973,9 @@ dependencies = [ "wildmatch", ] -[[package]] -name = "pr-tracker-fetcher-config" -version = "7.2.0" -dependencies = [ - "camino", - "confique", - "serde", -] - [[package]] name = "pr-tracker-store" -version = "7.2.0" +version = "7.3.0" dependencies = [ "derive_more", "getset", @@ -4231,7 +4210,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "util" -version = "7.2.0" +version = "7.3.0" dependencies = [ "clap", "db-context", diff --git a/Cargo.toml b/Cargo.toml index 4a91ea2e..89533fb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,4 +29,4 @@ authors = ["Molybdenum Software"] edition = "2024" license = "MIT" repository = "https://github.com/molybdenumsoftware/pr-tracker" -version = "7.2.0" +version = "7.3.0" diff --git a/README.md b/README.md index 35763ccd..b8193a84 100644 --- a/README.md +++ b/README.md @@ -1,104 +1 @@ -# pr-tracker - -## What - -This is a system that provides answers to the following question: - -> Which branches did a certain GitHub pull request "land" in? - -A pull request is considered landed in a branch when that branch contains that pull request's "merge commit"[^1]. - -As of 2024-02-02 GitHub does not provide an API for directly asking this question. - -## How - -In order to provide an API that answers this question the following are obtained: - -1. All pull requests and their merge commits (via GitHub's GraphQL API). -2. A clone of the repository. - -From these, all landings are deduced and stored in a (PostgreSQL) database. - -It is assumed that in the tracked branches, history is never rewritten. - -Two programs are provided: - -1. pr-tracker-fetcher: obtains data, determines landings and persists them. -2. pr-tracker-api: provides an HTTP endpoint for querying landings. - -## pr-tracker-fetcher - -Intended to be periodically executed. -Takes no arguments. -Expects [configuration via environment](https://molybdenumsoftware.github.io/pr-tracker/programs/pr_tracker_fetcher_config/struct.Environment.html). - -## pr-tracker-api - -Takes no arguments. -Expects [configuration via environment](https://molybdenumsoftware.github.io/pr-tracker/programs/pr_tracker_api_config/struct.Environment.html). - -- `/openapi.json` -- `/` redirects to API documentation - -## NixOS modules - -[Manual](https://molybdenumsoftware.github.io/pr-tracker/nixos-modules) - -## Versioning - -- This project uses [Conventional Commits v1](https://www.conventionalcommits.org/en/v1.0.0/) and [Semantic Versioning v2](https://semver.org/spec/v2.0.0.html). -- With regard to versioning, the documented executables and NixOS modules are public. - The libraries are private. -- This project, with all its components, is versioned as one. - -[^1]: Note that in GitHub, a pull request has a "merge commit" even having been merged without an actual merge commit. - -## Prior art - -- [Alyssa Ross' pr-tracker](https://nixpk.gs/pr-tracker.html) ([source](https://git.qyliss.net/pr-tracker)) - Server-side rendered web app. Computes landings for a given PR on the fly by invoking Git on the backend. -- [ocfox's nixpkgs-tracker](https://nixpkgs-tracker.ocfox.me/) ([source](https://github.com/ocfox/nixpkgs-tracker)) - Client-side rendered web app. Computes landings for a given PR on the fly using GitHub api. -- [Maralorn's nixpkgs-bot](https://blog.maralorn.de/projects#nixpkgs-bot) ([source](https://code.maralorn.de/maralorn/config/src/commit/b34d2e0d0adc62c30875edb475f1c09a752fe19e/packages/nixpkgs-bot)) - Matrix bot that provides notification of PR landings. - Periodically computes new PR landings using Git and sends messages. - -All of the above are [Nixpkgs](https://github.com/nixos/nixpkgs/) specific, whereas this project is not. -None of the above internally maintain a dataset of landings. -None of the above currently provide an HTTP API. - -## Vision - -### Push-driven updates - -The current architecture of obtaining data via polling allows instantaneous -and hopefully reliable responses. -However, the data can be stale. - -In the future, we intend to provide fresher data by subscribing to GitHub webhooks. - -Since the public cannot subscribe to GitHub webhooks, -this will require deployment by the repo owner. - -### Event record keeping - -Building upon the implementation of push-driven updates, -we intend to keep track of _when_ PRs land in branches. -This requires a dataset of landings. - -### Webhook service - -We intend to allow users to subscribe to webhook notifications of PR landings. -This provides a couple of benefits over subscribing to GitHub webhooks directly: - -- GitHub webhooks can only notify when a PR lands in its target branch. They - cannot notify when that PR lands in other branches. -- Only repo owners can subscribe to GitHub webhooks. - -### Backport PRs - -A backport PR is a re-application of another PR, targeting a different branch. - -We intend to adopt or invent a workflow whereby in backport PRs the original PR is declared. -Using that metadata, when providing landings for an original PR, -we intend to also include branches on which a backport PR had landed. +[See Manual](https://molybdenumsoftware.github.io/pr-tracker) diff --git a/crates/DATABASE_URL.md b/crates/DATABASE_URL.md deleted file mode 100644 index d784f4c4..00000000 --- a/crates/DATABASE_URL.md +++ /dev/null @@ -1 +0,0 @@ -[PostgreSQL connection URI](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING-URIS). diff --git a/crates/api-config/Cargo.toml b/crates/api-config/Cargo.toml deleted file mode 100644 index f741323b..00000000 --- a/crates/api-config/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[dependencies] -camino.workspace = true -confique.workspace = true -serde.workspace = true -serde_with.workspace = true -tracing-subscriber.workspace = true - -[package] -name = "pr-tracker-api-config" -publish = false -authors.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -version.workspace = true diff --git a/crates/api-config/PORT.md b/crates/api-config/PORT.md deleted file mode 100644 index f96f498d..00000000 --- a/crates/api-config/PORT.md +++ /dev/null @@ -1 +0,0 @@ -Port to listen on. diff --git a/crates/api-config/TRACING_FILTER.md b/crates/api-config/TRACING_FILTER.md deleted file mode 100644 index 0ba8f0eb..00000000 --- a/crates/api-config/TRACING_FILTER.md +++ /dev/null @@ -1 +0,0 @@ -Expected to deserialize into an [`EnvFilter`](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html). diff --git a/crates/api-config/src/lib.rs b/crates/api-config/src/lib.rs deleted file mode 100644 index 5f462697..00000000 --- a/crates/api-config/src/lib.rs +++ /dev/null @@ -1,41 +0,0 @@ -#![allow(non_snake_case, clippy::struct_field_names)] - -pub use environment::Environment; -use serde::Deserialize; -use serde_with::DisplayFromStr; -use serde_with::serde_as; -use tracing_subscriber::EnvFilter; - -mod environment { - use super::TracingFilter; - use confique::Config; - - /// See documentation for each field. - #[derive(Debug, Config)] - pub struct Environment { - #[doc = include_str!("../../DATABASE_URL.md")] - #[config(env = "PR_TRACKER_API_DATABASE_URL")] - pub PR_TRACKER_API_DATABASE_URL: String, - #[config(env = "PR_TRACKER_API_PORT")] - #[doc = include_str!("../PORT.md")] - pub PR_TRACKER_API_PORT: u16, - #[config(env = "PR_TRACKER_TRACING_FILTER")] - #[doc = include_str!("../TRACING_FILTER.md")] - pub PR_TRACKER_TRACING_FILTER: Option, - } -} - -#[serde_as] -#[derive(Deserialize, Debug)] -pub struct TracingFilter { - #[serde_as(as = "DisplayFromStr")] - pub env_filter: EnvFilter, -} - -impl Default for TracingFilter { - fn default() -> Self { - Self { - env_filter: EnvFilter::try_new("info").unwrap(), - } - } -} diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 14823922..be8f379c 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -2,13 +2,13 @@ camino = "*" poem = {features = ["requestid", "test"], version = "*"} poem-openapi = {features = ["swagger-ui"], version = "*"} -pr-tracker-api-config = {path = "../api-config"} sqlx = {default-features = false, features = ["macros", "migrate"], version = "*"} tokio = "*" anyhow.workspace = true confique.workspace = true pr-tracker-store.workspace = true serde.workspace = true +serde_with.workspace = true thiserror.workspace = true tracing-subscriber.workspace = true util.workspace = true diff --git a/crates/api/src/config.rs b/crates/api/src/config.rs new file mode 100644 index 00000000..4ba05f23 --- /dev/null +++ b/crates/api/src/config.rs @@ -0,0 +1,23 @@ +#![allow(non_snake_case, clippy::struct_field_names)] + +use serde::Deserialize; +use serde_with::DisplayFromStr; +use serde_with::serde_as; +use tracing_subscriber::EnvFilter; + +include!(env!("api_config_snippet")); + +#[serde_as] +#[derive(Deserialize, Debug)] +pub struct TracingFilter { + #[serde_as(as = "DisplayFromStr")] + pub env_filter: EnvFilter, +} + +impl Default for TracingFilter { + fn default() -> Self { + Self { + env_filter: EnvFilter::try_new("info").unwrap(), + } + } +} diff --git a/crates/api/src/main.rs b/crates/api/src/main.rs index 937ae700..cb3ada14 100644 --- a/crates/api/src/main.rs +++ b/crates/api/src/main.rs @@ -1,13 +1,15 @@ #![warn(clippy::pedantic)] +mod config; + use anyhow::Context; +use config::Environment; use confique::Config; use poem::{ Server, listener::{Listener, TcpListener}, }; use pr_tracker_api::endpoint; -use pr_tracker_api_config::Environment; #[tokio::main] async fn main() { diff --git a/crates/fetcher-config/BRANCH_PATTERNS.md b/crates/fetcher-config/BRANCH_PATTERNS.md deleted file mode 100644 index 36541efc..00000000 --- a/crates/fetcher-config/BRANCH_PATTERNS.md +++ /dev/null @@ -1,6 +0,0 @@ -JSON array of strings representing branch patterns to track. - -- `?` matches a single occurrence of any character. -- `*` matches zero or more occurrences of any character. - -No escape characters. diff --git a/crates/fetcher-config/Cargo.toml b/crates/fetcher-config/Cargo.toml deleted file mode 100644 index d87db030..00000000 --- a/crates/fetcher-config/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[dependencies] -camino.workspace = true -confique.workspace = true -serde.workspace = true - -[package] -name = "pr-tracker-fetcher-config" -publish = false -authors.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -version.workspace = true diff --git a/crates/fetcher-config/GITHUB_REPO_NAME.md b/crates/fetcher-config/GITHUB_REPO_NAME.md deleted file mode 100644 index 89b6f672..00000000 --- a/crates/fetcher-config/GITHUB_REPO_NAME.md +++ /dev/null @@ -1 +0,0 @@ -GitHub repository name. diff --git a/crates/fetcher-config/GITHUB_REPO_OWNER.md b/crates/fetcher-config/GITHUB_REPO_OWNER.md deleted file mode 100644 index 7f6cf37c..00000000 --- a/crates/fetcher-config/GITHUB_REPO_OWNER.md +++ /dev/null @@ -1 +0,0 @@ -GitHub repository owner. diff --git a/crates/fetcher-config/GITHUB_TOKEN.md b/crates/fetcher-config/GITHUB_TOKEN.md deleted file mode 100644 index f9d0da10..00000000 --- a/crates/fetcher-config/GITHUB_TOKEN.md +++ /dev/null @@ -1 +0,0 @@ -GitHub API token with read access to the repository's pull requests. diff --git a/crates/fetcher-config/src/lib.rs b/crates/fetcher-config/src/lib.rs deleted file mode 100644 index 8bccdfcc..00000000 --- a/crates/fetcher-config/src/lib.rs +++ /dev/null @@ -1,31 +0,0 @@ -#![allow(non_snake_case, clippy::struct_field_names)] - -pub use environment::Environment; - -mod environment { - use camino::Utf8PathBuf; - use confique::Config; - - /// See documentation for each field. - #[derive(Debug, Config)] - pub struct Environment { - #[doc = include_str!("../../DATABASE_URL.md")] - #[config(env = "PR_TRACKER_FETCHER_DATABASE_URL")] - pub PR_TRACKER_FETCHER_DATABASE_URL: String, - #[config(env = "PR_TRACKER_FETCHER_GITHUB_TOKEN")] - #[doc = include_str!("../GITHUB_TOKEN.md")] - pub PR_TRACKER_FETCHER_GITHUB_TOKEN: String, - #[config(env = "PR_TRACKER_FETCHER_GITHUB_REPO_OWNER")] - #[doc = include_str!("../GITHUB_REPO_OWNER.md")] - pub PR_TRACKER_FETCHER_GITHUB_REPO_OWNER: String, - #[config(env = "PR_TRACKER_FETCHER_GITHUB_REPO_NAME")] - #[doc = include_str!("../GITHUB_REPO_NAME.md")] - pub PR_TRACKER_FETCHER_GITHUB_REPO_NAME: String, - /// Cache directory (for repository clone). - #[config(env = "PR_TRACKER_FETCHER_CACHE_DIR")] - pub PR_TRACKER_FETCHER_CACHE_DIR: Utf8PathBuf, - #[config(env = "PR_TRACKER_FETCHER_BRANCH_PATTERNS")] - #[doc = include_str!("../BRANCH_PATTERNS.md")] - pub PR_TRACKER_FETCHER_BRANCH_PATTERNS: String, - } -} diff --git a/crates/fetcher/Cargo.toml b/crates/fetcher/Cargo.toml index 01313fe8..2c5a9ddd 100644 --- a/crates/fetcher/Cargo.toml +++ b/crates/fetcher/Cargo.toml @@ -4,7 +4,6 @@ env_logger = "*" graphql_client = {default-features = false, features = ["graphql_query_derive", "reqwest-rustls"], version = "*"} itertools = "*" log = "*" -pr-tracker-fetcher-config = {path = "../fetcher-config"} # Pinned for compatibility with graphql-client. # We can unpin once this is released: reqwest = {default-features = false, features = ["rustls-tls"], version = "^0.11"} diff --git a/crates/fetcher/src/config.rs b/crates/fetcher/src/config.rs new file mode 100644 index 00000000..72a43e93 --- /dev/null +++ b/crates/fetcher/src/config.rs @@ -0,0 +1,3 @@ +#![allow(non_snake_case, clippy::struct_field_names)] + +include!(env!("fetcher_config_snippet")); diff --git a/crates/fetcher/src/github.rs b/crates/fetcher/src/github.rs index ef1f4023..c58c573c 100644 --- a/crates/fetcher/src/github.rs +++ b/crates/fetcher/src/github.rs @@ -1,6 +1,7 @@ use anyhow::Context; use graphql_client::GraphQLQuery; use pr_tracker_store::{GitCommit, GithubPrQueryCursor, Pr, PrNumber}; +use reqwest::header::HeaderValue; // https://docs.github.com/en/graphql/overview/rate-limits-and-node-limits-for-the-graphql-api#node-limit const PAGE_SIZE: i64 = 100; @@ -51,10 +52,9 @@ impl GitHubGraphqlClient { page_size_override_for_testing: Option, ) -> anyhow::Result { let mut headers = reqwest::header::HeaderMap::new(); - headers.insert( - reqwest::header::AUTHORIZATION, - format!("Bearer {api_token}").parse()?, - ); + let mut authorization = HeaderValue::from_str(&format!("Bearer {api_token}"))?; + authorization.set_sensitive(true); + headers.insert(reqwest::header::AUTHORIZATION, authorization); let client = reqwest::Client::builder() .user_agent(USER_AGENT) @@ -81,27 +81,41 @@ impl GithubClient for GitHubGraphqlClient { cursor: cursor.map(|cursor| cursor.as_str().to_string()), page_size: self.page_size, }; - let resp = - graphql_client::reqwest::post_graphql::(self, API_URL, query_vars) - .await?; + + let body = PullsQuery::build_query(query_vars); + let request = self.post(API_URL).json(&body); + log::info!("request: {request:#?}"); + log::info!("request body: {body:#?}"); + let resp: graphql_client::Response = + request.send().await?.json().await?; // https://github.com/graphql-rust/graphql-client/issues/467 let data = resp .data + .as_ref() .context("response with no data, this might be due to an expired token")?; log::info!("rate limits: {:?}", data.rate_limit); - let repository = data.repository.context("data with no repo")?; - let response_prs = repository.pull_requests; - let nodes = response_prs.nodes.unwrap_or_default(); + let repository = data + .repository + .as_ref() + .context(format!("data with no repo\n{:#?}", &resp))?; + let nodes = repository + .pull_requests + .nodes + .as_deref() + .unwrap_or_default(); let prs = nodes - .into_iter() + .iter() .map(|node| -> anyhow::Result<_> { - let node = node.context("null PR node")?; + let node = node.as_ref().context("null PR node")?; - let commit = node.merge_commit.map(|commit| GitCommit(commit.oid)); + let commit = node + .merge_commit + .as_ref() + .map(|commit| GitCommit(commit.oid.clone())); let number: PrNumber = node .number @@ -112,9 +126,11 @@ impl GithubClient for GitHubGraphqlClient { }) .collect::>>()?; - let new_cursor = response_prs + let new_cursor = repository + .pull_requests .page_info .end_cursor + .clone() .map(GithubPrQueryCursor::new); Ok((prs, new_cursor)) diff --git a/crates/fetcher/src/main.rs b/crates/fetcher/src/main.rs index 10afa775..aaafc807 100644 --- a/crates/fetcher/src/main.rs +++ b/crates/fetcher/src/main.rs @@ -1,9 +1,11 @@ #![warn(clippy::pedantic)] +mod config; + use anyhow::{Context, ensure}; +use config::Environment; use confique::Config; use pr_tracker_fetcher::{github::GitHubGraphqlClient, run}; -use pr_tracker_fetcher_config::Environment; use pr_tracker_store::PgConnection; use sqlx_core::connection::Connection; use wildmatch::WildMatch; diff --git a/flake.lock b/flake.lock index 42bb845e..fb615330 100644 --- a/flake.lock +++ b/flake.lock @@ -47,11 +47,11 @@ "pyproject-nix": "pyproject-nix" }, "locked": { - "lastModified": 1735160684, - "narHash": "sha256-n5CwhmqKxifuD4Sq4WuRP/h5LO6f23cGnSAuJemnd/4=", + "lastModified": 1753366881, + "narHash": "sha256-jsoTEhkmn3weESMNRMLNk/ROW3fcHCr8Wgf5amzs5z8=", "owner": "nix-community", "repo": "dream2nix", - "rev": "8ce6284ff58208ed8961681276f82c2f8f978ef4", + "rev": "e6566e4ce924a8258499c379ee9552dba1883bce", "type": "github" }, "original": { @@ -99,11 +99,11 @@ ] }, "locked": { - "lastModified": 1743550720, - "narHash": "sha256-hIshGgKZCgWh6AYJpJmRgFdR3WUbkY04o82X05xqQiY=", + "lastModified": 1754091436, + "narHash": "sha256-XKqDMN1/Qj1DKivQvscI4vmHfDfvYR2pfuFOJiCeewM=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "c621e8422220273271f52058f618c94e405bb0f5", + "rev": "67df8c627c2c39c41dbec76a1f201929929ab0bd", "type": "github" }, "original": { @@ -121,11 +121,11 @@ ] }, "locked": { - "lastModified": 1747372754, - "narHash": "sha256-2Y53NGIX2vxfie1rOW0Qb86vjRZ7ngizoo+bnXU9D9k=", + "lastModified": 1750779888, + "narHash": "sha256-wibppH3g/E2lxU43ZQHC5yA/7kIKLGxVEnsnVK1BtRg=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "80479b6ec16fefd9c1db3ea13aeb038c60530f46", + "rev": "16ec914f6fb6f599ce988427d9d94efddf25fe6d", "type": "github" }, "original": { @@ -137,11 +137,11 @@ "github-graphql-schema": { "flake": false, "locked": { - "lastModified": 1746607928, - "narHash": "sha256-aFW52fhkF91sLbzE010/JUdyk/ZNzp4rpTHFj6yrFUo=", + "lastModified": 1747721806, + "narHash": "sha256-OsMtRMbtIA12nn1ZMoIbUKDee+VgptAOFgaGJe7wMUw=", "owner": "octokit", "repo": "graphql-schema", - "rev": "3b88e02665bbf85ce462a4e8e44c9595777f4a1b", + "rev": "a39eba89e0056a2cd6e87ef054f9748929943aae", "type": "github" }, "original": { @@ -204,11 +204,11 @@ ] }, "locked": { - "lastModified": 1747462581, - "narHash": "sha256-DpZzKTzrowBfpxMYAe5CsfqcN1TsRyCMaqoUpQ8BF5E=", + "lastModified": 1754115510, + "narHash": "sha256-7fVDO8gvsIkXtmxBSSYMseIRvxCd1m+HrtubtUhgp3Q=", "owner": "yusdacra", "repo": "nix-cargo-integration", - "rev": "f0fba443d429c90ad14fc51aa0d0bab81e83fd40", + "rev": "fa4b1c0aa460d1741a4e8cb695cf9b2b74b52db1", "type": "github" }, "original": { @@ -219,11 +219,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1747426788, - "narHash": "sha256-N4cp0asTsJCnRMFZ/k19V9akkxb7J/opG+K+jU57JGc=", + "lastModified": 1754151594, + "narHash": "sha256-S30TWshtDmNlU30u842RidFUraKj1f2dd4nrKRHm3gE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "12a55407652e04dcf2309436eb06fef0d3713ef3", + "rev": "7b6929d8b900de3142638310f8bc40cff4f2c507", "type": "github" }, "original": { @@ -279,18 +279,23 @@ } }, "pyproject-nix": { - "flake": false, + "inputs": { + "nixpkgs": [ + "nci", + "dream2nix", + "nixpkgs" + ] + }, "locked": { - "lastModified": 1702448246, - "narHash": "sha256-hFg5s/hoJFv7tDpiGvEvXP0UfFvFEDgTdyHIjDVHu1I=", - "owner": "davhau", + "lastModified": 1752481895, + "narHash": "sha256-luVj97hIMpCbwhx3hWiRwjP2YvljWy8FM+4W9njDhLA=", + "owner": "pyproject-nix", "repo": "pyproject.nix", - "rev": "5a06a2697b228c04dd2f35659b4b659ca74f7aeb", + "rev": "16ee295c25107a94e59a7fc7f2e5322851781162", "type": "github" }, "original": { - "owner": "davhau", - "ref": "dream2nix", + "owner": "pyproject-nix", "repo": "pyproject.nix", "type": "github" } @@ -316,11 +321,11 @@ ] }, "locked": { - "lastModified": 1747449297, - "narHash": "sha256-veyXchTz6eWwvuW5X49UluHkheHkFcqHJSwGuKBhrmQ=", + "lastModified": 1754102567, + "narHash": "sha256-d6nZ+1e4VDqW6VAwfx9EAUDJdPxSwqwGiuli32FEgoE=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "f44db7d7cea4528288780c6347756173a8248225", + "rev": "08ff39bf869cadca3102b39824f4c7025186b7dc", "type": "github" }, "original": { @@ -390,11 +395,11 @@ ] }, "locked": { - "lastModified": 1747469671, - "narHash": "sha256-bo1ptiFoNqm6m1B2iAhJmWCBmqveLVvxom6xKmtuzjg=", + "lastModified": 1754061284, + "narHash": "sha256-ONcNxdSiPyJ9qavMPJYAXDNBzYobHRxw0WbT38lKbwU=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "ab0378b61b0d85e73a8ab05d5c6029b5bd58c9fb", + "rev": "58bd4da459f0a39e506847109a2a5cfceb837796", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index dbc1eef4..fdcf0359 100644 --- a/flake.nix +++ b/flake.nix @@ -34,11 +34,13 @@ ./modules/filter-options.nix ./modules/formatting.nix ./modules/integration-tests + ./modules/manual.nix + ./modules/prior-art.nix + ./modules/introduction.nix + ./modules/vision.nix ./modules/nci.nix - ./modules/nixos-manual ./modules/nixos-modules-lib.nix ./modules/private-nixos-modules - ./modules/program-docs.nix ./modules/release ./modules/store.nix ./modules/util.nix diff --git a/modules/api/default.nix b/modules/api/default.nix index b269d281..c33adce0 100644 --- a/modules/api/default.nix +++ b/modules/api/default.nix @@ -1,23 +1,75 @@ -{ lib, ... }: +{ + lib, + api, + psqlConnectionUriMdLink, + environmentVariablesToMarkdown, + ... +}: { imports = [ ./nixos-module.nix ]; + _module.args.api.environmentVariables = lib.mapAttrs (name: v: v // { inherit name; }) { + PR_TRACKER_API_DATABASE_URL = { + description = # markdown + "${psqlConnectionUriMdLink}."; + rustType = "::std::string::String"; + }; + PR_TRACKER_API_PORT = { + description = + # markdown + "Port to listen on."; + rustType = "::core::primitive::u16"; + }; + PR_TRACKER_TRACING_FILTER = { + description = + # markdown + '' + Optional. + Expected to deserialize into an [`EnvFilter`](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html). + ''; + # Note: ideally we'd use `::core::option::Option`, but cannot because + # confique's derive macro seems not to support it. + rustType = "Option"; + }; + }; + perSystem = { + pkgs, self', config, + writeEnvironmentStructFile, ... }: { + chapters.api = { + title = "API"; + drv = pkgs.writeTextFile { + name = "api.md"; + text = + # markdown + '' + Takes no arguments. + + - `/openapi.json` + - `/` redirects to API documentation + + ## Environment Variables + + Reads the following environment variables. + + ${environmentVariablesToMarkdown api.environmentVariables} + ''; + }; + }; + nci = { crates = { - pr-tracker-api.drvConfig.mkDerivation.meta.mainProgram = "pr-tracker-api"; - pr-tracker-api-config.includeInProjectDocs = true; + pr-tracker-api.drvConfig = { + mkDerivation.meta.mainProgram = "pr-tracker-api"; + env.api_config_snippet = writeEnvironmentStructFile "api" api.environmentVariables; + }; }; - projects.default.fileset = lib.fileset.unions [ - ../../crates/api-config/PORT.md - ../../crates/api-config/TRACING_FILTER.md - ]; }; packages.api = config.nci.outputs.pr-tracker-api.packages.release; checks = { diff --git a/modules/api/nixos-module.nix b/modules/api/nixos-module.nix index f3a47bb3..f9f11621 100644 --- a/modules/api/nixos-module.nix +++ b/modules/api/nixos-module.nix @@ -2,6 +2,7 @@ mkNixosModuleLib, moduleLocation, privateNixosModules, + api, ... }: { @@ -45,12 +46,12 @@ options.services.pr-tracker.api.port = lib.mkOption { type = lib.types.port; - description = builtins.readFile ../../crates/api-config/PORT.md; + inherit (api.environmentVariables.PR_TRACKER_API_PORT) description; }; options.services.pr-tracker.api.tracingFilter = lib.mkOption { type = lib.types.nullOr lib.types.str; - description = builtins.readFile ../../crates/api-config/TRACING_FILTER.md; + inherit (api.environmentVariables.PR_TRACKER_TRACING_FILTER) description; default = null; }; @@ -68,29 +69,24 @@ systemd.services.pr-tracker-api.wantedBy = [ "multi-user.target" ]; systemd.services.pr-tracker-api.after = [ "network.target" - ] ++ lib.optional cfg.db.isLocal "postgresql.service"; - systemd.services.pr-tracker-api.bindsTo = lib.optional cfg.db.isLocal "postgresql.service"; + ] + ++ lib.optional cfg.db.isLocal "postgresql.target"; + systemd.services.pr-tracker-api.bindsTo = lib.optional cfg.db.isLocal "postgresql.target"; - systemd.services.pr-tracker-api.script = - let - databaseUrl = "postgresql://?${attrsToURLParams cfg.db.urlParams}"; - - passwordFile = lib.optional (cfg.db.passwordFile != null) '' - PASSWORD=$(${lib.getExe pkgs.urlencode} --encode-set component < ${cfg.db.passwordFile}) - PR_TRACKER_API_DATABASE_URL="$PR_TRACKER_API_DATABASE_URL&password=$PASSWORD" - ''; - in - lib.concatStringsSep "\n" ( - [ - "export PR_TRACKER_API_DATABASE_URL=${lib.escapeShellArg databaseUrl}" - "export PR_TRACKER_API_PORT=${lib.escapeShellArg (toString cfg.port)}" - ] - ++ (lib.optional ( - cfg.tracingFilter != null - ) "export PR_TRACKER_TRACING_FILTER=${lib.escapeShellArg cfg.tracingFilter}") - ++ passwordFile - ++ [ "exec ${lib.getExe cfg.package}" ] - ); + systemd.services.pr-tracker-api.script = lib.concatLines ( + [ + "export ${api.environmentVariables.PR_TRACKER_API_DATABASE_URL.name}=${lib.escapeShellArg "postgresql://?${attrsToURLParams cfg.db.urlParams}"}" + "export ${api.environmentVariables.PR_TRACKER_API_PORT.name}=${lib.escapeShellArg (toString cfg.port)}" + ] + ++ (lib.optional (cfg.tracingFilter != null) + "export ${api.environmentVariables.PR_TRACKER_TRACING_FILTER.name}=${lib.escapeShellArg cfg.tracingFilter}" + ) + ++ lib.optional (cfg.db.passwordFile != null) '' + PASSWORD=$(${lib.getExe pkgs.urlencode} --encode-set component < ${cfg.db.passwordFile}) + ${api.environmentVariables.PR_TRACKER_API_DATABASE_URL.name}+="&password=$PASSWORD" + '' + ++ [ "exec ${lib.getExe cfg.package}" ] + ); systemd.services.pr-tracker-api.serviceConfig.User = cfg.user; systemd.services.pr-tracker-api.serviceConfig.Group = cfg.group; systemd.services.pr-tracker-api.serviceConfig.Type = "notify"; diff --git a/modules/fetcher/default.nix b/modules/fetcher/default.nix index dfc257f7..0b8185d4 100644 --- a/modules/fetcher/default.nix +++ b/modules/fetcher/default.nix @@ -1,32 +1,98 @@ -{ inputs, ... }: +{ + lib, + inputs, + fetcher, + psqlConnectionUriMdLink, + environmentVariablesToMarkdown, + ... +}: { imports = [ ./nixos-module.nix ]; + _module.args.fetcher.environmentVariables = lib.mapAttrs (name: v: v // { inherit name; }) { + PR_TRACKER_FETCHER_DATABASE_URL = { + description = "${psqlConnectionUriMdLink}."; + rustType = "::std::string::String"; + }; + PR_TRACKER_FETCHER_GITHUB_TOKEN = { + description = + # markdown + "GitHub API token with read access to the repository's pull requests."; + rustType = "::std::string::String"; + }; + PR_TRACKER_FETCHER_GITHUB_REPO_OWNER = { + description = + # markdown + "GitHub repository owner."; + rustType = "::std::string::String"; + }; + PR_TRACKER_FETCHER_GITHUB_REPO_NAME = { + description = + # markdown + "GitHub repository name."; + rustType = "::std::string::String"; + }; + PR_TRACKER_FETCHER_CACHE_DIR = { + description = + # markdown + "Cache directory (for repository clone)."; + rustType = "::camino::Utf8PathBuf"; + }; + PR_TRACKER_FETCHER_BRANCH_PATTERNS = { + description = + # markdown + '' + JSON array of strings representing branch patterns to track. + + - `?` matches a single occurrence of any character. + - `*` matches zero or more occurrences of any character. + + No escape characters. + ''; + rustType = "::std::string::String"; + }; + }; + perSystem = { self', config, pkgs, - lib, + writeEnvironmentStructFile, ... }: { + chapters.fetcher = { + title = "Fetcher"; + drv = pkgs.writeTextFile { + name = "fetcher.md"; + text = + # markdown + '' + Intended to be periodically executed. + Takes no arguments. + + ## Environment Variables + + Reads the following environment variables. + + ${environmentVariablesToMarkdown fetcher.environmentVariables} + ''; + }; + }; + nci = { projects.default = { drvConfig.env = { GITHUB_GRAPHQL_SCHEMA = "${inputs.github-graphql-schema}/schema.graphql"; GIT = lib.getExe pkgs.git; }; - fileset = lib.fileset.unions [ - ../../crates/fetcher-config/BRANCH_PATTERNS.md - ../../crates/fetcher-config/GITHUB_TOKEN.md - ../../crates/fetcher-config/GITHUB_REPO_OWNER.md - ../../crates/fetcher-config/GITHUB_REPO_NAME.md - ]; }; crates = { - pr-tracker-fetcher.drvConfig.mkDerivation.meta.mainProgram = "pr-tracker-fetcher"; - pr-tracker-fetcher-config.includeInProjectDocs = true; + pr-tracker-fetcher.drvConfig = { + mkDerivation.meta.mainProgram = "pr-tracker-fetcher"; + env.fetcher_config_snippet = writeEnvironmentStructFile "fetcher" fetcher.environmentVariables; + }; }; }; diff --git a/modules/fetcher/nixos-module.nix b/modules/fetcher/nixos-module.nix index 6aa9c6fb..576bf19c 100644 --- a/modules/fetcher/nixos-module.nix +++ b/modules/fetcher/nixos-module.nix @@ -2,6 +2,7 @@ mkNixosModuleLib, moduleLocation, privateNixosModules, + fetcher, ... }: { @@ -45,7 +46,7 @@ options.services.pr-tracker.fetcher.branchPatterns = lib.mkOption { type = lib.types.listOf lib.types.str; - description = builtins.readFile ../../crates/fetcher-config/BRANCH_PATTERNS.md; + inherit (fetcher.environmentVariables.PR_TRACKER_FETCHER_BRANCH_PATTERNS) description; example = [ "release-*" ]; }; @@ -53,20 +54,19 @@ options.services.pr-tracker.fetcher.githubApiTokenFile = lib.mkOption { type = lib.types.path; - description = - "Path to a file containing a " + builtins.readFile ../../crates/fetcher-config/GITHUB_TOKEN.md; + description = "Path to a file containing a ${fetcher.environmentVariables.PR_TRACKER_FETCHER_GITHUB_TOKEN.description}"; example = "/run/secrets/github-api.token"; }; options.services.pr-tracker.fetcher.repo.owner = lib.mkOption { type = lib.types.str; - description = builtins.readFile ../../crates/fetcher-config/GITHUB_REPO_OWNER.md; + inherit (fetcher.environmentVariables.PR_TRACKER_FETCHER_GITHUB_REPO_OWNER) description; example = "NixOS"; }; options.services.pr-tracker.fetcher.repo.name = lib.mkOption { type = lib.types.str; - description = builtins.readFile ../../crates/fetcher-config/GITHUB_REPO_NAME.md; + inherit (fetcher.environmentVariables.PR_TRACKER_FETCHER_GITHUB_REPO_NAME) description; example = "nixpkgs"; }; @@ -92,31 +92,26 @@ systemd.services.pr-tracker-fetcher.description = "pr-tracker-fetcher"; systemd.services.pr-tracker-fetcher.after = [ "network.target" - ] ++ lib.optional cfg.db.isLocal "postgresql.service"; - systemd.services.pr-tracker-fetcher.requires = lib.optional cfg.db.isLocal "postgresql.service"; - systemd.services.pr-tracker-fetcher.script = - let - databaseUrl = "postgresql://?${attrsToURLParams cfg.db.urlParams}"; - - passwordFile = lib.optional (cfg.db.passwordFile != null) '' - PASSWORD=$(${lib.getExe pkgs.urlencode} --encode-set component < ${cfg.db.passwordFile}) - PR_TRACKER_FETCHER_DATABASE_URL="$PR_TRACKER_FETCHER_DATABASE_URL&password=$PASSWORD" - ''; - in - builtins.concatStringsSep "\n" ( - [ - "export PR_TRACKER_FETCHER_DATABASE_URL=${lib.escapeShellArg databaseUrl}" - "export PR_TRACKER_FETCHER_GITHUB_REPO_OWNER=${lib.escapeShellArg cfg.repo.owner}" - "export PR_TRACKER_FETCHER_GITHUB_REPO_NAME=${lib.escapeShellArg cfg.repo.name}" - "export PR_TRACKER_FETCHER_BRANCH_PATTERNS=${lib.escapeShellArg (builtins.toJSON cfg.branchPatterns)}" - "export PR_TRACKER_FETCHER_GITHUB_TOKEN=$(< ${cfg.githubApiTokenFile})" - # CACHE_DIRECTORY is set by systemd based on the CacheDirectory setting. - # See https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html#RuntimeDirectory= - "export PR_TRACKER_FETCHER_CACHE_DIR=$CACHE_DIRECTORY" - ] - ++ passwordFile - ++ [ "exec ${lib.getExe cfg.package}" ] - ); + ] + ++ lib.optional cfg.db.isLocal "postgresql.target"; + systemd.services.pr-tracker-fetcher.requires = lib.optional cfg.db.isLocal "postgresql.target"; + systemd.services.pr-tracker-fetcher.script = lib.concatLines ( + [ + "export ${fetcher.environmentVariables.PR_TRACKER_FETCHER_DATABASE_URL.name}=${lib.escapeShellArg "postgresql://?${attrsToURLParams cfg.db.urlParams}"}" + "export ${fetcher.environmentVariables.PR_TRACKER_FETCHER_GITHUB_REPO_OWNER.name}=${lib.escapeShellArg cfg.repo.owner}" + "export ${fetcher.environmentVariables.PR_TRACKER_FETCHER_GITHUB_REPO_NAME.name}=${lib.escapeShellArg cfg.repo.name}" + "export ${fetcher.environmentVariables.PR_TRACKER_FETCHER_BRANCH_PATTERNS.name}=${lib.escapeShellArg (builtins.toJSON cfg.branchPatterns)}" + "export ${fetcher.environmentVariables.PR_TRACKER_FETCHER_GITHUB_TOKEN.name}=$(< ${cfg.githubApiTokenFile})" + # CACHE_DIRECTORY is set by systemd based on the CacheDirectory setting. + # See https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html#RuntimeDirectory= + "export ${fetcher.environmentVariables.PR_TRACKER_FETCHER_CACHE_DIR.name}=$CACHE_DIRECTORY" + ] + ++ lib.optional (cfg.db.passwordFile != null) '' + PASSWORD=$(${lib.getExe pkgs.urlencode} --encode-set component < ${cfg.db.passwordFile}) + ${fetcher.environmentVariables.PR_TRACKER_FETCHER_DATABASE_URL.name}+="&password=$PASSWORD" + '' + ++ [ "exec ${lib.getExe cfg.package}" ] + ); systemd.services.pr-tracker-fetcher.serviceConfig.User = cfg.user; systemd.services.pr-tracker-fetcher.serviceConfig.Group = cfg.group; diff --git a/modules/formatting.nix b/modules/formatting.nix index 7a8e0b12..814f9bd2 100644 --- a/modules/formatting.nix +++ b/modules/formatting.nix @@ -4,10 +4,7 @@ inputs.treefmt-nix.flakeModule ]; perSystem = { - pre-commit.settings.hooks.nix-fmt = { - enable = true; - entry = "nix fmt -- --fail-on-change"; - }; + pre-commit.settings.hooks.treefmt.enable = true; treefmt = { projectRootFile = "flake.nix"; diff --git a/modules/introduction.nix b/modules/introduction.nix new file mode 100644 index 00000000..85758c73 --- /dev/null +++ b/modules/introduction.nix @@ -0,0 +1,46 @@ +{ + perSystem = + { + pkgs, + ... + }: + { + chapters.introduction = { + title = "Introduction"; + drv = pkgs.writeTextFile { + name = "introduction.md"; + text = + # markdown + '' + ## What + + This is a system that provides answers to the following question: + + > Which branches did a certain GitHub pull request "land" in? + + A pull request is considered landed in a branch when that branch contains that pull request's "merge commit"[^1]. + + As of 2024-02-02 GitHub does not provide an API for directly asking this question. + + ## How + + In order to provide an API that answers this question the following are obtained: + + 1. All pull requests and their merge commits (via GitHub's GraphQL API). + 2. A clone of the repository. + + From these, all landings are deduced and stored in a (PostgreSQL) database. + + It is assumed that in the tracked branches, history is never rewritten. + + Two programs are provided: + + 1. pr-tracker-fetcher: obtains data, determines landings and persists them. + 2. pr-tracker-api: provides an HTTP endpoint for querying landings. + + [^1]: Note that in GitHub, a pull request has a "merge commit" even having been merged without an actual merge commit. + ''; + }; + }; + }; +} diff --git a/modules/manual.nix b/modules/manual.nix new file mode 100644 index 00000000..7bb78173 --- /dev/null +++ b/modules/manual.nix @@ -0,0 +1,132 @@ +{ + lib, + self, + ... +}: +{ + perSystem = + psArgs@{ + self', + system, + pkgs, + ... + }: + let + filterOptions = import ../filterOptions.nix lib; + + configuration = lib.nixosSystem { + modules = [ + self.nixosModules.api + self.nixosModules.fetcher + { + nixpkgs.hostPlatform = system; + } + ]; + }; + + options = filterOptions ( + path: option: lib.any (lib.hasPrefix "${self}/") option.declarations + ) configuration.options; + + optionsDocs = pkgs.buildPackages.nixosOptionsDoc { + inherit options; + variablelistId = "options"; + transformOptions = options: builtins.removeAttrs options [ "declarations" ]; + }; + + optionsMd = + pkgs.runCommand "pr-tracker-nixos-options-html" { nativeBuildInputs = [ pkgs.nixos-render-docs ]; } + '' + nixos-render-docs options commonmark \ + --manpage-urls <(echo "{}") \ + --revision provide-because-required-but-seems-to-be-unused \ + ${optionsDocs.optionsJSON}/share/doc/nixos/options.json $out + ''; + + summaryMd = pkgs.writeTextFile { + name = "SUMMARY.md"; + text = + # markdown + '' + ${mkChapterLink psArgs.config.chapters.introduction} + + ${mkChapterLink psArgs.config.chapters.fetcher} + ${mkChapterLink psArgs.config.chapters.api} + + ${mkChapterLink psArgs.config.chapters.nixos} + + ${mkChapterLink psArgs.config.chapters.versioning} + ${mkChapterLink psArgs.config.chapters.prior-art} + ${mkChapterLink psArgs.config.chapters.vision} + ''; + }; + + mkChapterLink = { title, basename, ... }: "[${title}](${basename}.md)"; + in + { + options = { + chapters = lib.mkOption { + type = lib.types.lazyAttrsOf ( + lib.types.submodule ( + { name, ... }: + { + options = { + title = lib.mkOption { + type = lib.types.str; + }; + + basename = lib.mkOption { + type = lib.types.str; + default = name; + }; + + drv = lib.mkOption { + type = lib.types.package; + }; + }; + } + ) + ); + }; + }; + config = { + chapters = { + nixos = { + title = "NixOS Module"; + drv = optionsMd; + }; + }; + + packages.manual = + pkgs.runCommand "pr-tracker-nixos-manual" + { + nativeBuildInputs = [ + pkgs.mdbook + pkgs.coreutils + ]; + } + '' + mkdir src + + ln -s ${summaryMd} src/SUMMARY.md + + ${lib.pipe psArgs.config.chapters [ + (lib.mapAttrsToList ( + name: chapter: '' + ( + echo ${lib.escapeShellArg ("# " + chapter.title)} + echo "" + cat ${chapter.drv} + )> src/${chapter.basename}.md + '' + )) + lib.concatLines + ]} + + mdbook build --dest-dir $out + ''; + + checks."packages/manual" = self'.packages.manual; + }; + }; +} diff --git a/modules/nixos-manual/default.nix b/modules/nixos-manual/default.nix deleted file mode 100644 index 51363dee..00000000 --- a/modules/nixos-manual/default.nix +++ /dev/null @@ -1,76 +0,0 @@ -# https://gitlab.com/rycee/nur-expressions/-/blob/master/doc/default.nix -{ - lib, - inputs, - self, - ... -}: -{ - perSystem = - { - self', - system, - pkgs, - ... - }: - let - inherit (lib) - any - evalModules - hasPrefix - ; - - inherit (builtins) - removeAttrs - ; - - filterOptions = import ../../filterOptions.nix lib; - - configuration = evalModules { - modules = [ - self.nixosModules.api - self.nixosModules.fetcher - { - nixpkgs.hostPlatform = system; - system.stateVersion = "24.05"; - } - ] ++ (import "${inputs.nixpkgs}/nixos/modules/module-list.nix"); - }; - - options = filterOptions ( - path: option: any (hasPrefix "${self}/") option.declarations - ) configuration.options; - - optionsDocs = pkgs.buildPackages.nixosOptionsDoc { - inherit options; - variablelistId = "options"; - transformOptions = options: removeAttrs options [ "declarations" ]; - }; - in - { - packages.nixos-manual = pkgs.stdenv.mkDerivation { - name = "pr-tracker-nixos-modules-manual"; - src = ./.; - nativeBuildInputs = [ pkgs.nixos-render-docs ]; - - buildPhase = '' - mkdir $out - - manpage_urls=$(mktemp) - echo "{}" > $manpage_urls - - substituteInPlace ./manual.md \ - --subst-var-by \ - OPTIONS_JSON \ - ${optionsDocs.optionsJSON}/share/doc/nixos/options.json - - nixos-render-docs manual html \ - --manpage-urls $manpage_urls \ - --revision provide-because-required-but-seems-to-be-unused \ - manual.md $out/index.html - ''; - }; - - checks."packages/nixos-modules-manual" = self'.packages.nixos-manual; - }; -} diff --git a/modules/nixos-manual/manual.md b/modules/nixos-manual/manual.md deleted file mode 100644 index f8f9aea6..00000000 --- a/modules/nixos-manual/manual.md +++ /dev/null @@ -1,9 +0,0 @@ -# pr-tracker NixOS modules manual {#pr-tracker-nixos-modules-manual} - -## Options - -```{=include=} options -id-prefix: opt- -list-id: pr-tracker-nixos-modules-options -source: @OPTIONS_JSON@ -``` diff --git a/modules/nixos-modules-lib.nix b/modules/nixos-modules-lib.nix index afa6fbb4..f87af170 100644 --- a/modules/nixos-modules-lib.nix +++ b/modules/nixos-modules-lib.nix @@ -1,6 +1,7 @@ { lib, withSystem, + psqlConnectionUriMdLink, ... }: { @@ -22,7 +23,8 @@ db.urlParams = mkOption { type = types.nullOr (types.attrsOf types.str); description = '' - URL parameters from which to compose the ${builtins.readFile ../crates/DATABASE_URL.md} + URL parameters from which to compose the ${psqlConnectionUriMdLink}. + Required unless {option}`${options.services.pr-tracker.db.createLocally}` is true. ''; example = { diff --git a/modules/prior-art.nix b/modules/prior-art.nix new file mode 100644 index 00000000..31aaefcd --- /dev/null +++ b/modules/prior-art.nix @@ -0,0 +1,30 @@ +{ + perSystem = + { + pkgs, + ... + }: + { + chapters.prior-art = { + title = "Prior art"; + drv = pkgs.writeTextFile { + name = "prior-art.md"; + text = + # markdown + '' + - [Alyssa Ross' pr-tracker](https://nixpk.gs/pr-tracker.html) ([source](https://git.qyliss.net/pr-tracker)) + Server-side rendered web app. Computes landings for a given PR on the fly by invoking Git on the backend. + - [ocfox's nixpkgs-tracker](https://nixpkgs-tracker.ocfox.me/) ([source](https://github.com/ocfox/nixpkgs-tracker)) + Client-side rendered web app. Computes landings for a given PR on the fly using GitHub api. + - [Maralorn's nixpkgs-bot](https://blog.maralorn.de/projects#nixpkgs-bot) ([source](https://code.maralorn.de/maralorn/config/src/commit/b34d2e0d0adc62c30875edb475f1c09a752fe19e/packages/nixpkgs-bot)) + Matrix bot that provides notification of PR landings. + Periodically computes new PR landings using Git and sends messages. + + All of the above are [Nixpkgs](https://github.com/nixos/nixpkgs/) specific, whereas this project is not. + None of the above internally maintain a dataset of landings. + None of the above currently provide an HTTP API. + ''; + }; + }; + }; +} diff --git a/modules/private-nixos-modules/db.nix b/modules/private-nixos-modules/db.nix index 863d0741..421b63ed 100644 --- a/modules/private-nixos-modules/db.nix +++ b/modules/private-nixos-modules/db.nix @@ -95,9 +95,10 @@ { config = mkIf (cfg.db.createLocally && hasAttr program cfg && programCfg.enable) { services.postgresql.ensureUsers = [ { name = programCfg.user; } ]; - systemd.services.postgresql.postStart = lib.mkAfter '' - $PSQL '${cfg.db.name}' -c 'GRANT "${cfg.db.name}" TO "${programCfg.user}"' - $PSQL '${cfg.db.name}' -c 'ALTER DEFAULT PRIVILEGES FOR ROLE "${programCfg.user}" IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO "${cfg.db.name}"' + systemd.services.postgresql-setup.postStart = '' + psql '${cfg.db.name}' -c 'GRANT "${cfg.db.name}" TO "${programCfg.user}"' + psql '${cfg.db.name}' -c 'ALTER DEFAULT PRIVILEGES FOR ROLE "${programCfg.user}" IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO "${cfg.db.name}"' + psql '${cfg.db.name}' -c 'ALTER DEFAULT PRIVILEGES FOR ROLE "${programCfg.user}" IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO "${cfg.db.name}"' ''; }; } diff --git a/modules/program-docs.nix b/modules/program-docs.nix deleted file mode 100644 index e0b455a3..00000000 --- a/modules/program-docs.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ - perSystem = - { config, ... }: - { - packages.program-docs = config.nci.outputs.default.docs; - checks."packages/program-docs" = config.packages.program-docs; - nci.projects.default = { - fileset = ../crates/DATABASE_URL.md; - includeInProjectDocs = false; - }; - }; -} diff --git a/modules/release/default.nix b/modules/release/default.nix index d033e4f8..2e5d3b83 100644 --- a/modules/release/default.nix +++ b/modules/release/default.nix @@ -33,6 +33,21 @@ }; in { + chapters.versioning = { + title = "Versioning"; + drv = pkgs.writeTextFile { + name = "versioning.md"; + text = + # markdown + '' + - This project uses [Conventional Commits v1](https://www.conventionalcommits.org/en/v1.0.0/) and [Semantic Versioning v2](https://semver.org/spec/v2.0.0.html). + - With regard to versioning, the documented executables and NixOS modules are public. + The libraries are private. + - This project, with all its components, is versioned as one. + ''; + }; + }; + apps.bump-version = { type = "app"; program = lib.getExe bump-version; diff --git a/modules/util.nix b/modules/util.nix index f8fb8a33..8f7dc71f 100644 --- a/modules/util.nix +++ b/modules/util.nix @@ -1,7 +1,60 @@ +{ lib, prefixLines, ... }: { + _module.args = { + psqlConnectionUriMdLink = + # markdown + "[PostgreSQL connection URI](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING-URIS)"; + + prefixLines = + prefix: lines: + lib.pipe lines [ + (lib.splitString "\n") + (map (line: prefix + line)) + (lib.concatStringsSep "\n") + ]; + + environmentVariablesToMarkdown = variables: '' + | Name | Description | + |------|-------------| + ${lib.pipe variables [ + (lib.mapAttrsToList ( + name: envVar: "| `${name}` | ${lib.strings.replaceStrings [ "\n" ] [ "
" ] envVar.description} |" + )) + lib.concatLines + ]} + ''; + }; + perSystem = { pkgs, ... }: { + _module.args.writeEnvironmentStructFile = + crateName: environmentVariables: + let + fields = lib.pipe environmentVariables [ + (lib.mapAttrsToList ( + envName: envSpec: '' + ${prefixLines "/// " envSpec.description} + #[config(env = "${envName}")] + pub ${envName}: ${envSpec.rustType},'' + )) + (lib.concatStringsSep "\n") + ]; + in + + pkgs.writeTextFile { + name = "${crateName}-env-struct.rs"; + text = + # rust + '' + /// See documentation for each field. + #[derive(::std::fmt::Debug, ::confique::Config)] + pub struct Environment { + ${prefixLines " " fields} + } + ''; + }; + nci.projects.default.fileset = ../crates/util/migrations; treefmt.settings.global.excludes = [ "crates/util/migrations/*" ]; devshells.default.commands = [ diff --git a/modules/vision.nix b/modules/vision.nix new file mode 100644 index 00000000..9fb88665 --- /dev/null +++ b/modules/vision.nix @@ -0,0 +1,52 @@ +{ + perSystem = + { + pkgs, + ... + }: + { + chapters.vision = { + title = "Vision"; + drv = pkgs.writeTextFile { + name = "vision.md"; + text = + # markdown + '' + ## Push-driven updates + + The current architecture of obtaining data via polling allows instantaneous + and hopefully reliable responses. + However, the data can be stale. + + In the future, we intend to provide fresher data by subscribing to GitHub webhooks. + + Since the public cannot subscribe to GitHub webhooks, + this will require deployment by the repo owner. + + ## Event record keeping + + Building upon the implementation of push-driven updates, + we intend to keep track of _when_ PRs land in branches. + This requires a dataset of landings. + + ## Webhook service + + We intend to allow users to subscribe to webhook notifications of PR landings. + This provides a couple of benefits over subscribing to GitHub webhooks directly: + + - GitHub webhooks can only notify when a PR lands in its target branch. They + cannot notify when that PR lands in other branches. + - Only repo owners can subscribe to GitHub webhooks. + + ## Backport PRs + + A backport PR is a re-application of another PR, targeting a different branch. + + We intend to adopt or invent a workflow whereby in backport PRs the original PR is declared. + Using that metadata, when providing landings for an original PR, + we intend to also include branches on which a backport PR had landed. + ''; + }; + }; + }; +}