{ "id": "82bbca6b325d938a2c1d6f38aee8413c4db3be4a76ed231b4f89355a4923f822", "pubkey": "a008def15796fba9a0d6fab04e8fd57089285d9fd505da5a83fe8aad57a3564d", "created_at": 1731672093, "kind": 1617, "tags": [ [ "a", "30617:a008def15796fba9a0d6fab04e8fd57089285d9fd505da5a83fe8aad57a3564d:ngit" ], [ "r", "26689f97810fc656c7134c76e2a37d33b2e40ce7" ], [ "r", "6bcb58925ad5a7ec2421718fb2996add9080f7bc" ], [ "alt", "git patch: feat(blossom): blossom as remote using packs" ], [ "t", "root" ], [ "branch-name", "blossom-remote-with-packs" ], [ "p", "a008def15796fba9a0d6fab04e8fd57089285d9fd505da5a83fe8aad57a3564d" ], [ "commit", "6bcb58925ad5a7ec2421718fb2996add9080f7bc" ], [ "parent-commit", "ae87aedae9696f4c855ac3dc47e61faec9d07c15" ], [ "commit-pgp-sig", "-----BEGIN PGP SIGNATURE-----\n\niQIzBAABCAAdFiEEsRaN4Kb3mvfPwNXpaOFUhtc/deEFAmc3Ny8ACgkQaOFUhtc/\ndeFlLg/+I72KbX6DG2TcohHU+f0nFpBr/Xfmh03h2Hnq8OM492O3bXsHz13GsATf\npzJdRonXmYmE5VqRHodYNcjlFeSannRqjg02ogegWxydzf3u5sGBeENAWNedvBZ7\nXG8bIBOoL2fgObbH7CEx16TgsNOxTNneuxLcT7/PDqhBeaNM3iZqwozEXVlsn+U6\nM8naR1rzcRoC5RYFACQQUti2+0+bsYAAekdrJrQGCR5DUqn+H+q1zxuMbDhFYUjd\n6AcI7pAglrKpYAY4EBHmFIy0u1CLi5uU6X46/wJOmK1u6J+05Jg/Wuq+RCJpkwHP\nHi3FoG4MsVGouYh0D1y6keotpriSs7CeRrJokvF7V3CJUm3gOEhgqEDK5Zzy2kR7\nEJnfhxDfmLHHsHYCZymtWqvSpbaGkWKTNbN2p1Sp3kSJreZ6Rru1cVpVJo8swXiI\nFHlBmQ+4T3JVlocmrf7vNATBEp/7XrDdxTuZ/Jl2EyyrPBG2MMq6DbyJEDBSfrcQ\n6ly7vVzEhl8HP+nQohoJDWARiv/RsjYwqzOliHtX/zo3RSxQNkT+lVn82dMnKCLd\na4yxS6H8BfLPEHSBYgsyUWjlEUJgX2k0or9TxEVZU0qFlw6sn+E7Tp6QLIxReznL\nrXxZ2i/sXtqtkTKf68ynmcTCKfFgGGf0ZnmLi6010Vv8/s6rU1E=\n=OuMT\n-----END PGP SIGNATURE-----" ], [ "description", "feat(blossom): blossom as remote using packs\n\nThis is a WIP exploration of the use of blossom as an optional\nalternative to using a git server.\n\nThe incomplete code focuses on how blossom could fit with nip34\nto most efficently replace the git server. It is missing the actual\nblossom interaction which would hopefully would be facilited by\na new blossom feature in rust-nostr.\n\nThis implementation tries to minimise the number of blobs required\nfor download by using packs.\n\nIf a branch tip is at height 1304 it will split the commits in into\na number of packs. a pack the first 1024 commits, the next 256, the\nnext 16 and the final 8.\n\nI planned for the identification of blossom servers to mirror the\napproach taken for relays:\n 1. list repository blossom servers in repo announcement event\n kind 30617\n 2. also push to user blossom servers in the standard event for that\nThis is not implemented, along with the rest of the blossom aspects.\n\nI'm publishing this now as\nnostr:npub1elta7cneng3w8p9y4dw633qzdjr4kyvaparuyuttyrx6e8xp7xnq32cume\nhas recently published a POC of an alternative approach and it makes\nsense to this alternative idea.\n" ], [ "author", "DanConwayDev", "DanConwayDev@protonmail.com", "1731671830", "0" ], [ "committer", "DanConwayDev", "DanConwayDev@protonmail.com", "1731671830", "0" ] ], "content": "From 6bcb58925ad5a7ec2421718fb2996add9080f7bc Mon Sep 17 00:00:00 2001\nFrom: DanConwayDev \u003cDanConwayDev@protonmail.com\u003e\nDate: Fri, 15 Nov 2024 11:57:10 +0000\nSubject: [PATCH] feat(blossom): blossom as remote using packs\n\nThis is a WIP exploration of the use of blossom as an optional\nalternative to using a git server.\n\nThe incomplete code focuses on how blossom could fit with nip34\nto most efficently replace the git server. It is missing the actual\nblossom interaction which would hopefully would be facilited by\na new blossom feature in rust-nostr.\n\nThis implementation tries to minimise the number of blobs required\nfor download by using packs.\n\nIf a branch tip is at height 1304 it will split the commits in into\na number of packs. a pack the first 1024 commits, the next 256, the\nnext 16 and the final 8.\n\nI planned for the identification of blossom servers to mirror the\napproach taken for relays:\n 1. list repository blossom servers in repo announcement event\n kind 30617\n 2. also push to user blossom servers in the standard event for that\nThis is not implemented, along with the rest of the blossom aspects.\n\nI'm publishing this now as\nnostr:npub1elta7cneng3w8p9y4dw633qzdjr4kyvaparuyuttyrx6e8xp7xnq32cume\nhas recently published a POC of an alternative approach and it makes\nsense to this alternative idea.\n---\n Cargo.lock | 1 +\n Cargo.toml | 1 +\n src/bin/git_remote_nostr/fetch.rs | 4 ++++\n src/bin/git_remote_nostr/list.rs | 23 ++++++++++++++++++++++-\n src/bin/git_remote_nostr/push.rs | 124 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----\n src/lib/repo_state.rs | 17 ++++++++++++++++-\n 6 files changed, 163 insertions(+), 7 deletions(-)\n\ndiff --git a/Cargo.lock b/Cargo.lock\nindex b20b60a..72b37a2 100644\n--- a/Cargo.lock\n+++ b/Cargo.lock\n@@ -1805,6 +1805,7 @@ dependencies = [\n \"serde_json\",\n \"serde_yaml\",\n \"serial_test\",\n+ \"sha2\",\n \"test_utils\",\n \"tokio\",\n \"urlencoding\",\ndiff --git a/Cargo.toml b/Cargo.toml\nindex ed99aea..320a9f0 100644\n--- a/Cargo.toml\n+++ b/Cargo.toml\n@@ -38,6 +38,7 @@ serde_yaml = \"0.9.27\"\n tokio = \"1.33.0\"\n urlencoding = \"2.1.3\"\n zeroize = \"1.6.0\"\n+sha2 = \"0.10.8\"\n \n [dev-dependencies]\n assert_cmd = \"2.0.12\"\ndiff --git a/src/bin/git_remote_nostr/fetch.rs b/src/bin/git_remote_nostr/fetch.rs\nindex a972a2f..a1116c5 100644\n--- a/src/bin/git_remote_nostr/fetch.rs\n+++ b/src/bin/git_remote_nostr/fetch.rs\n@@ -49,6 +49,10 @@ pub async fn run_fetch(\n let term = console::Term::stderr();\n \n for git_server_url in \u0026repo_ref.git_server {\n+ if git_server_url.eq(\"blossom\") {\n+ // TODO download missing blobs\n+ continue;\n+ }\n let term = console::Term::stderr();\n if let Err(error) = fetch_from_git_server(\n git_repo,\ndiff --git a/src/bin/git_remote_nostr/list.rs b/src/bin/git_remote_nostr/list.rs\nindex 92faa6b..d71c2d1 100644\n--- a/src/bin/git_remote_nostr/list.rs\n+++ b/src/bin/git_remote_nostr/list.rs\n@@ -43,7 +43,28 @@ pub async fn run_list(\n \n let term = console::Term::stderr();\n \n- let remote_states = list_from_remotes(\u0026term, git_repo, \u0026repo_ref.git_server, decoded_nostr_url);\n+ let mut remote_states = list_from_remotes(\n+ \u0026term,\n+ git_repo,\n+ \u0026repo_ref\n+ .git_server\n+ .iter()\n+ // blossom will always match nostr state\n+ .filter(|s| !s.starts_with(\"blossom\"))\n+ .map(std::borrow::ToOwned::to_owned)\n+ .collect::\u003cVec\u003cString\u003e\u003e(),\n+ decoded_nostr_url,\n+ );\n+ if repo_ref.git_server.iter().any(|s| s.eq(\"blossom\")) {\n+ if let Some(nostr_state) = nostr_state.clone() {\n+ remote_states.insert(\"blossom\".to_owned(), nostr_state.state.clone());\n+ } else if let Some((_, state)) = remote_states.iter().last() {\n+ remote_states.insert(\"blossom\".to_owned(), state.clone());\n+ } else {\n+ // create blank state if no nostr state exists yet\n+ remote_states.insert(\"blossom\".to_owned(), HashMap::new());\n+ }\n+ }\n \n let mut state = if let Some(nostr_state) = nostr_state {\n for (name, value) in \u0026nostr_state.state {\ndiff --git a/src/bin/git_remote_nostr/push.rs b/src/bin/git_remote_nostr/push.rs\nindex db86c04..a12e8ba 100644\n--- a/src/bin/git_remote_nostr/push.rs\n+++ b/src/bin/git_remote_nostr/push.rs\n@@ -2,6 +2,7 @@ use core::str;\n use std::{\n collections::{HashMap, HashSet},\n io::Stdin,\n+ str::FromStr,\n sync::{Arc, Mutex},\n time::Instant,\n };\n@@ -11,7 +12,7 @@ use auth_git2::GitAuthenticator;\n use client::{get_events_from_cache, get_state_from_cache, send_events, sign_event, STATE_KIND};\n use console::Term;\n use git::{sha1_to_oid, RepoActions};\n-use git2::{Oid, Repository};\n+use git2::{Buf, Commit, Oid, Repository};\n use git_events::{\n generate_cover_letter_and_patch_events, generate_patch_event, get_commit_id_from_patch,\n };\n@@ -29,11 +30,17 @@ use ngit::{\n };\n use nostr::nips::nip10::Marker;\n use nostr_sdk::{\n- hashes::sha1::Hash as Sha1Hash, Event, EventBuilder, EventId, Kind, PublicKey, Tag,\n+ hashes::{\n+ hex::DisplayHex,\n+ sha1::Hash as Sha1Hash,\n+ sha256::{self, Hash as Sha256Hash},\n+ },\n+ Event, EventBuilder, EventId, Kind, PublicKey, Tag,\n };\n use nostr_signer::NostrSigner;\n use repo_ref::RepoRef;\n use repo_state::RepoState;\n+use sha2::{Digest, Sha256};\n \n use crate::{\n client::Client,\n@@ -74,7 +81,17 @@ pub async fn run_push(\n \n let list_outputs = match list_outputs {\n Some(outputs) =\u003e outputs,\n- _ =\u003e list_from_remotes(\u0026term, git_repo, \u0026repo_ref.git_server, decoded_nostr_url),\n+ _ =\u003e list_from_remotes(\n+ \u0026term,\n+ git_repo,\n+ \u0026repo_ref\n+ .git_server\n+ .iter()\n+ .filter(|s| !s.eq(\u0026\"blossom\"))\n+ .map(std::string::ToString::to_string)\n+ .collect(),\n+ decoded_nostr_url,\n+ ),\n };\n \n let nostr_state = get_state_from_cache(git_repo.get_path()?, repo_ref).await;\n@@ -150,11 +167,24 @@ pub async fn run_push(\n }\n }\n \n+ let mut blossom_packs: Option\u003cHashMap\u003csha256::Hash, Buf\u003e\u003e = None;\n if !git_server_refspecs.is_empty() {\n let new_state = generate_updated_state(git_repo, \u0026existing_state, \u0026git_server_refspecs)?;\n+ let blossom_hashes = if repo_ref.git_server.contains(\u0026\"blossom\".to_string()) {\n+ let (blossom_hashes, packs) = create_blossom_packs(\u0026new_state, git_repo)?;\n+ blossom_packs = Some(packs);\n+ blossom_hashes\n+ } else {\n+ HashSet::new()\n+ };\n \n- let new_repo_state =\n- RepoState::build(repo_ref.identifier.clone(), new_state, \u0026signer).await?;\n+ let new_repo_state = RepoState::build(\n+ repo_ref.identifier.clone(),\n+ new_state,\n+ blossom_hashes,\n+ \u0026signer,\n+ )\n+ .await?;\n \n events.push(new_repo_state.event);\n \n@@ -325,6 +355,13 @@ pub async fn run_push(\n \n // TODO make async - check gitlib2 callbacks work async\n \n+ if let Some(packs) = blossom_packs {\n+ // TODO: upload blossom packs\n+ for (_hash, _pack) in packs {\n+ // blossom::upload(pack)\n+ }\n+ }\n+\n for (git_server_url, remote_refspecs) in remote_refspecs {\n let remote_refspecs = remote_refspecs\n .iter()\n@@ -863,6 +900,71 @@ fn generate_updated_state(\n Ok(new_state)\n }\n \n+fn create_blossom_packs(\n+ state: \u0026HashMap\u003cString, String\u003e,\n+ git_repo: \u0026Repo,\n+) -\u003e Result\u003c(HashSet\u003csha256::Hash\u003e, HashMap\u003csha256::Hash, Buf\u003e)\u003e {\n+ let mut blossom_hashes = HashSet::new();\n+ let mut blossom_packs = HashMap::new();\n+ for commit_id in state.values() {\n+ if let Ok(oid) = Oid::from_str(commit_id) {\n+ if let Ok(commit) = git_repo.git_repo.find_commit(oid) {\n+ let height = get_height(\u0026commit, git_repo)?;\n+ let mut revwalk = git_repo.git_repo.revwalk()?;\n+ revwalk.push(oid)?;\n+ let mut counter = 0;\n+ for pack_size in split_into_powers_of_2(height) {\n+ let mut pack = git_repo.git_repo.packbuilder()?;\n+ while counter \u003c pack_size {\n+ if let Some(oid) = revwalk.next() {\n+ pack.insert_commit(oid?)?;\n+ counter += 1;\n+ }\n+ }\n+ let mut buffer = Buf::new();\n+ pack.write_buf(\u0026mut buffer)?;\n+ let hash = buffer_to_sha256_hash(\u0026buffer);\n+ blossom_hashes.insert(hash);\n+ blossom_packs.insert(hash, buffer);\n+ counter = 0;\n+ }\n+ }\n+ }\n+ }\n+ Ok((blossom_hashes, blossom_packs))\n+}\n+\n+fn get_height(commit: \u0026Commit, git_repo: \u0026Repo) -\u003e Result\u003cu32\u003e {\n+ let mut revwalk = git_repo.git_repo.revwalk()?;\n+ revwalk.push(commit.id())?;\n+ Ok(u32::try_from(revwalk.count())?)\n+}\n+\n+fn split_into_powers_of_2(height: u32) -\u003e Vec\u003cu32\u003e {\n+ let mut powers = Vec::new();\n+ let mut remaining = height;\n+\n+ // Decompose the height into powers of 2\n+ for i in (0..32).rev() {\n+ let power = 1 \u003c\u003c i; // Calculate 2^i\n+ while remaining \u003e= power {\n+ powers.push(power);\n+ remaining -= power;\n+ }\n+ }\n+\n+ powers\n+}\n+\n+fn buffer_to_sha256_hash(buffer: \u0026Buf) -\u003e sha256::Hash {\n+ let mut hasher = Sha256::new();\n+ hasher.update(buffer.as_ref());\n+ let hash = hasher\n+ .finalize()\n+ .to_hex_string(nostr_sdk::hashes::hex::Case::Lower);\n+ sha256::Hash::from_str(\u0026hash).unwrap()\n+}\n+\n async fn get_merged_status_events(\n term: \u0026console::Term,\n repo_ref: \u0026RepoRef,\n@@ -1186,6 +1288,7 @@ trait BuildRepoState {\n async fn build(\n identifier: String,\n state: HashMap\u003cString, String\u003e,\n+ blossom: HashSet\u003cSha256Hash\u003e,\n signer: \u0026NostrSigner,\n ) -\u003e Result\u003cRepoState\u003e;\n }\n@@ -1193,6 +1296,7 @@ impl BuildRepoState for RepoState {\n async fn build(\n identifier: String,\n state: HashMap\u003cString, String\u003e,\n+ blossom: HashSet\u003cSha256Hash\u003e,\n signer: \u0026NostrSigner,\n ) -\u003e Result\u003cRepoState\u003e {\n let mut tags = vec![Tag::identifier(identifier.clone())];\n@@ -1202,10 +1306,20 @@ impl BuildRepoState for RepoState {\n vec![value.clone()],\n ));\n }\n+ if !blossom.is_empty() {\n+ tags.push(Tag::custom(\n+ nostr_sdk::TagKind::Custom(\"blossom\".into()),\n+ blossom\n+ .iter()\n+ .map(std::string::ToString::to_string)\n+ .collect::\u003cVec\u003cString\u003e\u003e(),\n+ ));\n+ }\n let event = sign_event(EventBuilder::new(STATE_KIND, \"\", tags), signer).await?;\n Ok(RepoState {\n identifier,\n state,\n+ blossom,\n event,\n })\n }\ndiff --git a/src/lib/repo_state.rs b/src/lib/repo_state.rs\nindex c3a7606..19e78b6 100644\n--- a/src/lib/repo_state.rs\n+++ b/src/lib/repo_state.rs\n@@ -1,11 +1,17 @@\n-use std::collections::HashMap;\n+use std::{\n+ collections::{HashMap, HashSet},\n+ str::FromStr,\n+};\n \n use anyhow::{Context, Result};\n use git2::Oid;\n+use nostr_sdk::hashes::sha256::Hash;\n \n+#[derive(Clone)]\n pub struct RepoState {\n pub identifier: String,\n pub state: HashMap\u003cString, String\u003e,\n+ pub blossom: HashSet\u003cHash\u003e,\n pub event: nostr::Event,\n }\n \n@@ -14,6 +20,7 @@ impl RepoState {\n state_events.sort_by_key(|e| e.created_at);\n let event = state_events.first().context(\"no state events\")?;\n let mut state = HashMap::new();\n+ let mut blossom = HashSet::new();\n for tag in event.tags.iter() {\n if let Some(name) = tag.as_slice().first() {\n if [\"refs/heads/\", \"refs/tags\", \"HEAD\"]\n@@ -26,6 +33,13 @@ impl RepoState {\n }\n }\n }\n+ if name.eq(\"blossom\") {\n+ for s in tag.clone().to_vec() {\n+ if let Ok(hash) = Hash::from_str(\u0026s) {\n+ blossom.insert(hash);\n+ }\n+ }\n+ }\n }\n }\n Ok(RepoState {\n@@ -35,6 +49,7 @@ impl RepoState {\n .context(\"existing event must have an identifier\")?\n .to_string(),\n state,\n+ blossom,\n event: event.clone(),\n })\n }\n--\nlibgit2 1.8.1\n\n", "sig": "15ecda7351732241c3a86ab0113e0691e43afc3c5d60a1e9f3828b3c92d16164f4fd11cacfca8e660a40d38c48472282c6361e3cf2e4956c4257f8e2e68f7e86" }