Skip to content

Commit a897ef7

Browse files
feat(pacquet): support custom fetchers from pnpmfile (#12846)
Support custom fetchers from pnpmfiles in pacquet, with delegate-envelope parity in the TypeScript CLI (related to #11685) Pacquet already supported custom resolvers via its Node.js worker IPC but lacked the fetcher counterpart. This extends the same protocol pattern: - Define a CustomFetcher trait and get_custom_fetchers() on PnpmfileHooks - Add fetchers/fetcher target dispatch to the worker's NDJSON protocol and JS runner, mirroring the existing resolver path; the hook is invoked with the TS-parity args fetch(cafs, resolution, opts, fetchers) (cafs/fetchers are null over IPC) - Implement NodeJsCustomFetcher bridging the trait to the worker - Create CustomFetcherPicker for consulting fetchers in declared order - Consult custom fetchers before the built-in dispatch on both the frozen-lockfile and fresh-lockfile install paths; hook-load failures abort the install (PNPMFILE_FAIL) - Support delegation: { delegate: <resolution> } rewrites the resolution for the standard tarball/git path; non-delegate responses and custom-typed delegates fail the install - lockfile: add LockfileResolution::Custom preserving custom-typed resolution objects verbatim, so custom resolvers/fetchers can round-trip them; unclaimed custom-typed resolutions fail with the TS-parity UNSUPPORTED_RESOLUTION_TYPE error - TypeScript: pickFetcher now accepts the { delegate: <resolution> } envelope from custom fetchers (the portable delegation form), and hooks.types exports CustomFetcherDelegation Full CAS fetch in pacquet (where a fetcher produces file content directly rather than delegating) requires a streaming protocol extension (separate follow-up).
1 parent 5333a25 commit a897ef7

31 files changed

Lines changed: 1800 additions & 32 deletions

File tree

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
"@pnpm/hooks.types": minor
3+
"@pnpm/fetching.pick-fetcher": minor
4+
"pnpm": minor
5+
---
6+
7+
Custom fetchers exported from a pnpmfile can now delegate by returning a `{ delegate: <resolution> }` envelope: pnpm rewrites the package's resolution to the delegated shape and runs the built-in fetcher on it. This is the portable delegation form that also works in pacquet, where `cafs` and `fetchers` cannot be passed to the hook. Related to [pnpm/pnpm#11685](https://github.com/pnpm/pnpm/issues/11685).

pacquet/crates/cli/src/cli_args/cat_index.rs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,12 @@ fn metadata_store_index_keys(pkg_id: &str, metadata: &PackageMetadata) -> Vec<St
177177
LockfileResolution::Binary(resolution) => {
178178
vec![store_index_key(&resolution.integrity.to_string(), pkg_id)]
179179
}
180-
LockfileResolution::Directory(_) | LockfileResolution::Variations(_) => Vec::new(),
180+
// Custom resolutions delegate to a rewritten resolution at
181+
// install time; the store-index key belongs to that rewritten
182+
// shape, which isn't recoverable from the lockfile entry.
183+
LockfileResolution::Directory(_)
184+
| LockfileResolution::Variations(_)
185+
| LockfileResolution::Custom(_) => Vec::new(),
181186
}
182187
}
183188

pacquet/crates/cli/src/cli_args/install.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1042,9 +1042,12 @@ fn rewrite_resolution_registry(
10421042
rewrite_resolution_registry(&mut variant.resolution, rewrite);
10431043
}
10441044
}
1045+
// Custom resolutions are opaque — the benchmark rewrite can't
1046+
// know which of their fields (if any) is a registry URL.
10451047
LockfileResolution::Directory(_)
10461048
| LockfileResolution::Git(_)
1047-
| LockfileResolution::Registry(_) => {}
1049+
| LockfileResolution::Registry(_)
1050+
| LockfileResolution::Custom(_) => {}
10481051
}
10491052
}
10501053

pacquet/crates/cli/src/cli_args/switch_cli_version.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,8 @@ fn assert_integrity_only_resolution(
284284
| LockfileResolution::Directory(_)
285285
| LockfileResolution::Git(_)
286286
| LockfileResolution::Binary(_)
287-
| LockfileResolution::Variations(_) => Err(invalid_package_manager_lockfile(key)),
287+
| LockfileResolution::Variations(_)
288+
| LockfileResolution::Custom(_) => Err(invalid_package_manager_lockfile(key)),
288289
}
289290
}
290291

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
//! Custom fetchers from a `.pnpmfile.cjs` `fetchers` export, end to
2+
//! end: a custom resolver writes a custom-typed resolution into the
3+
//! lockfile, and the sibling fetcher claims it and delegates to the
4+
//! built-in tarball path — on the fresh-resolve install, on a frozen
5+
//! reinstall, and failing loudly when the fetcher is removed.
6+
7+
use assert_cmd::prelude::*;
8+
use command_extra::CommandExtra;
9+
use pacquet_testing_utils::bin::{AddMockedRegistry, CommandTempCwd};
10+
use std::{fs, path::Path, process::Command};
11+
12+
fn pacquet_at(workspace: &Path) -> Command {
13+
Command::cargo_bin("pacquet").expect("find the pacquet binary").with_current_dir(workspace)
14+
}
15+
16+
fn write_manifest(workspace: &Path) {
17+
let manifest = serde_json::json!({
18+
"dependencies": {
19+
"@pnpm.e2e/dep-of-pkg-with-1-dep": "100.0.0",
20+
},
21+
});
22+
fs::write(workspace.join("package.json"), manifest.to_string()).expect("write package.json");
23+
}
24+
25+
/// A resolver that claims `@pnpm.e2e/dep-of-pkg-with-1-dep` and
26+
/// resolves it to a `custom:e2e` resolution carrying the registry's
27+
/// real tarball URL and integrity, plus a fetcher that delegates that
28+
/// resolution back to the built-in tarball path. The fetcher uses the
29+
/// TS-parity hook signature `fetch(cafs, resolution, opts, fetchers)`.
30+
fn custom_type_pnpmfile(registry_url: &str, with_fetcher: bool) -> String {
31+
let fetchers = if with_fetcher {
32+
r"
33+
fetchers: [
34+
{
35+
canFetch (pkgId, resolution) {
36+
return resolution.type === 'custom:e2e';
37+
},
38+
fetch (cafs, resolution, opts, fetchers) {
39+
return { delegate: { tarball: resolution.url, integrity: resolution.integrity } };
40+
},
41+
},
42+
],
43+
"
44+
} else {
45+
""
46+
};
47+
format!(
48+
r"module.exports = {{
49+
resolvers: [
50+
{{
51+
canResolve (wanted) {{
52+
return wanted.alias === '@pnpm.e2e/dep-of-pkg-with-1-dep';
53+
}},
54+
async resolve () {{
55+
const response = await fetch('{registry_url}@pnpm.e2e%2Fdep-of-pkg-with-1-dep');
56+
const meta = await response.json();
57+
const picked = meta.versions['100.1.0'];
58+
return {{
59+
id: '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0',
60+
manifest: picked,
61+
resolution: {{
62+
type: 'custom:e2e',
63+
url: picked.dist.tarball,
64+
integrity: picked.dist.integrity,
65+
}},
66+
}};
67+
}},
68+
}},
69+
],{fetchers}
70+
}}
71+
",
72+
)
73+
}
74+
75+
fn installed_version(workspace: &Path) -> String {
76+
let manifest_path = workspace.join("node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep/package.json");
77+
let manifest: serde_json::Value =
78+
serde_json::from_str(&fs::read_to_string(manifest_path).expect("read installed manifest"))
79+
.expect("parse installed manifest");
80+
manifest["version"].as_str().expect("version is a string").to_string()
81+
}
82+
83+
#[test]
84+
fn custom_fetcher_delegates_a_custom_typed_resolution_on_fresh_and_frozen_installs() {
85+
let CommandTempCwd { pacquet, root, workspace, npmrc_info, .. } =
86+
CommandTempCwd::init().add_mocked_registry();
87+
let AddMockedRegistry { mock_instance, .. } = npmrc_info;
88+
89+
write_manifest(&workspace);
90+
fs::write(workspace.join(".pnpmfile.cjs"), custom_type_pnpmfile(&mock_instance.url(), true))
91+
.expect("write pnpmfile");
92+
93+
// Fresh resolve: the custom resolver writes the custom-typed
94+
// resolution, and the fetcher must delegate it during the same
95+
// install's fetch phase.
96+
pacquet.with_arg("install").assert().success();
97+
assert_eq!(installed_version(&workspace), "100.1.0");
98+
let lockfile = fs::read_to_string(workspace.join("pnpm-lock.yaml")).expect("read lockfile");
99+
assert!(
100+
lockfile.contains("type: custom:e2e"),
101+
"lockfile records the custom-typed resolution: {lockfile}",
102+
);
103+
104+
// Frozen reinstall: the lockfile is the source of truth now, so the
105+
// fetcher (loaded by the frozen path) is the only way to
106+
// materialize the custom-typed entry.
107+
fs::remove_dir_all(workspace.join("node_modules")).expect("remove node_modules");
108+
pacquet_at(&workspace).with_arg("install").assert().success();
109+
assert_eq!(installed_version(&workspace), "100.1.0");
110+
111+
drop((root, mock_instance)); // cleanup
112+
}
113+
114+
#[test]
115+
fn custom_typed_resolution_without_a_fetcher_fails_the_install() {
116+
let CommandTempCwd { pacquet, root, workspace, npmrc_info, .. } =
117+
CommandTempCwd::init().add_mocked_registry();
118+
let AddMockedRegistry { mock_instance, .. } = npmrc_info;
119+
120+
write_manifest(&workspace);
121+
fs::write(workspace.join(".pnpmfile.cjs"), custom_type_pnpmfile(&mock_instance.url(), false))
122+
.expect("write pnpmfile");
123+
124+
let output = pacquet.with_arg("install").assert().failure();
125+
let stderr = String::from_utf8_lossy(&output.get_output().stderr).into_owned();
126+
assert!(
127+
stderr.contains(r#"Cannot fetch dependency with custom resolution type "custom:e2e""#),
128+
"stderr: {stderr}",
129+
);
130+
131+
drop((root, mock_instance)); // cleanup
132+
}
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
use std::sync::Arc;
2+
3+
use serde_json::Value;
4+
5+
use crate::{CustomFetcher, HookError};
6+
7+
/// Adapts a slice of [`CustomFetcher`] instances to a single "pick fetcher"
8+
/// call: iterate the custom fetchers in declared order, return the first
9+
/// one that claims the package via `can_fetch`.
10+
///
11+
/// This mirrors the TypeScript `pickFetcher` logic in
12+
/// `pnpm11/fetching/pick-fetcher/src/index.ts`, where custom fetchers are
13+
/// tried before built-in fetchers.
14+
pub struct CustomFetcherPicker {
15+
fetchers: Vec<Arc<dyn CustomFetcher>>,
16+
}
17+
18+
impl CustomFetcherPicker {
19+
#[must_use]
20+
pub fn new(fetchers: Vec<Arc<dyn CustomFetcher>>) -> Self {
21+
Self { fetchers }
22+
}
23+
24+
#[must_use]
25+
pub fn is_empty(&self) -> bool {
26+
self.fetchers.is_empty()
27+
}
28+
29+
/// Consult each custom fetcher's `can_fetch` in declared order. Returns
30+
/// `Some(fetch_result)` from the first fetcher that claims the package,
31+
/// or `None` if no custom fetcher handles it (the caller falls through to
32+
/// built-in fetchers).
33+
pub async fn try_fetch(
34+
&self,
35+
pkg_id: &str,
36+
resolution: &Value,
37+
opts: &Value,
38+
) -> Result<Option<Value>, HookError> {
39+
for fetcher in &self.fetchers {
40+
if !fetcher.has_can_fetch() || !fetcher.has_fetch() {
41+
continue;
42+
}
43+
if fetcher.can_fetch(pkg_id, resolution.clone()).await? {
44+
let result = fetcher.fetch(pkg_id, resolution.clone(), opts.clone()).await?;
45+
return Ok(Some(result));
46+
}
47+
}
48+
Ok(None)
49+
}
50+
}
51+
52+
#[cfg(test)]
53+
mod tests;
Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
use std::sync::{
2+
Arc, Mutex,
3+
atomic::{AtomicUsize, Ordering},
4+
};
5+
6+
use async_trait::async_trait;
7+
use serde_json::{Value, json};
8+
9+
use super::CustomFetcherPicker;
10+
use crate::{CustomFetcher, HookError};
11+
12+
struct ScriptedFetcher {
13+
can_fetch: bool,
14+
response: Value,
15+
can_fetch_calls: AtomicUsize,
16+
fetch_calls: AtomicUsize,
17+
seen_pkg_ids: Mutex<Vec<String>>,
18+
seen_resolutions: Mutex<Vec<Value>>,
19+
seen_opts: Mutex<Vec<Value>>,
20+
}
21+
22+
impl ScriptedFetcher {
23+
fn answering(response: Value) -> Self {
24+
ScriptedFetcher {
25+
can_fetch: true,
26+
response,
27+
can_fetch_calls: AtomicUsize::new(0),
28+
fetch_calls: AtomicUsize::new(0),
29+
seen_pkg_ids: Mutex::new(Vec::new()),
30+
seen_resolutions: Mutex::new(Vec::new()),
31+
seen_opts: Mutex::new(Vec::new()),
32+
}
33+
}
34+
}
35+
36+
#[async_trait]
37+
impl CustomFetcher for ScriptedFetcher {
38+
async fn can_fetch(&self, pkg_id: &str, resolution: Value) -> Result<bool, HookError> {
39+
self.can_fetch_calls.fetch_add(1, Ordering::SeqCst);
40+
self.seen_pkg_ids.lock().unwrap().push(pkg_id.to_string());
41+
self.seen_resolutions.lock().unwrap().push(resolution);
42+
Ok(self.can_fetch)
43+
}
44+
45+
async fn fetch(
46+
&self,
47+
_pkg_id: &str,
48+
_resolution: Value,
49+
opts: Value,
50+
) -> Result<Value, HookError> {
51+
self.fetch_calls.fetch_add(1, Ordering::SeqCst);
52+
self.seen_opts.lock().unwrap().push(opts);
53+
Ok(self.response.clone())
54+
}
55+
}
56+
57+
fn fetch_result() -> Value {
58+
json!({
59+
"filesIndex": {
60+
"package.json": { "integrity": "sha512-abc123", "mode": 420 },
61+
"index.js": { "integrity": "sha512-def456", "mode": 420 },
62+
}
63+
})
64+
}
65+
66+
#[tokio::test]
67+
async fn returns_result_from_first_matching_fetcher() {
68+
let fetcher = Arc::new(ScriptedFetcher::answering(fetch_result()));
69+
let picker = CustomFetcherPicker::new(vec![Arc::clone(&fetcher) as Arc<dyn CustomFetcher>]);
70+
71+
let resolution = json!({ "type": "@custom/registry", "url": "https://example.com/pkg" });
72+
let opts = json!({ "manifest": { "name": "foo", "version": "1.0.0" } });
73+
74+
let result = picker.try_fetch("foo@1.0.0", &resolution, &opts).await.unwrap();
75+
76+
assert!(result.is_some());
77+
assert_eq!(result.unwrap(), fetch_result());
78+
assert_eq!(fetcher.can_fetch_calls.load(Ordering::SeqCst), 1);
79+
assert_eq!(fetcher.fetch_calls.load(Ordering::SeqCst), 1);
80+
}
81+
82+
#[tokio::test]
83+
async fn returns_none_when_no_fetcher_claims_package() {
84+
let fetcher = Arc::new(ScriptedFetcher {
85+
can_fetch: false,
86+
..ScriptedFetcher::answering(fetch_result())
87+
});
88+
let picker = CustomFetcherPicker::new(vec![Arc::clone(&fetcher) as Arc<dyn CustomFetcher>]);
89+
90+
let resolution = json!({ "type": "@custom/registry" });
91+
let opts = json!({});
92+
93+
let result = picker.try_fetch("foo@1.0.0", &resolution, &opts).await.unwrap();
94+
95+
assert!(result.is_none());
96+
assert_eq!(fetcher.can_fetch_calls.load(Ordering::SeqCst), 1);
97+
assert_eq!(fetcher.fetch_calls.load(Ordering::SeqCst), 0);
98+
}
99+
100+
#[tokio::test]
101+
async fn tries_fetchers_in_order_stops_at_first_match() {
102+
let first =
103+
Arc::new(ScriptedFetcher { can_fetch: false, ..ScriptedFetcher::answering(json!({})) });
104+
let second = Arc::new(ScriptedFetcher::answering(fetch_result()));
105+
let third = Arc::new(ScriptedFetcher::answering(json!({"other": true})));
106+
107+
let picker = CustomFetcherPicker::new(vec![
108+
Arc::clone(&first) as Arc<dyn CustomFetcher>,
109+
Arc::clone(&second) as Arc<dyn CustomFetcher>,
110+
Arc::clone(&third) as Arc<dyn CustomFetcher>,
111+
]);
112+
113+
let resolution = json!({ "tarball": "https://example.com/foo.tgz" });
114+
let opts = json!({});
115+
116+
let result = picker.try_fetch("foo@1.0.0", &resolution, &opts).await.unwrap();
117+
118+
assert_eq!(result.unwrap(), fetch_result());
119+
assert_eq!(first.can_fetch_calls.load(Ordering::SeqCst), 1);
120+
assert_eq!(first.fetch_calls.load(Ordering::SeqCst), 0);
121+
assert_eq!(second.can_fetch_calls.load(Ordering::SeqCst), 1);
122+
assert_eq!(second.fetch_calls.load(Ordering::SeqCst), 1);
123+
assert_eq!(
124+
third.can_fetch_calls.load(Ordering::SeqCst),
125+
0,
126+
"must not consult fetchers after a match",
127+
);
128+
}
129+
130+
#[tokio::test]
131+
async fn skips_fetcher_without_can_fetch() {
132+
let fetcher = Arc::new(NoCanFetchFetcher);
133+
let picker = CustomFetcherPicker::new(vec![fetcher]);
134+
135+
let result = picker.try_fetch("foo@1.0.0", &json!({}), &json!({})).await.unwrap();
136+
137+
assert!(result.is_none());
138+
}
139+
140+
struct NoCanFetchFetcher;
141+
142+
#[async_trait]
143+
impl CustomFetcher for NoCanFetchFetcher {
144+
fn has_can_fetch(&self) -> bool {
145+
false
146+
}
147+
148+
async fn can_fetch(&self, _: &str, _: Value) -> Result<bool, HookError> {
149+
panic!("should not be called");
150+
}
151+
152+
async fn fetch(&self, _: &str, _: Value, _: Value) -> Result<Value, HookError> {
153+
panic!("should not be called");
154+
}
155+
}
156+
157+
#[tokio::test]
158+
async fn empty_picker_returns_none() {
159+
let picker = CustomFetcherPicker::new(vec![]);
160+
assert!(picker.is_empty());
161+
162+
let result = picker.try_fetch("foo@1.0.0", &json!({}), &json!({})).await.unwrap();
163+
assert!(result.is_none());
164+
}

0 commit comments

Comments
 (0)