[RFC 140 2a] pkgs/by-name: Enforce for new packages

pkgs/by-name/README.md

@@ -1,11 +1,9 @@
 # Name-based package directories
 
 The structure of this directory maps almost directly to top-level package attributes.
-This is the recommended way to add new top-level packages to Nixpkgs [when possible](#limitations).
+Add new top-level packages to Nixpkgs using this mechanism [whenever possible](#limitations).
 
-Packages found in the named-based structure do not need to be explicitly added to the
-`top-level/all-packages.nix` file unless they require overriding the default value
-of an implicit attribute (see below).
+Packages found in the name-based structure are automatically included, without needing to be added to `all-packages.nix`. However if the implicit attribute defaults need to be changed for a package, this [must still be declared in `all-packages.nix`](#changing-implicit-attribute-defaults).
 
 ## Example
 

pkgs/test/nixpkgs-check-by-name/README.md

@@ -45,6 +45,8 @@ The current ratchets are:
 
 - New manual definitions of `pkgs.${name}` (e.g. in `pkgs/top-level/all-packages.nix`) with `args = { }`
   (see [nix evaluation checks](#nix-evaluation-checks)) must not be introduced.
+- New top-level packages defined using `pkgs.callPackage` must be defined with a package directory.
+  - Once a top-level package uses `pkgs/by-name`, it also can't be moved back out of it.
 
 ## Development
 

pkgs/test/nixpkgs-check-by-name/src/eval.nix

@@ -79,15 +79,37 @@ let
         };
       };
 
-  attrInfos = map (name: [
-    name
-    (
+  byNameAttrs = builtins.listToAttrs (map (name: {
+    inherit name;
+    value.ByName =
       if ! pkgs ? ${name} then
         { Missing = null; }
       else
-        { Existing = attrInfo name pkgs.${name}; }
-    )
-  ]) attrs;
+        { Existing = attrInfo name pkgs.${name}; };
+  }) attrs);
 
+  # Information on all attributes that exist but are not in pkgs/by-name.
+  # We need this to enforce pkgs/by-name for new packages
+  nonByNameAttrs = builtins.mapAttrs (name: value:
+    let
+      output = attrInfo name value;
+      result = builtins.tryEval (builtins.deepSeq output null);
+    in
+    {
+      NonByName =
+        if result.success then
+          { EvalSuccess = output; }
+        else
+          { EvalFailure = null; };
+    }
+  ) (builtins.removeAttrs pkgs attrs);
+
+  # All attributes
+  attributes = byNameAttrs // nonByNameAttrs;
 in
-attrInfos
+# We output them in the form [ [ <name> <value> ] ]` such that the Rust side
+# doesn't need to sort them again to get deterministic behavior (good for testing)
+map (name: [
+  name
+  attributes.${name}
+]) (builtins.attrNames attributes)

pkgs/test/nixpkgs-check-by-name/src/eval.rs

@@ -11,6 +11,21 @@ use std::process;
 use tempfile::NamedTempFile;
 
 /// Attribute set of this structure is returned by eval.nix
+#[derive(Deserialize)]
+enum Attribute {
+    /// An attribute that should be defined via pkgs/by-name
+    ByName(ByNameAttribute),
+    /// An attribute not defined via pkgs/by-name
+    NonByName(NonByNameAttribute),
+}
+
+#[derive(Deserialize)]
+enum NonByNameAttribute {
+    /// The attribute doesn't evaluate
+    EvalFailure,
+    EvalSuccess(AttributeInfo),
+}
+
 #[derive(Deserialize)]
 enum ByNameAttribute {
     /// The attribute doesn't exist at all
@@ -120,7 +135,7 @@ pub fn check_values(
         anyhow::bail!("Failed to run command {command:?}");
     }
     // Parse the resulting JSON value
-    let attributes: Vec<(String, ByNameAttribute)> = serde_json::from_slice(&result.stdout)
+    let attributes: Vec<(String, Attribute)> = serde_json::from_slice(&result.stdout)
         .with_context(|| {
             format!(
                 "Failed to deserialise {}",
@@ -133,30 +148,86 @@ pub fn check_values(
             let relative_package_file = structure::relative_file_for_package(&attribute_name);
 
             use ratchet::RatchetState::*;
+            use Attribute::*;
             use AttributeInfo::*;
             use ByNameAttribute::*;
             use CallPackageVariant::*;
+            use NonByNameAttribute::*;
 
             let check_result = match attribute_value {
-                Missing => NixpkgsProblem::UndefinedAttr {
+                // The attribute succeeds evaluation and is NOT defined in pkgs/by-name
+                NonByName(EvalSuccess(attribute_info)) => {
+                    let uses_by_name = match attribute_info {
+                        // In these cases the package doesn't qualify for being in pkgs/by-name,
+                        // so the UsesByName ratchet is already as tight as it can be
+                        NonAttributeSet => Success(Tight),
+                        NonCallPackage => Success(Tight),
+                        // This is an odd case when _internalCallByNamePackageFile is used to define a package.
+                        CallPackage(CallPackageInfo {
+                            call_package_variant: Auto,
+                            ..
+                        }) => NixpkgsProblem::InternalCallPackageUsed {
+                            attr_name: attribute_name.clone(),
+                        }
+                        .into(),
+                        // Only derivations can be in pkgs/by-name,
+                        // so this attribute doesn't qualify
+                        CallPackage(CallPackageInfo {
+                            is_derivation: false,
+                            ..
+                        }) => Success(Tight),
+
+                        // The case of an attribute that qualifies:
+                        // - Uses callPackage
+                        // - Is a derivation
+                        CallPackage(CallPackageInfo {
+                            is_derivation: true,
+                            call_package_variant: Manual { path, empty_arg },
+                        }) => Success(Loose(ratchet::UsesByName {
+                            call_package_path: path,
+                            empty_arg,
+                        })),
+                    };
+                    uses_by_name.map(|x| ratchet::Package {
+                        empty_non_auto_called: Tight,
+                        uses_by_name: x,
+                    })
+                }
+                NonByName(EvalFailure) => {
+                    // This is a bit of an odd case: We don't even _know_ whether this attribute
+                    // would qualify for using pkgs/by-name. We can either:
+                    // - Assume it's not using pkgs/by-name, which has the problem that if a
+                    //   package evaluation gets broken temporarily, the fix can remove it from
+                    //   pkgs/by-name again
+                    // - Assume it's using pkgs/by-name already, which has the problem that if a
+                    //   package evaluation gets broken temporarily, fixing it requires a move to
+                    //   pkgs/by-name
+                    // We choose the latter, since we want to move towards pkgs/by-name, not away
+                    // from it
+                    Success(ratchet::Package {
+                        empty_non_auto_called: Tight,
+                        uses_by_name: Tight,
+                    })
+                }
+                ByName(Missing) => NixpkgsProblem::UndefinedAttr {
                     relative_package_file: relative_package_file.clone(),
                     package_name: attribute_name.clone(),
                 }
                 .into(),
-                Existing(NonAttributeSet) => NixpkgsProblem::NonDerivation {
+                ByName(Existing(NonAttributeSet)) => NixpkgsProblem::NonDerivation {
                     relative_package_file: relative_package_file.clone(),
                     package_name: attribute_name.clone(),
                 }
                 .into(),
-                Existing(NonCallPackage) => NixpkgsProblem::WrongCallPackage {
+                ByName(Existing(NonCallPackage)) => NixpkgsProblem::WrongCallPackage {
                     relative_package_file: relative_package_file.clone(),
                     package_name: attribute_name.clone(),
                 }
                 .into(),
-                Existing(CallPackage(CallPackageInfo {
+                ByName(Existing(CallPackage(CallPackageInfo {
                     is_derivation,
                     call_package_variant,
-                })) => {
+                }))) => {
                     let check_result = if !is_derivation {
                         NixpkgsProblem::NonDerivation {
                             relative_package_file: relative_package_file.clone(),
@@ -170,6 +241,7 @@ pub fn check_values(
                     check_result.and(match &call_package_variant {
                         Auto => Success(ratchet::Package {
                             empty_non_auto_called: Tight,
+                            uses_by_name: Tight,
                         }),
                         Manual { path, empty_arg } => {
                             let correct_file = if let Some(call_package_path) = path {
@@ -186,6 +258,7 @@ pub fn check_values(
                                     } else {
                                         Tight
                                     },
+                                    uses_by_name: Tight,
                                 })
                             } else {
                                 NixpkgsProblem::WrongCallPackage {
@@ -203,7 +276,7 @@ pub fn check_values(
     ));
 
     Ok(check_result.map(|elems| ratchet::Nixpkgs {
-        package_names,
+        package_names: elems.iter().map(|(name, _)| name.to_owned()).collect(),
         package_map: elems.into_iter().collect(),
     }))
 }

pkgs/test/nixpkgs-check-by-name/src/main.rs

@@ -226,7 +226,9 @@ mod tests {
     }
 
     fn test_nixpkgs(name: &str, path: &Path, expected_errors: &str) -> anyhow::Result<()> {
-        let extra_nix_path = Path::new("tests/mock-nixpkgs.nix");
+        let mock_nixpkgs_path = Path::new("tests/mock-nixpkgs.nix");
+        let real_lib_path = Path::new("../../../lib");
+        let extra_nix_path = [mock_nixpkgs_path, real_lib_path];
 
         let base_path = path.join("base");
         let base_nixpkgs = if base_path.exists() {
@@ -238,7 +240,7 @@ mod tests {
         // We don't want coloring to mess up the tests
         let writer = temp_env::with_var("NO_COLOR", Some("1"), || -> anyhow::Result<_> {
             let mut writer = vec![];
-            process(base_nixpkgs, &path, &[&extra_nix_path], &mut writer)
+            process(base_nixpkgs, &path, &extra_nix_path, &mut writer)
                 .with_context(|| format!("Failed test case {name}"))?;
             Ok(writer)
         })?;

pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs

@@ -1,3 +1,4 @@
+use crate::structure;
 use crate::utils::PACKAGE_NIX_FILENAME;
 use rnix::parser::ParseError;
 use std::ffi::OsString;
@@ -87,6 +88,19 @@ pub enum NixpkgsProblem {
         text: String,
         io_error: io::Error,
     },
+    InternalCallPackageUsed {
+        attr_name: String,
+    },
+    MovedOutOfByName {
+        package_name: String,
+        call_package_path: Option<PathBuf>,
+        empty_arg: bool,
+    },
+    NewPackageNotUsingByName {
+        package_name: String,
+        call_package_path: Option<PathBuf>,
+        empty_arg: bool,
+    },
 }
 
 impl fmt::Display for NixpkgsProblem {
@@ -213,6 +227,53 @@ impl fmt::Display for NixpkgsProblem {
                     subpath.display(),
                     text,
                 ),
+            NixpkgsProblem::InternalCallPackageUsed { attr_name } =>
+                write!(
+                    f,
+                    "pkgs.{attr_name}: This attribute is defined using `_internalCallByNamePackageFile`, which is an internal function not intended for manual use.",
+                ),
+            NixpkgsProblem::MovedOutOfByName { package_name, call_package_path, empty_arg } => {
+                let call_package_arg =
+                    if let Some(path) = &call_package_path {
+                        format!("./{}", path.display())
+                    } else {
+                        "...".into()
+                    };
+                if *empty_arg {
+                    write!(
+                        f,
+                        "pkgs.{package_name}: This top-level package was previously defined in {}, but is now manually defined as `callPackage {call_package_arg} {{ }}` (e.g. in `pkgs/top-level/all-packages.nix`). Please move the package back and remove the manual `callPackage`.",
+                        structure::relative_file_for_package(package_name).display(),
+                        )
+                } else {
+                    // This can happen if users mistakenly assume that for custom arguments,
+                    // pkgs/by-name can't be used.
+                    write!(
+                        f,
+                        "pkgs.{package_name}: This top-level package was previously defined in {}, but is now manually defined as `callPackage {call_package_arg} {{ ... }}` (e.g. in `pkgs/top-level/all-packages.nix`). While the manual `callPackage` is still needed, it's not necessary to move the package files.",
+                        structure::relative_file_for_package(package_name).display(),
+                        )
+                }
+            },
+            NixpkgsProblem::NewPackageNotUsingByName { package_name, call_package_path, empty_arg } => {
+                let call_package_arg =
+                    if let Some(path) = &call_package_path {
+                        format!("./{}", path.display())
+                    } else {
+                        "...".into()
+                    };
+                let extra =
+                    if *empty_arg {
+                        "Since the second `callPackage` argument is `{ }`, no manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is needed anymore."
+                    } else {
+                        "Since the second `callPackage` argument is not `{ }`, the manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is still needed."
+                    };
+                write!(
+                    f,
+                    "pkgs.{package_name}: This is a new top-level package of the form `callPackage {call_package_arg} {{ }}`. Please define it in {} instead. See `pkgs/by-name/README.md` for more details. {extra}",
+                    structure::relative_file_for_package(package_name).display(),
+                )
+            },
         }
     }
 }