Merge pull request #80958 from adisbladis/poetry2nix-1_5_0

poetry2nix: 1.4.0 -> 1.5.0

pkgs/development/tools/poetry/default.nix

@@ -8,12 +8,6 @@ poetry2nix.mkPoetryApplication {
   pyproject = ./pyproject.toml;
   poetrylock = ./poetry.lock;
 
-  overrides = [ (poetry2nix.defaultPoetryOverrides.overrideOverlay (self: super: {
-    # Needed because poetry2nix currently doesn't handle pyproject.toml python bounds
-    # See https://github.com/nix-community/poetry2nix/issues/50
-    importlib-metadata = if python.pythonOlder "3.8" then super.importlib-metadata else null;
-  }))];
-
   src = fetchFromGitHub (lib.importJSON ./src.json);
 
   # "Vendor" dependencies (for build-system support)

pkgs/development/tools/poetry2nix/poetry2nix/default.nix

@@ -25,19 +25,21 @@ let
   # Get license by id falling back to input string
   getLicenseBySpdxId = spdxId: spdxLicenses.${spdxId} or spdxId;
 
-  #
+  /*
-  # Returns an attrset { python, poetryPackages } for the given lockfile
+     Returns an attrset { python, poetryPackages, pyProject, poetryLock } for the given pyproject/lockfile.
-  #
+  */
-  mkPoetryPython =
+  mkPoetryPackages =
-    { poetrylock
+    { pyproject
+    , poetrylock
     , poetryPkg
     , overrides ? [ defaultPoetryOverrides ]
     , meta ? {}
     , python ? pkgs.python3
     , pwd ? null
     }@attrs: let
-      lockData = readTOML poetrylock;
+      pyProject = readTOML pyproject;
-      lockFiles = lib.getAttrFromPath [ "metadata" "files" ] lockData;
+      poetryLock = readTOML poetrylock;
+      lockFiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock;
 
       specialAttrs = [
         "overrides"
@@ -48,11 +50,18 @@ let
 
       evalPep508 = mkEvalPep508 python;
 
-      # Filter packages by their PEP508 markers
+      # Filter packages by their PEP508 markers & pyproject interpreter version
       partitions = let
-        supportsPythonVersion = pkgMeta: if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true;
+        supportsPythonVersion = pkgMeta: let
+          pep508Result = if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true;
+
+          flatDeps = (pyProject.tool.poetry.dependencies or {}) // (pyProject.tool.poetry.dev-dependencies or {});
+          constraints = flatDeps.${pkgMeta.name}.python or "";
+          pyprojectResult = isCompatible python.pythonVersion constraints;
+        in
+          pyprojectResult && pep508Result;
       in
-        lib.partition supportsPythonVersion lockData.package;
+        lib.partition supportsPythonVersion poetryLock.package;
 
       compatible = partitions.right;
       incompatible = partitions.wrong;
@@ -82,18 +91,22 @@ let
           );
         in
           lockPkgs;
-
       overlays = builtins.map getFunctorFn (
         [
           (
-            self: super: {
+            self: super: let
-              mkPoetryDep = self.callPackage ./mk-poetry-dep.nix {
+              hooks = self.callPackage ./hooks {};
-                inherit pkgs lib python poetryLib;
+            in
-              };
+              {
-              poetry = poetryPkg;
+                mkPoetryDep = self.callPackage ./mk-poetry-dep.nix {
-              # The canonical name is setuptools-scm
+                  inherit pkgs lib python poetryLib;
-              setuptools-scm = super.setuptools_scm;
+                };
-            }
+                poetry = poetryPkg;
+                # The canonical name is setuptools-scm
+                setuptools-scm = super.setuptools_scm;
+
+                inherit (hooks) removePathDependenciesHook;
+              }
           )
           # Null out any filtered packages, we don't want python.pkgs from nixpkgs
           (self: super: builtins.listToAttrs (builtins.map (x: { name = x.name; value = null; }) incompatible))
@@ -110,6 +123,8 @@ let
       {
         python = py;
         poetryPackages = map (pkg: py.pkgs.${pkg.name}) compatible;
+        poetryLock = poetryLock;
+        inherit pyProject;
       };
 
   /* Returns a package with a python interpreter and all packages specified in the poetry.lock lock file.
@@ -118,7 +133,8 @@ let
        poetry2nix.mkPoetryEnv { poetrylock = ./poetry.lock; python = python3; }
   */
   mkPoetryEnv =
-    { poetrylock
+    { pyproject
+    , poetrylock
     , overrides ? [ defaultPoetryOverrides ]
     , meta ? {}
     , pwd ? null
@@ -126,9 +142,9 @@ let
     }:
       let
         poetryPkg = poetry.override { inherit python; };
-        py = mkPoetryPython (
+        py = mkPoetryPackages (
           {
-            inherit poetryPkg poetrylock overrides meta python pwd;
+            inherit poetryPkg pyproject poetrylock overrides meta python pwd;
           }
         );
       in
@@ -147,13 +163,12 @@ let
     }@attrs: let
       poetryPkg = poetry.override { inherit python; };
 
-      py = (
+      poetryPython = mkPoetryPackages {
-        mkPoetryPython {
+        inherit poetryPkg pyproject poetrylock overrides meta python pwd;
-          inherit poetryPkg poetrylock overrides meta python pwd;
+      };
-        }
+      py = poetryPython.python;
-      ).python;
 
-      pyProject = readTOML pyproject;
+      inherit (poetryPython) pyProject;
 
       specialAttrs = [
         "overrides"
@@ -187,21 +202,13 @@ let
 
           buildInputs = mkInput "buildInputs" buildSystemPkgs;
           propagatedBuildInputs = mkInput "propagatedBuildInputs" (getDeps "dependencies") ++ ([ py.pkgs.setuptools ]);
-          nativeBuildInputs = mkInput "nativeBuildInputs" [ pkgs.yj ];
+          nativeBuildInputs = mkInput "nativeBuildInputs" [ pkgs.yj py.pkgs.removePathDependenciesHook ];
           checkInputs = mkInput "checkInputs" (getDeps "dev-dependencies");
 
           passthru = {
             python = py;
           };
 
-          postPatch = (passedAttrs.postPatch or "") + ''
-            # Tell poetry not to resolve the path dependencies. Any version is
-            # fine !
-            yj -tj < pyproject.toml | ${python.interpreter} ${./pyproject-without-path.py} > pyproject.json
-            yj -jt < pyproject.json > pyproject.toml
-            rm pyproject.json
-          '';
-
           meta = meta // {
             inherit (pyProject.tool.poetry) description homepage;
             license = getLicenseBySpdxId (pyProject.tool.poetry.license or "unknown");
@@ -240,7 +247,7 @@ let
 
 in
 {
-  inherit mkPoetryEnv mkPoetryApplication cli doc;
+  inherit mkPoetryEnv mkPoetryApplication mkPoetryPackages cli doc;
 
   /*
   The default list of poetry2nix override overlays

pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix

@@ -0,0 +1,25 @@
+{ python
+, callPackage
+, makeSetupHook
+, yj
+}:
+
+let
+  pythonInterpreter = python.pythonForBuild.interpreter;
+in
+{
+
+  removePathDependenciesHook = callPackage (
+    {}:
+      makeSetupHook {
+        name = "remove-path-dependencies.sh";
+        deps = [];
+        substitutions = {
+          inherit pythonInterpreter;
+          yj = "${yj}/bin/yj";
+          pyprojectPatchScript = "${./pyproject-without-path.py}";
+        };
+      } ./remove-path-dependencies.sh
+  ) {};
+
+}

pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-path-dependencies.sh

@@ -0,0 +1,8 @@
+remove-path-dependencies-hook() {
+    # Tell poetry not to resolve the path dependencies. Any version is fine!
+    @yj@ -tj < pyproject.toml | @pythonInterpreter@ @pyprojectPatchScript@ > pyproject.json
+    @yj@ -jt < pyproject.json > pyproject.toml
+    rm pyproject.json
+}
+
+postPatchHooks+=(remove-path-dependencies-hook)

pkgs/development/tools/poetry2nix/poetry2nix/lib.nix

@@ -8,27 +8,30 @@ let
     genList (i: if i == idx then value else (builtins.elemAt list i)) (length list)
   );
 
-  # Returns true if pythonVersion matches with the expression in pythonVersions
+  # Compare a semver expression with a version
-  isCompatible = pythonVersion: pythonVersions:
+  isCompatible = version: let
-    let
+    operators = {
-      operators = {
+      "||" = cond1: cond2: cond1 || cond2;
-        "||" = cond1: cond2: cond1 || cond2;
+      "," = cond1: cond2: cond1 && cond2; # , means &&
-        "," = cond1: cond2: cond1 && cond2; # , means &&
+      "&&" = cond1: cond2: cond1 && cond2;
-      };
+    };
-      # split string at "," and "||"
+    splitRe = "(" + (builtins.concatStringsSep "|" (builtins.map (x: lib.replaceStrings [ "|" ] [ "\\|" ] x) (lib.attrNames operators))) + ")";
-      tokens = builtins.filter (x: x != "") (builtins.split "(,|\\|\\|)" pythonVersions);
+  in
-      combine = acc: v:
+    expr:
-        let
+      let
-          isOperator = builtins.typeOf v == "list";
+        tokens = builtins.filter (x: x != "") (builtins.split splitRe expr);
-          operator = if isOperator then (builtins.elemAt v 0) else acc.operator;
+        combine = acc: v:
-        in
+          let
-          if isOperator then (acc // { inherit operator; }) else {
+            isOperator = builtins.typeOf v == "list";
-            inherit operator;
+            operator = if isOperator then (builtins.elemAt v 0) else acc.operator;
-            state = operators."${operator}" acc.state (satisfiesSemver pythonVersion v);
+          in
-          };
+            if isOperator then (acc // { inherit operator; }) else {
-      initial = { operator = ","; state = true; };
+              inherit operator;
-    in
+              state = operators."${operator}" acc.state (satisfiesSemver version v);
-      (builtins.foldl' combine initial tokens).state;
+            };
+        initial = { operator = "&&"; state = true; };
+      in
+        if expr == "" then true else (builtins.foldl' combine initial tokens).state;
 
   fromTOML = builtins.fromTOML or
     (
@@ -65,7 +68,7 @@ let
       else { pkg = []; str = null; };
 
   # Fetch the artifacts from the PyPI index. Since we get all
-  # info we need from the lock file we don't use nixpkgs' fetchPypi
+  # info we need from the lock file we don't use nixpkgs' fetchPyPi
   # as it modifies casing while not providing anything we don't already
   # have.
   #
@@ -101,5 +104,6 @@ in
     isCompatible
     readTOML
     getBuildSystemPkgs
+    satisfiesSemver
     ;
 }

pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix

@@ -83,7 +83,13 @@ pythonPackages.callPackage (
             else (builtins.elemAt (lib.strings.splitString "-" name) 2);
         };
 
-      baseBuildInputs = lib.optional (name != "setuptools_scm" && name != "setuptools-scm") pythonPackages.setuptools-scm;
+      # Prevent infinite recursion
+      skipSetupToolsSCM = [
+        "setuptools_scm"
+        "setuptools-scm"
+        "toml" # Toml is an extra for setuptools-scm
+      ];
+      baseBuildInputs = lib.optional (! lib.elem name skipSetupToolsSCM) pythonPackages.setuptools-scm;
 
       format = if isLocal then "pyproject" else if isGit then "setuptools" else fileInfo.format;
 
@@ -100,7 +106,11 @@ pythonPackages.callPackage (
         # Stripping pre-built wheels lead to `ELF load command address/offset not properly aligned`
         dontStrip = format == "wheel";
 
-        nativeBuildInputs = if (!isSource && (getManyLinuxDeps fileInfo.name).str != null) then [ autoPatchelfHook ] else [];
+        nativeBuildInputs = (if (!isSource && (getManyLinuxDeps fileInfo.name).str != null) then [ autoPatchelfHook ] else [])
+        ++ lib.optional (isLocal) pkgs.yj
+        ++ lib.optional (format == "pyproject") pythonPackages.removePathDependenciesHook
+        ;
+
         buildInputs = (
           baseBuildInputs
           ++ lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg

pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix

@@ -103,7 +103,7 @@ self: super:
   );
 
   # importlib-metadata has an incomplete dependency specification
-  importlib-metadata = super.importlib-metadata.overrideAttrs (
+  importlib-metadata = if super.importlib-metadata == null then null else super.importlib-metadata.overrideAttrs (
     old: {
       propagatedBuildInputs = old.propagatedBuildInputs ++ lib.optional self.python.isPy2 self.pathlib2;
     }

pkgs/development/tools/poetry2nix/update

@@ -15,7 +15,7 @@ mv poetry2nix-master/* .
 
 mkdir build
 cp *.nix *.json *.py build/
-cp -r bin build/
+cp -r hooks bin build/
 rm build/shell.nix build/generate.py build/overlay.nix build/flake.nix
 
 cat > build/README.md << EOF