diff --git a/pkgs/development/tools/poetry/default.nix b/pkgs/development/tools/poetry/default.nix index 5035677beb00..1517e53f219c 100644 --- a/pkgs/development/tools/poetry/default.nix +++ b/pkgs/development/tools/poetry/default.nix @@ -8,12 +8,6 @@ poetry2nix.mkPoetryApplication { pyproject = ./pyproject.toml; poetrylock = ./poetry.lock; - overrides = [ (poetry2nix.defaultPoetryOverrides.overrideOverlay (self: super: { - # Needed because poetry2nix currently doesn't handle pyproject.toml python bounds - # See https://github.com/nix-community/poetry2nix/issues/50 - importlib-metadata = if python.pythonOlder "3.8" then super.importlib-metadata else null; - }))]; - src = fetchFromGitHub (lib.importJSON ./src.json); # "Vendor" dependencies (for build-system support) diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix index 0d1c243d0975..ac866a139d97 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix @@ -25,19 +25,21 @@ let # Get license by id falling back to input string getLicenseBySpdxId = spdxId: spdxLicenses.${spdxId} or spdxId; - # - # Returns an attrset { python, poetryPackages } for the given lockfile - # - mkPoetryPython = - { poetrylock + /* + Returns an attrset { python, poetryPackages, pyProject, poetryLock } for the given pyproject/lockfile. + */ + mkPoetryPackages = + { pyproject + , poetrylock , poetryPkg , overrides ? [ defaultPoetryOverrides ] , meta ? {} , python ? pkgs.python3 , pwd ? null }@attrs: let - lockData = readTOML poetrylock; - lockFiles = lib.getAttrFromPath [ "metadata" "files" ] lockData; + pyProject = readTOML pyproject; + poetryLock = readTOML poetrylock; + lockFiles = lib.getAttrFromPath [ "metadata" "files" ] poetryLock; specialAttrs = [ "overrides" @@ -48,11 +50,18 @@ let evalPep508 = mkEvalPep508 python; - # Filter packages by their PEP508 markers + # Filter packages by their PEP508 markers & pyproject interpreter version partitions = let - supportsPythonVersion = pkgMeta: if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true; + supportsPythonVersion = pkgMeta: let + pep508Result = if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true; + + flatDeps = (pyProject.tool.poetry.dependencies or {}) // (pyProject.tool.poetry.dev-dependencies or {}); + constraints = flatDeps.${pkgMeta.name}.python or ""; + pyprojectResult = isCompatible python.pythonVersion constraints; + in + pyprojectResult && pep508Result; in - lib.partition supportsPythonVersion lockData.package; + lib.partition supportsPythonVersion poetryLock.package; compatible = partitions.right; incompatible = partitions.wrong; @@ -82,18 +91,22 @@ let ); in lockPkgs; - overlays = builtins.map getFunctorFn ( [ ( - self: super: { - mkPoetryDep = self.callPackage ./mk-poetry-dep.nix { - inherit pkgs lib python poetryLib; - }; - poetry = poetryPkg; - # The canonical name is setuptools-scm - setuptools-scm = super.setuptools_scm; - } + self: super: let + hooks = self.callPackage ./hooks {}; + in + { + mkPoetryDep = self.callPackage ./mk-poetry-dep.nix { + inherit pkgs lib python poetryLib; + }; + poetry = poetryPkg; + # The canonical name is setuptools-scm + setuptools-scm = super.setuptools_scm; + + inherit (hooks) removePathDependenciesHook; + } ) # Null out any filtered packages, we don't want python.pkgs from nixpkgs (self: super: builtins.listToAttrs (builtins.map (x: { name = x.name; value = null; }) incompatible)) @@ -110,6 +123,8 @@ let { python = py; poetryPackages = map (pkg: py.pkgs.${pkg.name}) compatible; + poetryLock = poetryLock; + inherit pyProject; }; /* Returns a package with a python interpreter and all packages specified in the poetry.lock lock file. @@ -118,7 +133,8 @@ let poetry2nix.mkPoetryEnv { poetrylock = ./poetry.lock; python = python3; } */ mkPoetryEnv = - { poetrylock + { pyproject + , poetrylock , overrides ? [ defaultPoetryOverrides ] , meta ? {} , pwd ? null @@ -126,9 +142,9 @@ let }: let poetryPkg = poetry.override { inherit python; }; - py = mkPoetryPython ( + py = mkPoetryPackages ( { - inherit poetryPkg poetrylock overrides meta python pwd; + inherit poetryPkg pyproject poetrylock overrides meta python pwd; } ); in @@ -147,13 +163,12 @@ let }@attrs: let poetryPkg = poetry.override { inherit python; }; - py = ( - mkPoetryPython { - inherit poetryPkg poetrylock overrides meta python pwd; - } - ).python; + poetryPython = mkPoetryPackages { + inherit poetryPkg pyproject poetrylock overrides meta python pwd; + }; + py = poetryPython.python; - pyProject = readTOML pyproject; + inherit (poetryPython) pyProject; specialAttrs = [ "overrides" @@ -187,21 +202,13 @@ let buildInputs = mkInput "buildInputs" buildSystemPkgs; propagatedBuildInputs = mkInput "propagatedBuildInputs" (getDeps "dependencies") ++ ([ py.pkgs.setuptools ]); - nativeBuildInputs = mkInput "nativeBuildInputs" [ pkgs.yj ]; + nativeBuildInputs = mkInput "nativeBuildInputs" [ pkgs.yj py.pkgs.removePathDependenciesHook ]; checkInputs = mkInput "checkInputs" (getDeps "dev-dependencies"); passthru = { python = py; }; - postPatch = (passedAttrs.postPatch or "") + '' - # Tell poetry not to resolve the path dependencies. Any version is - # fine ! - yj -tj < pyproject.toml | ${python.interpreter} ${./pyproject-without-path.py} > pyproject.json - yj -jt < pyproject.json > pyproject.toml - rm pyproject.json - ''; - meta = meta // { inherit (pyProject.tool.poetry) description homepage; license = getLicenseBySpdxId (pyProject.tool.poetry.license or "unknown"); @@ -240,7 +247,7 @@ let in { - inherit mkPoetryEnv mkPoetryApplication cli doc; + inherit mkPoetryEnv mkPoetryApplication mkPoetryPackages cli doc; /* The default list of poetry2nix override overlays diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix new file mode 100644 index 000000000000..dd24b2634e00 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix @@ -0,0 +1,25 @@ +{ python +, callPackage +, makeSetupHook +, yj +}: + +let + pythonInterpreter = python.pythonForBuild.interpreter; +in +{ + + removePathDependenciesHook = callPackage ( + {}: + makeSetupHook { + name = "remove-path-dependencies.sh"; + deps = []; + substitutions = { + inherit pythonInterpreter; + yj = "${yj}/bin/yj"; + pyprojectPatchScript = "${./pyproject-without-path.py}"; + }; + } ./remove-path-dependencies.sh + ) {}; + +} diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pyproject-without-path.py b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-path.py similarity index 100% rename from pkgs/development/tools/poetry2nix/poetry2nix/pyproject-without-path.py rename to pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-path.py diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-path-dependencies.sh b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-path-dependencies.sh new file mode 100644 index 000000000000..900fe66908bf --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-path-dependencies.sh @@ -0,0 +1,8 @@ +remove-path-dependencies-hook() { + # Tell poetry not to resolve the path dependencies. Any version is fine! + @yj@ -tj < pyproject.toml | @pythonInterpreter@ @pyprojectPatchScript@ > pyproject.json + @yj@ -jt < pyproject.json > pyproject.toml + rm pyproject.json +} + +postPatchHooks+=(remove-path-dependencies-hook) diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix index d49e6ce36db3..64ba0293132e 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix @@ -8,27 +8,30 @@ let genList (i: if i == idx then value else (builtins.elemAt list i)) (length list) ); - # Returns true if pythonVersion matches with the expression in pythonVersions - isCompatible = pythonVersion: pythonVersions: - let - operators = { - "||" = cond1: cond2: cond1 || cond2; - "," = cond1: cond2: cond1 && cond2; # , means && - }; - # split string at "," and "||" - tokens = builtins.filter (x: x != "") (builtins.split "(,|\\|\\|)" pythonVersions); - combine = acc: v: - let - isOperator = builtins.typeOf v == "list"; - operator = if isOperator then (builtins.elemAt v 0) else acc.operator; - in - if isOperator then (acc // { inherit operator; }) else { - inherit operator; - state = operators."${operator}" acc.state (satisfiesSemver pythonVersion v); - }; - initial = { operator = ","; state = true; }; - in - (builtins.foldl' combine initial tokens).state; + # Compare a semver expression with a version + isCompatible = version: let + operators = { + "||" = cond1: cond2: cond1 || cond2; + "," = cond1: cond2: cond1 && cond2; # , means && + "&&" = cond1: cond2: cond1 && cond2; + }; + splitRe = "(" + (builtins.concatStringsSep "|" (builtins.map (x: lib.replaceStrings [ "|" ] [ "\\|" ] x) (lib.attrNames operators))) + ")"; + in + expr: + let + tokens = builtins.filter (x: x != "") (builtins.split splitRe expr); + combine = acc: v: + let + isOperator = builtins.typeOf v == "list"; + operator = if isOperator then (builtins.elemAt v 0) else acc.operator; + in + if isOperator then (acc // { inherit operator; }) else { + inherit operator; + state = operators."${operator}" acc.state (satisfiesSemver version v); + }; + initial = { operator = "&&"; state = true; }; + in + if expr == "" then true else (builtins.foldl' combine initial tokens).state; fromTOML = builtins.fromTOML or ( @@ -65,7 +68,7 @@ let else { pkg = []; str = null; }; # Fetch the artifacts from the PyPI index. Since we get all - # info we need from the lock file we don't use nixpkgs' fetchPypi + # info we need from the lock file we don't use nixpkgs' fetchPyPi # as it modifies casing while not providing anything we don't already # have. # @@ -101,5 +104,6 @@ in isCompatible readTOML getBuildSystemPkgs + satisfiesSemver ; } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix index cbfa3dafa753..5113ad25e4b6 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix @@ -83,7 +83,13 @@ pythonPackages.callPackage ( else (builtins.elemAt (lib.strings.splitString "-" name) 2); }; - baseBuildInputs = lib.optional (name != "setuptools_scm" && name != "setuptools-scm") pythonPackages.setuptools-scm; + # Prevent infinite recursion + skipSetupToolsSCM = [ + "setuptools_scm" + "setuptools-scm" + "toml" # Toml is an extra for setuptools-scm + ]; + baseBuildInputs = lib.optional (! lib.elem name skipSetupToolsSCM) pythonPackages.setuptools-scm; format = if isLocal then "pyproject" else if isGit then "setuptools" else fileInfo.format; @@ -100,7 +106,11 @@ pythonPackages.callPackage ( # Stripping pre-built wheels lead to `ELF load command address/offset not properly aligned` dontStrip = format == "wheel"; - nativeBuildInputs = if (!isSource && (getManyLinuxDeps fileInfo.name).str != null) then [ autoPatchelfHook ] else []; + nativeBuildInputs = (if (!isSource && (getManyLinuxDeps fileInfo.name).str != null) then [ autoPatchelfHook ] else []) + ++ lib.optional (isLocal) pkgs.yj + ++ lib.optional (format == "pyproject") pythonPackages.removePathDependenciesHook + ; + buildInputs = ( baseBuildInputs ++ lib.optional (!isSource) (getManyLinuxDeps fileInfo.name).pkg diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix index 86260c037036..6316dcf51c0c 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix @@ -103,7 +103,7 @@ self: super: ); # importlib-metadata has an incomplete dependency specification - importlib-metadata = super.importlib-metadata.overrideAttrs ( + importlib-metadata = if super.importlib-metadata == null then null else super.importlib-metadata.overrideAttrs ( old: { propagatedBuildInputs = old.propagatedBuildInputs ++ lib.optional self.python.isPy2 self.pathlib2; } diff --git a/pkgs/development/tools/poetry2nix/update b/pkgs/development/tools/poetry2nix/update index a5d367590be1..4840c23c3b81 100755 --- a/pkgs/development/tools/poetry2nix/update +++ b/pkgs/development/tools/poetry2nix/update @@ -15,7 +15,7 @@ mv poetry2nix-master/* . mkdir build cp *.nix *.json *.py build/ -cp -r bin build/ +cp -r hooks bin build/ rm build/shell.nix build/generate.py build/overlay.nix build/flake.nix cat > build/README.md << EOF