Merge pull request #196945 from yaxitech/pulumi-packages

Build `pulumi` binary and packages from source

pkgs/development/python-modules/pulumi/default.nix

@@ -1,42 +1,22 @@
 { lib
 , buildPythonPackage
-, fetchpatch
 , fetchFromGitHub
 , protobuf
 , dill
 , grpcio
-, pulumi-bin
+, pulumi
 , isPy27
 , semver
+, pytestCheckHook
 , pyyaml
 , six
-
-
-# for tests
-, go
-, pulumictl
-, pylint
-, pytest
-, pytest-timeout
-, wheel
-, pytest-asyncio
-
-, mypy
 }:
-let
-  data = import ./data.nix {};
-in
 buildPythonPackage rec {
-  pname = "pulumi";
-  version = pulumi-bin.version;
-  disabled = isPy27;
+  inherit (pulumi) version src;
 
-  src = fetchFromGitHub {
-    owner = "pulumi";
-    repo = "pulumi";
-    rev = "v${pulumi-bin.version}";
-    sha256 = "sha256-vqEZEHTpJV65a3leWwYhyi3dzAsN67BXOvk5hnTPeuI=";
-  };
+  pname = "pulumi";
+
+  disabled = isPy27;
 
   propagatedBuildInputs = [
     semver
@@ -48,28 +28,35 @@ buildPythonPackage rec {
   ];
 
   checkInputs = [
-    pulumi-bin
-    pulumictl
-    mypy
-    go
-    pytest
-    pytest-timeout
-    pytest-asyncio
-    wheel
+    pulumi.pkgs.pulumi-language-python
+    pytestCheckHook
   ];
 
-  sourceRoot="source/sdk/python/lib";
+  pytestFlagsArray = [
+    "test/"
+  ];
+
+  sourceRoot = "source/sdk/python/lib";
+
   # we apply the modifications done in the pulumi/sdk/python/Makefile
   # but without the venv code
   postPatch = ''
     cp ../../README.md .
-    sed -i "s/\''${VERSION}/${version}/g" setup.py
+    substituteInPlace setup.py \
+      --replace "3.0.0" "${version}" \
+      --replace "grpcio==1.47" "grpcio"
   '';
 
-  # disabled because tests try to fetch go packages from the net
-  doCheck = false;
+  # Allow local networking in tests on Darwin
+  __darwinAllowLocalNetworking = true;
 
-  pythonImportsCheck = ["pulumi"];
+  # Verify that the version substitution works
+  preCheck = ''
+    pip show "${pname}" | grep "Version: ${version}" > /dev/null \
+      || (echo "ERROR: Version substitution seems to be broken"; exit 1)
+  '';
+
+  pythonImportsCheck = [ "pulumi" ];
 
   meta = with lib; {
     description = "Modern Infrastructure as Code. Any cloud, any language";

pkgs/tools/admin/pulumi-bin/default.nix

@@ -0,0 +1,43 @@
+{ lib, stdenv, fetchurl, autoPatchelfHook, makeWrapper, installShellFiles }:
+
+with lib;
+
+let
+  data = import ./data.nix {};
+in stdenv.mkDerivation {
+  pname = "pulumi";
+  version = data.version;
+
+  postUnpack = ''
+    mv pulumi-* pulumi
+  '';
+
+  srcs = map (x: fetchurl x) data.pulumiPkgs.${stdenv.hostPlatform.system};
+
+  installPhase = ''
+    install -D -t $out/bin/ *
+  '' + optionalString stdenv.isLinux ''
+    wrapProgram $out/bin/pulumi --set LD_LIBRARY_PATH "${stdenv.cc.cc.lib}/lib"
+  '' + ''
+    installShellCompletion --cmd pulumi \
+      --bash <($out/bin/pulumi completion bash) \
+      --fish <($out/bin/pulumi completion fish) \
+      --zsh  <($out/bin/pulumi completion zsh)
+  '';
+
+  nativeBuildInputs = [ installShellFiles ] ++ optionals stdenv.isLinux [ autoPatchelfHook makeWrapper ];
+
+  meta = {
+    homepage = "https://pulumi.io/";
+    description = "Pulumi is a cloud development platform that makes creating cloud programs easy and productive";
+    sourceProvenance = with sourceTypes; [ binaryNativeCode ];
+    license = with licenses; [ asl20 ];
+    platforms = builtins.attrNames data.pulumiPkgs;
+    maintainers = with maintainers; [
+      ghuntley
+      peterromfeldhk
+      jlesquembre
+      cpcloud
+    ];
+  };
+}

pkgs/tools/admin/pulumi-packages/base.nix

@@ -0,0 +1,128 @@
+{ buildGoModule
+, fetchFromGitHub
+, python3Packages
+}:
+let
+  mkBasePackage =
+    { pname
+    , src
+    , version
+    , vendorHash
+    , cmd
+    , extraLdflags
+    , ...
+    }@args: buildGoModule (rec {
+      inherit pname src vendorHash version;
+
+      sourceRoot = "${src.name}/provider";
+
+      subPackages = [ "cmd/${cmd}" ];
+
+      doCheck = false;
+
+      ldflags = [
+        "-s"
+        "-w"
+      ] ++ extraLdflags;
+    } // args);
+
+  mkPythonPackage =
+    { meta
+    , pname
+    , src
+    , version
+    , ...
+    }: python3Packages.callPackage
+      ({ buildPythonPackage, pythonOlder, parver, pulumi, semver }:
+      buildPythonPackage rec {
+        inherit pname meta src version;
+        format = "setuptools";
+
+        disabled = pythonOlder "3.7";
+
+        sourceRoot = "${src.name}/sdk/python";
+
+        propagatedBuildInputs = [
+          parver
+          pulumi
+          semver
+        ];
+
+        postPatch = ''
+          sed -i \
+            -e 's/^VERSION = .*/VERSION = "${version}"/g' \
+            -e 's/^PLUGIN_VERSION = .*/PLUGIN_VERSION = "${version}"/g' \
+            setup.py
+        '';
+
+        # Auto-generated; upstream does not have any tests.
+        # Verify that the version substitution works
+        checkPhase = ''
+          runHook preCheck
+
+          pip show "${pname}" | grep "Version: ${version}" > /dev/null \
+            || (echo "ERROR: Version substitution seems to be broken"; exit 1)
+
+          runHook postCheck
+        '';
+
+        pythonImportsCheck = [
+          (builtins.replaceStrings [ "-" ] [ "_" ] pname)
+        ];
+      })
+      { };
+in
+{ owner
+, repo
+, rev
+, version
+, hash
+, vendorHash
+, cmdGen
+, cmdRes
+, extraLdflags
+, meta
+, fetchSubmodules ? false
+, ...
+}@args:
+let
+  src = fetchFromGitHub {
+    name = "source-${repo}-${rev}";
+    inherit owner repo rev hash fetchSubmodules;
+  };
+
+  pulumi-gen = mkBasePackage rec {
+    inherit src version vendorHash extraLdflags;
+
+    cmd = cmdGen;
+    pname = cmdGen;
+  };
+in
+mkBasePackage ({
+  inherit meta src version vendorHash extraLdflags;
+
+  pname = repo;
+
+  nativeBuildInputs = [
+    pulumi-gen
+  ];
+
+  cmd = cmdRes;
+
+  postConfigure = ''
+    pushd ..
+
+    chmod +w sdk/
+    ${cmdGen} schema
+
+    popd
+
+    VERSION=v${version} go generate cmd/${cmdRes}/main.go
+  '';
+
+  passthru.sdks.python = mkPythonPackage {
+    inherit meta src version;
+
+    pname = repo;
+  };
+} // args)

pkgs/tools/admin/pulumi-packages/default.nix

@@ -0,0 +1,11 @@
+{ callPackage }:
+let
+  mkPulumiPackage = callPackage ./base.nix { };
+  callPackage' = p: args: callPackage p (args // { inherit mkPulumiPackage; });
+in
+{
+  pulumi-aws-native = callPackage' ./pulumi-aws-native.nix { };
+  pulumi-azure-native = callPackage' ./pulumi-azure-native.nix { };
+  pulumi-language-python = callPackage ./pulumi-language-python.nix { };
+  pulumi-random = callPackage' ./pulumi-random.nix { };
+}

pkgs/tools/admin/pulumi-packages/pulumi-aws-native.nix

@@ -0,0 +1,34 @@
+{ lib
+, mkPulumiPackage
+}:
+mkPulumiPackage rec {
+  owner = "pulumi";
+  repo = "pulumi-aws-native";
+  version = "0.38.0";
+  rev = "v${version}";
+  hash = "sha256-v7jNPCrjtfi9KYD4RhiphMIpV23g/CBV/sKPBkMulu0=";
+  vendorHash = "sha256-Yu9tNakwXWYdrjzI6/MFRzVBhJAEOjsmq9iBAQlR0AI=";
+  cmdGen = "pulumi-gen-aws-native";
+  cmdRes = "pulumi-resource-aws-native";
+  extraLdflags = [
+    "-X github.com/pulumi/${repo}/provider/pkg/version.Version=v${version}"
+  ];
+
+  fetchSubmodules = true;
+  postConfigure = ''
+    pushd ..
+
+    ${cmdGen} schema aws-cloudformation-schema ${version}
+
+    popd
+  '';
+
+  __darwinAllowLocalNetworking = true;
+
+  meta = with lib; {
+    description = "Native AWS Pulumi Provider";
+    homepage = "https://github.com/pulumi/pulumi-aws-native";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ veehaitch trundle ];
+  };
+}

pkgs/tools/admin/pulumi-packages/pulumi-azure-native.nix

@@ -0,0 +1,24 @@
+{ lib
+, mkPulumiPackage
+}:
+mkPulumiPackage rec {
+  owner = "pulumi";
+  repo = "pulumi-azure-native";
+  version = "1.81.0";
+  rev = "v${version}";
+  hash = "sha256-xiifVjvtt4bKi0fBYLU/Gfkx2tziLIq2vddRNWwuyz0=";
+  vendorHash = "sha256-VSwT5I5casJiBpXAcV9vLEWU9XWuDTktmfGqE6H/HX4=";
+  cmdGen = "pulumi-gen-azure-native";
+  cmdRes = "pulumi-resource-azure-native";
+  extraLdflags = [
+    "-X github.com/pulumi/${repo}/provider/pkg/version.Version=v${version}"
+  ];
+  fetchSubmodules = true;
+  __darwinAllowLocalNetworking = true;
+  meta = with lib; {
+    description = "Native Azure Pulumi Provider";
+    homepage = "https://github.com/pulumi/pulumi-azure-native";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ veehaitch trundle ];
+  };
+}

pkgs/tools/admin/pulumi-packages/pulumi-language-python.nix

@@ -0,0 +1,45 @@
+{ lib
+, buildGoModule
+, pulumi
+, python3
+}:
+buildGoModule rec {
+  inherit (pulumi) version src;
+
+  pname = "pulumi-language-python";
+
+  sourceRoot = "${src.name}/sdk";
+
+  vendorHash = "sha256-IZIdLmNGMFjRdkLPoE9UyON3pX/GBIgz/rv108v8iLY=";
+
+  postPatch = ''
+    # Requires network
+    substituteInPlace python/python_test.go \
+      --replace "TestRunningPipInVirtualEnvironment" \
+                "SkipTestRunningPipInVirtualEnvironment"
+
+    substituteInPlace python/cmd/pulumi-language-python/main_test.go \
+      --replace "TestDeterminePulumiPackages" \
+                "SkipTestDeterminePulumiPackages"
+  '';
+
+  subPackages = [
+    "python/cmd/pulumi-language-python"
+  ];
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version=${version}"
+  ];
+
+  checkInputs = [
+    python3
+  ];
+
+  postInstall = ''
+    cp python/cmd/pulumi-language-python-exec    $out/bin
+    cp python/dist/pulumi-resource-pulumi-python $out/bin
+    cp python/dist/pulumi-analyzer-policy-python $out/bin
+  '';
+}

pkgs/tools/admin/pulumi-packages/pulumi-random.nix

@@ -0,0 +1,23 @@
+{ lib
+, mkPulumiPackage
+}:
+mkPulumiPackage rec {
+  owner = "pulumi";
+  repo = "pulumi-random";
+  version = "4.8.2";
+  rev = "v${version}";
+  hash = "sha256-tFEtBgNpl8090RuVMEkyGmdfpZK8wvOD4iog1JRq+GY=";
+  vendorHash = "sha256-H3mpKxb1lt+du3KterYPV6WWs1D0XXlmemMyMiZBnqs=";
+  cmdGen = "pulumi-tfgen-random";
+  cmdRes = "pulumi-resource-random";
+  extraLdflags = [
+    "-X github.com/pulumi/${repo}/provider/v4/pkg/version.Version=v${version}"
+  ];
+  __darwinAllowLocalNetworking = true;
+  meta = with lib; {
+    description = "A Pulumi provider that safely enables randomness for resources";
+    homepage = "https://github.com/pulumi/pulumi-random";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ veehaitch trundle ];
+  };
+}

pkgs/tools/admin/pulumi/default.nix

@@ -1,43 +1,108 @@
-{ lib, stdenv, fetchurl, autoPatchelfHook, makeWrapper, installShellFiles }:
+{ stdenv
+, lib
+, buildGoModule
+, coreutils
+, fetchFromGitHub
+, installShellFiles
+, git
+  # passthru
+, runCommand
+, makeWrapper
+, pulumi
+, pulumiPackages
+}:
 
-with lib;
-
-let
-  data = import ./data.nix {};
-in stdenv.mkDerivation {
+buildGoModule rec {
   pname = "pulumi";
-  version = data.version;
+  version = "3.43.1";
 
-  postUnpack = ''
-    mv pulumi-* pulumi
+  src = fetchFromGitHub {
+    owner = pname;
+    repo = pname;
+    rev = "v${version}";
+    hash = "sha256-LbPXCwU6aJp+z5scfej5Reo2X8QUvZpASWkcDBBF1J0=";
+  };
+
+  vendorSha256 = "sha256-APGiCqHdXDRCFx0W8RDeL89sskYZ2vzdpg4ePE7KixA=";
+
+  sourceRoot = "source/pkg";
+
+  nativeBuildInputs = [ installShellFiles ];
+
+  # Bundle release metadata
+  ldflags = [
+    # Omit the symbol table and debug information.
+    "-s"
+    # Omit the DWARF symbol table.
+    "-w"
+  ] ++ importpathFlags;
+
+  importpathFlags = [
+    "-X github.com/pulumi/pulumi/pkg/v3/version.Version=v${version}"
+  ];
+
+  doCheck = true;
+
+  checkInputs = [
+    git
+  ];
+
+  preCheck = ''
+    # The tests require `version.Version` to be unset
+    ldflags=''${ldflags//"$importpathFlags"/}
+
+    # Create some placeholders for plugins used in tests. Otherwise, Pulumi
+    # tries to donwload them and fails, resulting in really long test runs
+    dummyPluginPath=$(mktemp -d)
+    for name in pulumi-{resource-pkg{A,B},-pkgB}; do
+      ln -s ${coreutils}/bin/true "$dummyPluginPath/$name"
+    done
+
+    export PATH=$dummyPluginPath''${PATH:+:}$PATH
+
+    # Code generation tests also download dependencies from network
+    rm codegen/{docs,dotnet,go,nodejs,python,schema}/*_test.go
+  '' + lib.optionalString stdenv.isDarwin ''
+    export PULUMI_HOME=$(mktemp -d)
   '';
 
-  srcs = map (x: fetchurl x) data.pulumiPkgs.${stdenv.hostPlatform.system};
+  # Allow tests that bind or connect to localhost on macOS.
+  __darwinAllowLocalNetworking = true;
 
-  installPhase = ''
-    install -D -t $out/bin/ *
-  '' + optionalString stdenv.isLinux ''
-    wrapProgram $out/bin/pulumi --set LD_LIBRARY_PATH "${stdenv.cc.cc.lib}/lib"
-  '' + ''
+  doInstallCheck = true;
+  installCheckPhase = ''
+    PULUMI_SKIP_UPDATE_CHECK=1 $out/bin/pulumi version | grep v${version} > /dev/null
+  '';
+
+  postInstall = ''
     installShellCompletion --cmd pulumi \
-      --bash <($out/bin/pulumi completion bash) \
-      --fish <($out/bin/pulumi completion fish) \
-      --zsh  <($out/bin/pulumi completion zsh)
+      --bash <($out/bin/pulumi gen-completion bash) \
+      --fish <($out/bin/pulumi gen-completion fish) \
+      --zsh  <($out/bin/pulumi gen-completion zsh)
   '';
 
-  nativeBuildInputs = [ installShellFiles ] ++ optionals stdenv.isLinux [ autoPatchelfHook makeWrapper ];
+  passthru = {
+    pkgs = pulumiPackages;
+    withPackages = f: runCommand "${pulumi.name}-with-packages"
+      {
+        nativeBuildInputs = [ makeWrapper ];
+      }
+      ''
+        mkdir -p $out/bin
+        makeWrapper ${pulumi}/bin/pulumi $out/bin/pulumi \
+          --suffix PATH : ${lib.makeSearchPath "bin" (f pulumiPackages)}
+      '';
+  };
 
-  meta = {
+  meta = with lib; {
     homepage = "https://pulumi.io/";
     description = "Pulumi is a cloud development platform that makes creating cloud programs easy and productive";
-    sourceProvenance = with sourceTypes; [ binaryNativeCode ];
-    license = with licenses; [ asl20 ];
-    platforms = builtins.attrNames data.pulumiPkgs;
+    sourceProvenance = sourceTypes.fromSource;
+    license = licenses.asl20;
+    platforms = platforms.unix;
     maintainers = with maintainers; [
-      ghuntley
-      peterromfeldhk
-      jlesquembre
-      cpcloud
+      trundle
+      veehaitch
     ];
   };
 }

pkgs/top-level/all-packages.nix

@@ -10249,7 +10249,13 @@ with pkgs;
 
   prism = callPackage ../applications/video/prism { };
 
-  pulumi-bin = callPackage ../tools/admin/pulumi { };
+  pulumi = callPackage ../tools/admin/pulumi { };
+
+  pulumiPackages = recurseIntoAttrs (
+    callPackage ../tools/admin/pulumi-packages { }
+  );
+
+  pulumi-bin = callPackage ../tools/admin/pulumi-bin { };
 
   p0f = callPackage ../tools/security/p0f { };
 

pkgs/top-level/python-packages.nix

@@ -1074,10 +1074,16 @@ self: super: with self; {
 
   pad4pi = callPackage ../development/python-modules/pad4pi { };
 
-  pulumi = callPackage ../development/python-modules/pulumi { };
+  pulumi = callPackage ../development/python-modules/pulumi { inherit (pkgs) pulumi; };
 
   pulumi-aws = callPackage ../development/python-modules/pulumi-aws { };
 
+  pulumi-aws-native = pkgs.pulumiPackages.pulumi-aws-native.sdks.python;
+
+  pulumi-azure-native = pkgs.pulumiPackages.pulumi-azure-native.sdks.python;
+
+  pulumi-random = pkgs.pulumiPackages.pulumi-random.sdks.python;
+
   backcall = callPackage ../development/python-modules/backcall { };
 
   backoff = callPackage ../development/python-modules/backoff { };