From 2bfa6aa848a89e4ba8308128801812e5018dcd44 Mon Sep 17 00:00:00 2001
From: Jan Tojnar <jtojnar@gmail.com>
Date: Fri, 24 Jul 2020 15:17:42 +0200
Subject: [PATCH 1/4] nixosTests.installed-tests: Add the test data to VM
 closure
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flatpak’s installed tests build Flatpak runtimes, among other things.
Upstream code does this by copying some programs on `PATH`
as well as some possible dependencies from host’s /usr.
We patch the code to use `nix-store --query --requisites`
to make the dependency discovery easier.

The Flatpak’s installed tests add `socat` to `PATH` and later run
`nix-store --query --requisites` on its location but it was failing with

    error: path '/nix/store/qcyf7nq5vvfw32967sv4j6z190inrbrc-socat-1.7.3.4' is not valid

The issue occurred because, while the host Nix store is bind mounted into the test VM,
the VM’s store uses its own database that only contains the packages in the VM’s closure.
Since the test commands are not actually part of the VM but only passed through PTY,
the `flatpak.installedTests` derivation was not part of the VM’s closure, so `nix-store`
in the VM could not get information about its dependency `socat`.

Let’s make the `installedTests` of the tested package part of the test VM’s closure
by passing it as a global environment variable. This will also have the added benefit
that user no longer has to type the path when running the installed tests manually in the VM;
they can just use `gnome-desktop-testing-runner -d $TESTED_PACKAGE_INSTALLED_TESTS`,
which is much more conducive to tab completion.
---
 nixos/tests/installed-tests/default.nix | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/nixos/tests/installed-tests/default.nix b/nixos/tests/installed-tests/default.nix
index b6bdfea22770..889a00d4b568 100644
--- a/nixos/tests/installed-tests/default.nix
+++ b/nixos/tests/installed-tests/default.nix
@@ -50,6 +50,12 @@ let
 
             environment.systemPackages = with pkgs; [ gnome-desktop-testing ];
 
+            # The installed tests need to be added to the test VM’s closure.
+            # Otherwise, their dependencies might not actually be registered
+            # as valid paths in the VM’s Nix store database,
+            # and `nix-store --query` commands run as part of the tests
+            # (for example when building Flatpak runtimes) will fail.
+            environment.variables.TESTED_PACKAGE_INSTALLED_TESTS = "${tested.installedTests}/share";
           };
 
           testScript =

From 8d53e88346d923e7d3114a8cfc09d45accbe5bc8 Mon Sep 17 00:00:00 2001
From: Jan Tojnar <jtojnar@gmail.com>
Date: Fri, 24 Jul 2020 16:32:34 +0200
Subject: [PATCH 2/4] nixosTests.installed-tests.flatpak: Fix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Along with the `socat` fix in the parent commit, this makes
the Flatpak’s installed tests finally pass again.

The tests seem to need slightly over 2G of disk space,
and it appears that the test suite was ported to Python 3 in 1.5.1:
https://github.com/flatpak/flatpak/commit/2b6641575db75d8af8228749798f4ea635797577
---
 nixos/tests/installed-tests/flatpak.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nixos/tests/installed-tests/flatpak.nix b/nixos/tests/installed-tests/flatpak.nix
index 091c99326629..6b83d006f162 100644
--- a/nixos/tests/installed-tests/flatpak.nix
+++ b/nixos/tests/installed-tests/flatpak.nix
@@ -10,9 +10,9 @@ makeInstalledTest {
     services.xserver.displayManager.gdm.enable = false;
     services.gnome3.core-utilities.enable = false;
     services.flatpak.enable = true;
-    environment.systemPackages = with pkgs; [ gnupg ostree python2 ];
+    environment.systemPackages = with pkgs; [ gnupg ostree python3 ];
     virtualisation.memorySize = 2047;
-    virtualisation.diskSize = 1024;
+    virtualisation.diskSize = 3072;
   };
 
   testRunnerFlags = "--timeout 3600";

From 98710d255298ebc6a347edcf5aa9c4a39000f671 Mon Sep 17 00:00:00 2001
From: Jan Tojnar <jtojnar@gmail.com>
Date: Fri, 24 Jul 2020 09:04:16 +0200
Subject: [PATCH 3/4] =?UTF-8?q?flatpak:=201.6.3=20=E2=86=92=201.8.1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Changes:
* https://github.com/flatpak/flatpak/releases/tag/1.7.1
* https://github.com/flatpak/flatpak/releases/tag/1.7.2
* https://github.com/flatpak/flatpak/releases/tag/1.7.3
* https://github.com/flatpak/flatpak/releases/tag/1.8.0
* https://github.com/flatpak/flatpak/releases/tag/1.8.1

Commits:
https://github.com/flatpak/flatpak/compare/1.6.3...1.7.1
https://github.com/flatpak/flatpak/compare/1.7.1...1.8.1

Notable packaging changes:
* Flatpak now ships a sysusers.d file for allowing systemd to create the required users.
  https://github.com/flatpak/flatpak/commit/4df019063b962d17a378fb92d073f8ceb6a6ca62
* Completion support for fish shell
* If an app has filesystem access, the host /lib is accessible as /run/host/lib, etc.
* New filesystem permission "host-etc" and "host-os" give access to system /usr and /etc.
  https://github.com/flatpak/flatpak/commit/fe2536b8441858e3b22f6780dca64a516ee4e48c
* We now always expose the host timezone data, allowing us the expose the host /etc/localtime in a way that works better, fixing several apps that had timezone issues.
  https://github.com/flatpak/flatpak/commit/dc4e198766d91aa8871c276b9ed8d7f3ba003822
* We now ship a systemd unit (not installed by default) to automatically detect plugged in usb sticks with sideload repos.
* By default we no longer install the gdm env.d file, as the systemd generators work better
  https://github.com/flatpak/flatpak/commit/7c3a85bf4337874719324a03cc5875708c84ee85
* Use variant-schema-compiler for some GVariant code
  https://github.com/flatpak/flatpak/pull/3366
* zstd compression for oci deltas:
  https://github.com/flatpak/flatpak/commit/bfa71e208a3672d965f560fd5250787d9b933493

Additionally:
* Remove glibcLocales which is not used since 1.4 bump because glibc contains a locale archive with C.UTF-8
  https://github.com/NixOS/nixpkgs/commit/1728bc8d22ffe5e340dfb7cf3a8627f3638f5616
* Stop using aliases for docbook-xsl-nons and pkg-config packages
* Stop using autoreconfHook, the autogen.sh script contains some extra that are necessary when building from git.
* Increase disk space for installed tests, they were running out.
* Enable building developer documentation.
---
 nixos/modules/services/desktops/flatpak.nix   |  1 +
 .../development/libraries/flatpak/default.nix | 54 ++++++++++++++-----
 .../libraries/flatpak/fix-test-paths.patch    | 31 ++++++-----
 3 files changed, 62 insertions(+), 24 deletions(-)

diff --git a/nixos/modules/services/desktops/flatpak.nix b/nixos/modules/services/desktops/flatpak.nix
index 7fb0024f37dc..7da92cc9f264 100644
--- a/nixos/modules/services/desktops/flatpak.nix
+++ b/nixos/modules/services/desktops/flatpak.nix
@@ -42,6 +42,7 @@ in {
     # It has been possible since https://github.com/flatpak/flatpak/releases/tag/1.3.2
     # to build a SELinux policy module.
 
+    # TODO: use sysusers.d
     users.users.flatpak = {
       description = "Flatpak system helper";
       group = "flatpak";
diff --git a/pkgs/development/libraries/flatpak/default.nix b/pkgs/development/libraries/flatpak/default.nix
index ceca7e24fb6e..40cea4f730e7 100644
--- a/pkgs/development/libraries/flatpak/default.nix
+++ b/pkgs/development/libraries/flatpak/default.nix
@@ -1,21 +1,23 @@
 { stdenv
 , fetchurl
-, autoreconfHook
+, fetchpatch
+, autoconf
+, automake
+, libtool
 , docbook_xml_dtd_412
 , docbook_xml_dtd_42
 , docbook_xml_dtd_43
-, docbook_xsl
+, docbook-xsl-nons
 , which
 , libxml2
 , gobject-introspection
 , gtk-doc
 , intltool
 , libxslt
-, pkgconfig
+, pkg-config
 , xmlto
 , appstream-glib
 , substituteAll
-, glibcLocales
 , yacc
 , xdg-dbus-proxy
 , p11-kit
@@ -39,6 +41,7 @@
 , nixosTests
 , libsoup
 , lzma
+, zstd
 , ostree
 , polkit
 , python3
@@ -54,14 +57,14 @@
 
 stdenv.mkDerivation rec {
   pname = "flatpak";
-  version = "1.6.3";
+  version = "1.8.1";
 
   # TODO: split out lib once we figure out what to do with triggerdir
-  outputs = [ "out" "dev" "man" "doc" "installedTests" ];
+  outputs = [ "out" "dev" "man" "doc" "devdoc" "installedTests" ];
 
   src = fetchurl {
     url = "https://github.com/flatpak/flatpak/releases/download/${version}/${pname}-${version}.tar.xz";
-    sha256 = "17s8nqdxd4xdy7ag9bw06adxccha78jmlsa3zpqnl3qh92pg0hji";
+    sha256 = "ZpFLZvmmQHk4bMCXpAoZ+oQZVo33+0VvLkB/D3asnq0=";
   };
 
   patches = [
@@ -69,7 +72,7 @@ stdenv.mkDerivation rec {
     # https://github.com/flatpak/flatpak/issues/1460
     (substituteAll {
       src = ./fix-test-paths.patch;
-      inherit coreutils gettext glibcLocales socat gtk3;
+      inherit coreutils gettext socat gtk3;
       smi = shared-mime-info;
       dfu = desktop-file-utils;
       hicolorIconTheme = hicolor-icon-theme;
@@ -101,21 +104,31 @@ stdenv.mkDerivation rec {
 
     # But we want the GDK_PIXBUF_MODULE_FILE from the wrapper affect the icon validator.
     ./validate-icon-pixbuf.patch
+
+    # Fix `flatpak/test-oci-registry@{user,system}.wrap.test` installed tests.
+    # https://github.com/flatpak/flatpak/pull/3762
+    (fetchpatch {
+      url = "https://github.com/flatpak/flatpak/commit/c1447dadecd50f384b6d11dac18b014245267d00.patch";
+      sha256 = "UAA/wGr8/aMbx5MV+8Ilro2kgKkx2QOn88lDUjCgeDA=";
+    })
   ];
 
   nativeBuildInputs = [
-    autoreconfHook
+    autoconf
+    automake
+    libtool
     libxml2
+    # TODO: replace with docbook_xml_dtd_45 https://github.com/flatpak/flatpak/pull/3760
     docbook_xml_dtd_412
     docbook_xml_dtd_42
     docbook_xml_dtd_43
-    docbook_xsl
+    docbook-xsl-nons
     which
     gobject-introspection
     gtk-doc
     intltool
     libxslt
-    pkgconfig
+    pkg-config
     xmlto
     appstream-glib
     yacc
@@ -134,6 +147,7 @@ stdenv.mkDerivation rec {
     libseccomp
     libsoup
     lzma
+    # zstd # TODO: broken paths in .pc file
     polkit
     python3
     systemd
@@ -166,6 +180,7 @@ stdenv.mkDerivation rec {
     "--with-system-dbus-proxy=${xdg-dbus-proxy}/bin/xdg-dbus-proxy"
     "--with-dbus-config-dir=${placeholder "out"}/share/dbus-1/system.d"
     "--localstatedir=/var"
+    "--enable-gtk-doc"
     "--enable-installed-tests"
   ];
 
@@ -174,9 +189,24 @@ stdenv.mkDerivation rec {
     "installed_test_metadir=${placeholder "installedTests"}/share/installed-tests/flatpak"
   ];
 
-  postPatch = ''
+  postPatch = let
+    vsc-py = python3.withPackages (pp: [
+      pp.pyparsing
+    ]);
+  in ''
     patchShebangs buildutil
     patchShebangs tests
+    PATH=${stdenv.lib.makeBinPath [vsc-py]}:$PATH patchShebangs --build variant-schema-compiler/variant-schema-compiler
+  '';
+
+  preConfigure = ''
+    # TODO: remove the condition once autogen.sh is shipped in the tarball
+    # https://github.com/flatpak/flatpak/pull/3761
+    if [[ -f autogen.sh ]]; then
+        NOCONFIGURE=1 ./autogen.sh
+    else
+        autoreconf --install --force --verbose
+    fi
   '';
 
   passthru = {
diff --git a/pkgs/development/libraries/flatpak/fix-test-paths.patch b/pkgs/development/libraries/flatpak/fix-test-paths.patch
index 8ea2f0159a31..dcba66681858 100644
--- a/pkgs/development/libraries/flatpak/fix-test-paths.patch
+++ b/pkgs/development/libraries/flatpak/fix-test-paths.patch
@@ -1,5 +1,5 @@
 diff --git a/app/flatpak-builtins-build-export.c b/app/flatpak-builtins-build-export.c
-index 5de89d62..bf6bdb52 100644
+index 43fd0563..4067bd36 100644
 --- a/app/flatpak-builtins-build-export.c
 +++ b/app/flatpak-builtins-build-export.c
 @@ -458,7 +458,7 @@ validate_desktop_file (GFile      *desktop_file,
@@ -12,10 +12,10 @@ index 5de89d62..bf6bdb52 100644
      {
        if (!g_error_matches (local_error, G_SPAWN_ERROR, G_SPAWN_ERROR_NOENT))
 diff --git a/tests/libtest.sh b/tests/libtest.sh
-index e64be49f..a9a53e12 100644
+index acb2095b..b361da16 100644
 --- a/tests/libtest.sh
 +++ b/tests/libtest.sh
-@@ -367,7 +367,7 @@ if [ -z "${FLATPAK_BWRAP:-}" ]; then
+@@ -431,7 +431,7 @@ if [ -z "${FLATPAK_BWRAP:-}" ]; then
      # running installed-tests: assume we know what we're doing
      _flatpak_bwrap_works=true
  elif ! "$FLATPAK_BWRAP" --unshare-ipc --unshare-net --unshare-pid \
@@ -24,7 +24,7 @@ index e64be49f..a9a53e12 100644
      _flatpak_bwrap_works=false
  else
      _flatpak_bwrap_works=true
-@@ -440,7 +440,7 @@ dbus-daemon --fork --config-file=session.conf --print-address=3 --print-pid=4 \
+@@ -504,7 +504,7 @@ dbus-daemon --fork --config-file=session.conf --print-address=3 --print-pid=4 \
  export DBUS_SESSION_BUS_ADDRESS="$(cat dbus-session-bus-address)"
  DBUS_SESSION_BUS_PID="$(cat dbus-session-bus-pid)"
  
@@ -33,7 +33,7 @@ index e64be49f..a9a53e12 100644
      assert_not_reached "Failed to start dbus-daemon"
  fi
  
-@@ -449,7 +449,7 @@ gdb_bt () {
+@@ -519,7 +519,7 @@ commit_to_path () {
  }
  
  cleanup () {
@@ -43,10 +43,10 @@ index e64be49f..a9a53e12 100644
      fusermount -u $XDG_RUNTIME_DIR/doc || :
      kill $(jobs -p) &> /dev/null || true
 diff --git a/tests/make-test-app.sh b/tests/make-test-app.sh
-index e51e21a6..7d39efb5 100755
+index 612f914f..4b890da2 100755
 --- a/tests/make-test-app.sh
 +++ b/tests/make-test-app.sh
-@@ -149,13 +149,13 @@ msgid "Hello world"
+@@ -151,13 +151,13 @@ msgid "Hello world"
  msgstr "Hallo Welt"
  EOF
  mkdir -p ${DIR}/files/de/share/de/LC_MESSAGES
@@ -63,10 +63,10 @@ index e51e21a6..7d39efb5 100755
  flatpak build-finish ${DIR}
  mkdir -p repos
 diff --git a/tests/make-test-runtime.sh b/tests/make-test-runtime.sh
-index 5d2c309b..cf61a3cf 100755
+index af289625..7db51c17 100755
 --- a/tests/make-test-runtime.sh
 +++ b/tests/make-test-runtime.sh
-@@ -25,9 +25,10 @@ EOF
+@@ -28,9 +28,10 @@ EOF
  
  # On Debian derivatives, /usr/sbin and /sbin aren't in ordinary users'
  # PATHs, but ldconfig is kept in /sbin
@@ -78,7 +78,7 @@ index 5d2c309b..cf61a3cf 100755
  mkdir -p ${DIR}/usr/bin
  mkdir -p ${DIR}/usr/lib
  ln -s ../lib ${DIR}/usr/lib64
-@@ -37,48 +38,24 @@ if test -f /sbin/ldconfig.real; then
+@@ -40,40 +41,17 @@ if test -f /sbin/ldconfig.real; then
  else
      cp `which ldconfig` ${DIR}/usr/bin
  fi
@@ -129,11 +129,18 @@ index 5d2c309b..cf61a3cf 100755
  done
  ln -s bash ${DIR}/usr/bin/sh
  
+@@ -84,11 +62,13 @@ echo "Hello world, from a runtime$EXTRA"
+ EOF
+ chmod a+x ${DIR}/usr/bin/runtime_hello.sh
+ 
 -# We copy the C.UTF8 locale and call it en_US. Its a bit of a lie, but
 -# the real en_US locale is often not available, because its in the
 -# local archive.
 -mkdir -p ${DIR}/usr/lib/locale/
 -cp -r /usr/lib/locale/C.* ${DIR}/usr/lib/locale/en_US
++# We do not copy C.UTF8 locale because it is in locale archive and
++# that is already copied with glibc.
++
 +mv ${DIR}/nix/store ${DIR}/usr/store # files outside /usr are not permitted, we will have to replace /nix/store with /usr/store
 +chmod -R u+w ${DIR} # nix store has read-only directories which would cause problems during clean-up, files need to be writable for sed
 +find ${DIR} -type f -print0 | xargs -0 sed -i 's~/nix/store/~/usr/store/~g' # replace hardcoded paths
@@ -142,10 +149,10 @@ index 5d2c309b..cf61a3cf 100755
  if [ x$COLLECTION_ID != x ]; then
      collection_args=--collection-id=${COLLECTION_ID}
 diff --git a/tests/testlibrary.c b/tests/testlibrary.c
-index 44ae28e3..76bf619f 100644
+index 509ce6cc..6a333c9c 100644
 --- a/tests/testlibrary.c
 +++ b/tests/testlibrary.c
-@@ -1343,7 +1343,7 @@ check_bwrap_support (void)
+@@ -1399,7 +1399,7 @@ check_bwrap_support (void)
      {
        gint exit_code = 0;
        char *argv[] = { (char *) bwrap, "--unshare-ipc", "--unshare-net",

From 379038b4dc818c89c931a4ce6cfc14c56fc811ec Mon Sep 17 00:00:00 2001
From: Jan Tojnar <jtojnar@gmail.com>
Date: Fri, 24 Jul 2020 21:02:02 +0200
Subject: [PATCH 4/4] nixosTests.flatpak: clean up

GNOME is not necessary. Portals probably are not either,
but the NixOS module requires them.

Not sure why it did not work without GNOME before.
---
 nixos/tests/installed-tests/flatpak.nix | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/nixos/tests/installed-tests/flatpak.nix b/nixos/tests/installed-tests/flatpak.nix
index 6b83d006f162..8aeeaca90f61 100644
--- a/nixos/tests/installed-tests/flatpak.nix
+++ b/nixos/tests/installed-tests/flatpak.nix
@@ -5,10 +5,7 @@ makeInstalledTest {
   withX11 = true;
 
   testConfig = {
-    services.xserver.desktopManager.gnome3.enable = true; # TODO: figure out minimal environment where the tests work
-    # common/x11.nix enables the auto display manager (lightdm)
-    services.xserver.displayManager.gdm.enable = false;
-    services.gnome3.core-utilities.enable = false;
+    xdg.portal.enable = true;
     services.flatpak.enable = true;
     environment.systemPackages = with pkgs; [ gnupg ostree python3 ];
     virtualisation.memorySize = 2047;