Merge pull request #155850 from cpcloud/bump-ibis-framework

python3Packages.ibis-framework: 1.3.0 -> 2.1.1

pkgs/development/python-modules/ibis-framework/default.nix

@@ -1,10 +1,23 @@
-{ lib, buildPythonPackage, fetchPypi, fetchpatch, isPy27, pythonAtLeast
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, pythonOlder
+, pytestCheckHook
+, atpublic
+, cached-property
+, clickhouse-driver
+, click
+, dask
 , graphviz
+, importlib-metadata
 , multipledispatch
 , numpy
 , pandas
+, parsy
 , pyarrow
 , pytest
+, pytest-mock
+, pytest-xdist
 , pytz
 , regex
 , requests
@@ -12,54 +25,117 @@
 , tables
 , toolz
 }:
+let
+  # ignore tests for which dependencies are not available
+  backends = [
+    "csv"
+    "dask"
+    "hdf5"
+    "pandas"
+    "parquet"
+    "sqlite"
+  ];
+
+  backendsString = lib.concatStringsSep " " backends;
+
+  ibisTestingData = fetchFromGitHub {
+    owner = "ibis-project";
+    repo = "testing-data";
+    rev = "743201a35c6b968cf55b054f9d28949ea15d1f0a";
+    sha256 = "sha256-xuSE6wHP3aF8lnEE2SuFbTRBu49ecRmc1F3HPcszptI=";
+  };
+in
 
 buildPythonPackage rec {
   pname = "ibis-framework";
-  version = "1.3.0";
-  disabled = isPy27 || pythonAtLeast "3.8";
+  version = "2.1.1";
+  disabled = pythonOlder "3.7";
 
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "1my94a11jzg1hv6ln8wxklbqrg6z5l2l77vr89aq0829yyxacmv7";
+  src = fetchFromGitHub {
+    repo = "ibis";
+    owner = "ibis-project";
+    rev = version;
+    sha256 = "sha256-n3fR6wvcSfIo7760seB+5SxtoYSqQmqkzZ9VlNQF200=";
   };
 
-  patches = [
-    # fix tests for pandas 1.1
-    (fetchpatch {
-      url = "https://github.com/ibis-project/ibis/commit/53ef3cefc4ae90d61f3612310cb36da2bcd11305.diff";
-      sha256 = "1i5yjmqridjqpggiinsjaz5spcxca5bd48vy7a0mj4mm1b5flw2m";
-    })
-  ];
-
   propagatedBuildInputs = [
+    atpublic
+    cached-property
+    clickhouse-driver
+    dask
+    graphviz
     multipledispatch
     numpy
     pandas
+    parsy
+    pyarrow
     pytz
     regex
-    toolz
-    sqlalchemy
     requests
-    graphviz
+    sqlalchemy
     tables
-    pyarrow
-  ];
+    toolz
+  ] ++ lib.optionals (pythonOlder "3.8") [ importlib-metadata ];
 
   checkInputs = [
+    pytestCheckHook
+    click
     pytest
+    pytest-mock
+    pytest-xdist
   ];
 
-  # ignore tests which require test dataset, or frameworks not available
-  checkPhase = ''
-    pytest ibis \
-      --ignore=ibis/tests/all \
-      --ignore=ibis/{sql,spark}
+  # these tests are broken upstream: https://github.com/ibis-project/ibis/issues/3291
+  disabledTests = [
+    "test_summary_numeric"
+    "test_summary_non_numeric"
+    "test_batting_most_hits"
+    "test_join_with_window_function"
+    "test_where_long"
+    "test_quantile_groupby"
+    "test_summary_numeric"
+    "test_summary_numeric_group_by"
+    "test_summary_non_numeric"
+    "test_searched_case_column"
+    "test_simple_case_column"
+    "test_summary_non_numeric_group_by"
+  ];
+
+  pytestFlagsArray = [
+    "--numprocesses $NIX_BUILD_CORES"
+    "ibis/tests"
+    "ibis/backends/tests"
+    "ibis/backends/{${lib.concatStringsSep "," backends}}/tests"
+  ];
+
+  preCheck = ''
+    set -euo pipefail
+
+    export IBIS_TEST_DATA_DIRECTORY
+    IBIS_TEST_DATA_DIRECTORY="$(mktemp -d)"
+
+    # copy the test data to a writable directory
+    cp -r ${ibisTestingData}/* "$IBIS_TEST_DATA_DIRECTORY"
+
+    find "$IBIS_TEST_DATA_DIRECTORY" -type d -exec chmod u+rwx {} +
+    find "$IBIS_TEST_DATA_DIRECTORY" -type f -exec chmod u+rw {} +
+
+    # load data
+    for backend in ${backendsString}; do
+      python ci/datamgr.py "$backend" &
+    done
+
+    wait
+
+    export PYTEST_BACKENDS="${backendsString}"
   '';
 
+  pythonImportsCheck = [ "ibis" ] ++ (map (backend: "ibis.backends.${backend}") backends);
+
   meta = with lib; {
     description = "Productivity-centric Python Big Data Framework";
     homepage = "https://github.com/ibis-project/ibis";
     license = licenses.asl20;
-    maintainers = [ maintainers.costrouc ];
+    maintainers = with maintainers; [ costrouc cpcloud ];
   };
 }