3
0
Fork 0
forked from mirrors/nixpkgs

Merge pull request #44086 from erikarvstedt/paperless

paperless: add package and service
This commit is contained in:
worldofpeace 2019-05-08 17:17:49 -04:00 committed by GitHub
commit bb7e5566c7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 694 additions and 0 deletions

View file

@ -339,6 +339,7 @@
rss2email = 312;
cockroachdb = 313;
zoneminder = 314;
paperless = 315;
# When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399!
@ -638,6 +639,7 @@
rss2email = 312;
cockroachdb = 313;
zoneminder = 314;
paperless = 315;
# When adding a gid, make sure it doesn't match an existing
# uid. Users and groups with the same name should have equal

View file

@ -435,6 +435,7 @@
./services/misc/octoprint.nix
./services/misc/osrm.nix
./services/misc/packagekit.nix
./services/misc/paperless.nix
./services/misc/parsoid.nix
./services/misc/phd.nix
./services/misc/plex.nix

View file

@ -0,0 +1,185 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.paperless;
defaultUser = "paperless";
manage = cfg.package.withConfig {
config = {
PAPERLESS_CONSUMPTION_DIR = cfg.consumptionDir;
PAPERLESS_INLINE_DOC = "true";
PAPERLESS_DISABLE_LOGIN = "true";
} // cfg.extraConfig;
inherit (cfg) dataDir ocrLanguages;
paperlessPkg = cfg.package;
};
in
{
options.services.paperless = {
enable = mkOption {
type = lib.types.bool;
default = false;
description = ''
Enable Paperless.
When started, the Paperless database is automatically created if it doesn't
exist and updated if the Paperless package has changed.
Both tasks are achieved by running a Django migration.
'';
};
dataDir = mkOption {
type = types.str;
default = "/var/lib/paperless";
description = "Directory to store the Paperless data.";
};
consumptionDir = mkOption {
type = types.str;
default = "${cfg.dataDir}/consume";
defaultText = "\${dataDir}/consume";
description = "Directory from which new documents are imported.";
};
consumptionDirIsPublic = mkOption {
type = types.bool;
default = false;
description = "Whether all users can write to the consumption dir.";
};
ocrLanguages = mkOption {
type = with types; nullOr (listOf string);
default = null;
description = ''
Languages available for OCR via Tesseract, specified as
<literal>ISO 639-2/T</literal> language codes.
If unset, defaults to all available languages.
'';
example = [ "eng" "spa" "jpn" ];
};
address = mkOption {
type = types.str;
default = "localhost";
description = "Server listening address.";
};
port = mkOption {
type = types.int;
default = 28981;
description = "Server port to listen on.";
};
extraConfig = mkOption {
type = types.attrs;
default = {};
description = ''
Extra paperless config options.
The config values are evaluated as double-quoted Bash string literals.
See <literal>paperless-src/paperless.conf.example</literal> for available options.
To enable user authentication, set <literal>PAPERLESS_DISABLE_LOGIN = "false"</literal>
and run the shell command <literal>$dataDir/paperless-manage createsuperuser</literal>.
To define secret options without storing them in /nix/store, use the following pattern:
<literal>PAPERLESS_PASSPHRASE = "$(&lt; /etc/my_passphrase_file)"</literal>
'';
example = literalExample ''
{
PAPERLESS_OCR_LANGUAGE = "deu";
}
'';
};
user = mkOption {
type = types.str;
default = defaultUser;
description = "User under which Paperless runs.";
};
package = mkOption {
type = types.package;
default = pkgs.paperless;
defaultText = "pkgs.paperless";
description = "The Paperless package to use.";
};
manage = mkOption {
type = types.package;
readOnly = true;
default = manage;
description = ''
A script to manage the Paperless instance.
It wraps Django's manage.py and is also available at
<literal>$dataDir/manage-paperless</literal>
'';
};
};
config = mkIf cfg.enable {
systemd.tmpfiles.rules = [
"d '${cfg.dataDir}' - ${cfg.user} ${cfg.user} - -"
] ++ (optional cfg.consumptionDirIsPublic
"d '${cfg.consumptionDir}' 777 ${cfg.user} ${cfg.user} - -"
# If the consumption dir is not created here, it's automatically created by
# 'manage' with the default permissions.
);
systemd.services.paperless-consumer = {
description = "Paperless document consumer";
serviceConfig = {
User = cfg.user;
ExecStart = "${manage} document_consumer";
Restart = "always";
};
after = [ "systemd-tmpfiles-setup.service" ];
wantedBy = [ "multi-user.target" ];
preStart = ''
if [[ $(readlink ${cfg.dataDir}/paperless-manage) != ${manage} ]]; then
ln -sf ${manage} ${cfg.dataDir}/paperless-manage
fi
${manage.setupEnv}
# Auto-migrate on first run or if the package has changed
versionFile="$PAPERLESS_DBDIR/src-version"
if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then
python $paperlessSrc/manage.py migrate
echo ${cfg.package} > "$versionFile"
fi
'';
};
systemd.services.paperless-server = {
description = "Paperless document server";
serviceConfig = {
User = cfg.user;
ExecStart = "${manage} runserver --noreload ${cfg.address}:${toString cfg.port}";
Restart = "always";
};
# Bind to `paperless-consumer` so that the server never runs
# during migrations
bindsTo = [ "paperless-consumer.service" ];
after = [ "paperless-consumer.service" ];
wantedBy = [ "multi-user.target" ];
};
users = optionalAttrs (cfg.user == defaultUser) {
users = [{
name = defaultUser;
group = defaultUser;
uid = config.ids.uids.paperless;
home = cfg.dataDir;
}];
groups = [{
name = defaultUser;
gid = config.ids.gids.paperless;
}];
};
};
}

View file

@ -189,6 +189,7 @@ in
pam-oath-login = handleTest ./pam-oath-login.nix {};
pam-u2f = handleTest ./pam-u2f.nix {};
pantheon = handleTest ./pantheon.nix {};
paperless = handleTest ./paperless.nix {};
peerflix = handleTest ./peerflix.nix {};
pgjwt = handleTest ./pgjwt.nix {};
pgmanage = handleTest ./pgmanage.nix {};

29
nixos/tests/paperless.nix Normal file
View file

@ -0,0 +1,29 @@
import ./make-test.nix ({ lib, ... } : {
name = "paperless";
meta = with lib.maintainers; {
maintainers = [ earvstedt ];
};
machine = { pkgs, ... }: {
environment.systemPackages = with pkgs; [ imagemagick jq ];
services.paperless = {
enable = true;
ocrLanguages = [ "eng" ];
};
};
testScript = ''
$machine->waitForUnit("paperless-consumer.service");
# Create test doc
$machine->succeed('convert -size 400x40 xc:white -font "DejaVu-Sans" -pointsize 20 -fill black \
-annotate +5+20 "hello world 16-10-2005" /var/lib/paperless/consume/doc.png');
$machine->waitForUnit("paperless-server.service");
# Wait until server accepts connections
$machine->waitUntilSucceeds("curl -s localhost:28981");
# Wait until document is consumed
$machine->waitUntilSucceeds('(($(curl -s localhost:28981/api/documents/ | jq .count) == 1))');
$machine->succeed("curl -s localhost:28981/api/documents/ | jq '.results | .[0] | .created'")
=~ /2005-10-16/ or die;
'';
})

View file

@ -0,0 +1,170 @@
{ stdenv
, lib
, fetchFromGitHub
, makeWrapper
, callPackage
, python3
, file
, imagemagick7
, ghostscript
, optipng
, poppler
, tesseract
, unpaper
}:
## Usage
# ${paperless}/bin/paperless wraps manage.py
# ${paperless}/share/paperless/setup-env.sh can be sourced from a
# shell script to setup a Paperless environment
# paperless.withConfig is a convenience function to setup a
# configured Paperless instance. (See ./withConfig.nix)
# For WSGI with gunicorn, use a shell script like this:
# let
# pythonEnv = paperless.python.withPackages (ps: paperless.runtimePackages ++ [ ps.gunicorn ]);
# in
# writers.writeBash "run-gunicorn" ''
# source ${paperless}/share/paperless/setup-env.sh
# PYTHONPATH=$paperlessSrc ${pythonEnv}/bin/gunicorn paperless.wsgi
# ''
let
paperless = stdenv.mkDerivation rec {
name = "paperless-${version}";
version = "2.7.0";
src = fetchFromGitHub {
owner = "the-paperless-project";
repo = "paperless";
rev = version;
sha256 = "0pkmyky1crjnsg7r0gfk0fadisfsgzlsq6afpz16wx4hp6yvkkf7";
};
nativeBuildInputs = [ makeWrapper ];
doCheck = true;
dontInstall = true;
pythonEnv = python.withPackages (_: runtimePackages);
pythonCheckEnv = python.withPackages (_: (runtimePackages ++ checkPackages));
unpackPhase = ''
srcDir=$out/share/paperless
mkdir -p $srcDir
cp -r --no-preserve=mode $src/src/* $src/LICENSE $srcDir
'';
buildPhase = let
# Paperless has explicit runtime checks that expect these binaries to be in PATH
extraBin = lib.makeBinPath [ imagemagick7 ghostscript optipng tesseract unpaper ];
in ''
${python.interpreter} -m compileall $srcDir
makeWrapper $pythonEnv/bin/python $out/bin/paperless \
--set PATH ${extraBin} --add-flags $out/share/paperless/manage.py
# A shell snippet that can be sourced to setup a paperless env
cat > $out/share/paperless/setup-env.sh <<EOF
export PATH="$pythonEnv/bin:${extraBin}''${PATH:+:}$PATH"
export paperlessSrc=$out/share/paperless
EOF
'';
checkPhase = ''
source $out/share/paperless/setup-env.sh
tmpDir=$(realpath testsTmp)
mkdir $tmpDir
export HOME=$tmpDir
export PAPERLESS_MEDIADIR=$tmpDir
cd $paperlessSrc
# Prevent tests from writing to the derivation output
chmod -R -w $out
# Disable cache to silence a pytest warning ("could not create cache")
$pythonCheckEnv/bin/pytest -p no:cacheprovider
'';
passthru = {
withConfig = callPackage ./withConfig.nix {};
inherit python runtimePackages checkPackages tesseract;
};
meta = with lib; {
description = "Scan, index, and archive all of your paper documents";
homepage = https://github.com/the-paperless-project/paperless;
license = licenses.gpl3;
maintainers = [ maintainers.earvstedt ];
};
};
python = python3.override {
packageOverrides = self: super: {
# Paperless only supports Django 2.0
django = django_2_0 super;
pyocr = pyocrWithUserTesseract super;
# These are pre-release versions, hence they are private to this pkg
django-filter = self.callPackage ./python-modules/django-filter.nix {};
django-crispy-forms = self.callPackage ./python-modules/django-crispy-forms.nix {};
};
};
django_2_0 = pyPkgs: pyPkgs.django_2_1.overrideDerivation (_: rec {
pname = "Django";
version = "2.0.12";
name = "${pname}-${version}";
src = pyPkgs.fetchPypi {
inherit pname version;
sha256 = "15s8z54k0gf9brnz06521bikm60ddw5pn6v3nbvnl47j1jjsvwz2";
};
});
runtimePackages = with python.pkgs; [
dateparser
dateutil
django
django-cors-headers
django-crispy-forms
django-filter
django_extensions
djangoql
djangorestframework
factory_boy
filemagic
fuzzywuzzy
langdetect
pdftotext
pillow
psycopg2
pyocr
python-dotenv
python-gnupg
pytz
termcolor
] ++ (lib.optional stdenv.isLinux inotify-simple);
checkPackages = with python.pkgs; [
pytest
pytest-django
pytest-env
pytest_xdist
];
pyocrWithUserTesseract = pyPkgs:
let
pyocr = pyPkgs.pyocr.override { inherit tesseract; };
in
if pyocr.outPath == pyPkgs.pyocr.outPath then
pyocr
else
# The user has provided a custom tesseract derivation that might be
# missing some languages that are required for PyOCR's tests. Disable them to
# avoid build errors.
pyocr.overridePythonAttrs (attrs: {
doCheck = false;
});
in
paperless

View file

@ -0,0 +1,36 @@
{ lib, buildPythonPackage, fetchFromGitHub
, pytest, pytest-django, django }:
buildPythonPackage rec {
pname = "django-crispy-forms";
version = "2019.04.21";
src = fetchFromGitHub {
owner = "django-crispy-forms";
repo = "django-crispy-forms";
rev = "e25a5326697e5b545689b3a914e516404a6911bb";
sha256 = "12zqa76q6i7j47aqvhilivpbdplgp9zw2q8zfcjzlgclrqafaj39";
};
# For reasons unknown, the source dir must contain a dash
# for the tests to run successfully
postUnpack = ''
mv $sourceRoot source-
export sourceRoot=source-
'';
checkInputs = [ pytest pytest-django django ];
checkPhase = ''
PYTHONPATH="$(pwd):$PYTHONPATH" \
DJANGO_SETTINGS_MODULE=crispy_forms.tests.test_settings \
pytest crispy_forms/tests
'';
meta = with lib; {
description = "The best way to have DRY Django forms";
homepage = https://github.com/maraujop/django-crispy-forms;
license = licenses.mit;
maintainers = with maintainers; [ earvstedt ];
};
}

View file

@ -0,0 +1,26 @@
{ lib, buildPythonPackage, python, pythonOlder, fetchFromGitHub
, django, django-crispy-forms, djangorestframework, mock, pytz }:
buildPythonPackage rec {
pname = "django-filter";
version = "2.1.0-pre";
disabled = pythonOlder "3.4";
src = fetchFromGitHub {
owner = "carltongibson";
repo = pname;
rev = "24adad8c48bc9e7c7539b6510ffde4ce4effdc29";
sha256 = "0hv4w95jnlzp9vdximl6bb27fyi75001jhvsbs0ikkd8amq8iaj7";
};
checkInputs = [ django django-crispy-forms djangorestframework mock pytz ];
checkPhase = "${python.interpreter} runtests.py";
meta = with lib; {
description = "A reusable Django application for allowing users to filter querysets dynamically.";
homepage = https://github.com/carltongibson/django-filter;
license = licenses.bsd3;
maintainers = with maintainers; [ earvstedt ];
};
}

View file

@ -0,0 +1,68 @@
{ paperless, lib, writers }:
## Usage
#
# nix-build --out-link ./paperless -E '
# (import <nixpkgs> {}).paperless.withConfig {
# dataDir = /tmp/paperless-data;
# config = {
# PAPERLESS_DISABLE_LOGIN = "true";
# };
# }'
#
# Setup DB
# ./paperless migrate
#
# Consume documents in ${dataDir}/consume
# ./paperless document_consumer --oneshot
#
# Start web interface
# ./paperless runserver --noreload localhost:8000
{ config ? {}, dataDir ? null, ocrLanguages ? null
, paperlessPkg ? paperless, extraCmds ? "" }:
with lib;
let
paperless = if ocrLanguages == null then
paperlessPkg
else
(paperlessPkg.override {
tesseract = paperlessPkg.tesseract.override {
enableLanguages = ocrLanguages;
};
}).overrideDerivation (_: {
# `ocrLanguages` might be missing some languages required by the tests.
doCheck = false;
});
envVars = (optionalAttrs (dataDir != null) {
PAPERLESS_CONSUMPTION_DIR = "${dataDir}/consume";
PAPERLESS_MEDIADIR = "${dataDir}/media";
PAPERLESS_STATICDIR = "${dataDir}/static";
PAPERLESS_DBDIR = "${dataDir}";
}) // config;
envVarDefs = mapAttrsToList (n: v: ''export ${n}="${toString v}"'') envVars;
setupEnvVars = builtins.concatStringsSep "\n" envVarDefs;
setupEnv = ''
source ${paperless}/share/paperless/setup-env.sh
${setupEnvVars}
${optionalString (dataDir != null) ''
mkdir -p "$PAPERLESS_CONSUMPTION_DIR" \
"$PAPERLESS_MEDIADIR" \
"$PAPERLESS_STATICDIR" \
"$PAPERLESS_DBDIR"
''}
'';
runPaperless = writers.writeBash "paperless" ''
set -e
${setupEnv}
${extraCmds}
exec python $paperlessSrc/manage.py "$@"
'';
in
runPaperless // {
inherit paperless setupEnv;
}

View file

@ -0,0 +1,28 @@
{ lib, buildPythonPackage, fetchPypi, python
, django, ply }:
buildPythonPackage rec {
pname = "djangoql";
version = "0.12.6";
src = fetchPypi {
inherit pname version;
sha256 = "1mwv1ljznj9mn74ncvcyfmj6ygs8xm2rajpxm88gcac9hhdmk5gs";
};
propagatedBuildInputs = [ ply ];
checkInputs = [ django ];
checkPhase = ''
export PYTHONPATH=test_project:$PYTHONPATH
${python.executable} test_project/manage.py test core.tests
'';
meta = with lib; {
description = "Advanced search language for Django";
homepage = https://github.com/ivelum/djangoql;
license = licenses.mit;
maintainers = with maintainers; [ earvstedt ];
};
}

View file

@ -0,0 +1,29 @@
{ stdenv, lib, buildPythonPackage, fetchFromGitHub, file
, isPy3k, mock, unittest2 }:
buildPythonPackage rec {
pname = "filemagic";
version = "1.6";
# Don't use the PyPI source because it's missing files required for testing
src = fetchFromGitHub {
owner = "aliles";
repo = "filemagic";
rev = "138649062f769fb10c256e454a3e94ecfbf3017b";
sha256 = "1jxf928jjl2v6zv8kdnfqvywdwql1zqkm1v5xn1d5w0qjcg38d4n";
};
postPatch = ''
substituteInPlace magic/api.py --replace "ctypes.util.find_library('magic')" \
"'${file}/lib/libmagic${stdenv.hostPlatform.extensions.sharedLibrary}'"
'';
checkInputs = [ (if isPy3k then mock else unittest2) ];
meta = with lib; {
description = "File type identification using libmagic";
homepage = https://github.com/aliles/filemagic;
license = licenses.asl20;
maintainers = with maintainers; [ earvstedt ];
};
}

View file

@ -0,0 +1,22 @@
{ lib, buildPythonPackage, fetchPypi }:
buildPythonPackage rec {
pname = "inotify-simple";
version = "1.1.8";
src = fetchPypi {
pname = "inotify_simple";
inherit version;
sha256 = "1pfqvnynwh318cakldhg7535kbs02asjsgv6s0ki12i7fgfi0b7w";
};
# The package has no tests
doCheck = false;
meta = with lib; {
description = "A simple Python wrapper around inotify";
homepage = https://github.com/chrisjbillington/inotify_simple;
license = licenses.bsd2;
maintainers = with maintainers; [ earvstedt ];
};
}

View file

@ -0,0 +1,21 @@
{ lib, buildPythonPackage, fetchPypi, six }:
buildPythonPackage rec {
pname = "langdetect";
version = "1.0.7";
src = fetchPypi {
inherit pname version;
extension = "zip";
sha256 = "0c5zm6c7xzsigbb9c7v4r33fcpz911zscfwvh3dq1qxdy3ap18ci";
};
propagatedBuildInputs = [ six ];
meta = with lib; {
description = "Python port of Google's language-detection library";
homepage = https://github.com/Mimino666/langdetect;
license = licenses.asl20;
maintainers = with maintainers; [ earvstedt ];
};
}

View file

@ -0,0 +1,20 @@
{ lib, buildPythonPackage, fetchPypi, poppler }:
buildPythonPackage rec {
pname = "pdftotext";
version = "2.1.1";
src = fetchPypi {
inherit pname version;
sha256 = "1jwc2zpss0983wqqi0kpichasljsxar9c4ma8vycn8maw3pi3bg3";
};
buildInputs = [ poppler ];
meta = with lib; {
description = "Simple PDF text extraction";
homepage = https://github.com/jalan/pdftotext;
license = licenses.mit;
maintainers = with maintainers; [ earvstedt ];
};
}

View file

@ -0,0 +1,20 @@
{ lib, buildPythonPackage, fetchPypi, pytest }:
buildPythonPackage rec {
pname = "pytest-env";
version = "0.6.2";
src = fetchPypi {
inherit pname version;
sha256 = "1hl0ln0cicdid4qjk7mv90lw9xkb0v71dlj7q7rn89vzxxm9b53y";
};
checkInputs = [ pytest ];
meta = with lib; {
description = "Pytest plugin used to set environment variables";
homepage = https://github.com/MobileDynasty/pytest-env;
license = licenses.mit;
maintainers = with maintainers; [ earvstedt ];
};
}

View file

@ -0,0 +1,20 @@
{ lib, buildPythonPackage, fetchPypi, click, ipython }:
buildPythonPackage rec {
pname = "python-dotenv";
version = "0.10.1";
src = fetchPypi {
inherit pname version;
sha256 = "1q4sp6ppjiqlsz3h43q9iya4n3qkhx6ng16bcbacfxdyrp9xvcf9";
};
checkInputs = [ click ipython ];
meta = with lib; {
description = "Add .env support to your django/flask apps in development and deployments";
homepage = https://github.com/theskumar/python-dotenv;
license = licenses.bsdOriginal;
maintainers = with maintainers; [ earvstedt ];
};
}

View file

@ -4988,6 +4988,8 @@ in
paper-gtk-theme = callPackage ../misc/themes/paper { };
paperless = callPackage ../applications/office/paperless { };
paperwork = callPackage ../applications/office/paperwork { };
papertrail = callPackage ../tools/text/papertrail { };

View file

@ -408,6 +408,8 @@ in {
fdint = callPackage ../development/python-modules/fdint { };
filemagic = callPackage ../development/python-modules/filemagic { };
fuse = callPackage ../development/python-modules/fuse-python {
inherit (pkgs) fuse pkgconfig;
};
@ -470,6 +472,8 @@ in {
imutils = callPackage ../development/python-modules/imutils { };
inotify-simple = callPackage ../development/python-modules/inotify-simple { };
intake = callPackage ../development/python-modules/intake { };
intelhex = callPackage ../development/python-modules/intelhex { };
@ -482,6 +486,8 @@ in {
mpi = pkgs.openmpi;
};
langdetect = callPackage ../development/python-modules/langdetect { };
libmr = callPackage ../development/python-modules/libmr { };
limitlessled = callPackage ../development/python-modules/limitlessled { };
@ -578,6 +584,8 @@ in {
pdfminer = callPackage ../development/python-modules/pdfminer_six { };
pdftotext = callPackage ../development/python-modules/pdftotext { };
pdfx = callPackage ../development/python-modules/pdfx { };
perf = callPackage ../development/python-modules/perf { };
@ -761,6 +769,8 @@ in {
pytest-click = callPackage ../development/python-modules/pytest-click { };
pytest-env = callPackage ../development/python-modules/pytest-env { };
pytest-mypy = callPackage ../development/python-modules/pytest-mypy { };
pytest-pylint = callPackage ../development/python-modules/pytest-pylint { };
@ -771,6 +781,8 @@ in {
python-dbusmock = callPackage ../development/python-modules/python-dbusmock { };
python-dotenv = callPackage ../development/python-modules/python-dotenv { };
python-engineio = callPackage ../development/python-modules/python-engineio { };
python-hosts = callPackage ../development/python-modules/python-hosts { };
@ -2491,6 +2503,8 @@ in {
django_pipeline = callPackage ../development/python-modules/django-pipeline { };
djangoql = callPackage ../development/python-modules/djangoql { };
dj-database-url = callPackage ../development/python-modules/dj-database-url { };
dj-email-url = callPackage ../development/python-modules/dj-email-url { };