From a6a729b207f995674c931cea3c2d818cc7f0ffc4 Mon Sep 17 00:00:00 2001 From: Charles Strahan Date: Fri, 20 Jun 2014 23:05:48 -0400 Subject: [PATCH 1/3] add nix-prefetch-zip --- .../nix-prefetch-scripts/default.nix | 12 +- .../nix-prefetch-scripts/nix-prefetch-zip | 147 ++++++++++++++++++ 2 files changed, 155 insertions(+), 4 deletions(-) create mode 100755 pkgs/tools/package-management/nix-prefetch-scripts/nix-prefetch-zip diff --git a/pkgs/tools/package-management/nix-prefetch-scripts/default.nix b/pkgs/tools/package-management/nix-prefetch-scripts/default.nix index d7210b2f616b..8f5e7c4a544c 100644 --- a/pkgs/tools/package-management/nix-prefetch-scripts/default.nix +++ b/pkgs/tools/package-management/nix-prefetch-scripts/default.nix @@ -1,4 +1,4 @@ -{ stdenv, makeWrapper, git, subversion, mercurial, bazaar, cvs }: +{ stdenv, makeWrapper, git, subversion, mercurial, bazaar, cvs, unzip, curl }: stdenv.mkDerivation { name = "nix-prefetch-scripts"; @@ -11,9 +11,12 @@ stdenv.mkDerivation { function copyScript { local name=nix-prefetch-$1; local src=$2; - local exe=$3/bin; cp $src $out/bin/$name; - wrapProgram $out/bin/$name --suffix PATH : "$exe" + for dep in ''${@:3}; do + local exe=$dep/bin; + local wrapArgs="$wrapArgs --suffix PATH : $exe" + done + wrapProgram $out/bin/$name $wrapArgs } copyScript "hg" ${../../../build-support/fetchhg/nix-prefetch-hg} ${mercurial} @@ -21,6 +24,7 @@ stdenv.mkDerivation { copyScript "svn" ${../../../build-support/fetchsvn/nix-prefetch-svn} ${subversion} copyScript "bzr" ${../../../build-support/fetchbzr/nix-prefetch-bzr} ${bazaar} copyScript "cvs" ${../../../build-support/fetchcvs/nix-prefetch-cvs} ${cvs} + copyScript "zip" ${./nix-prefetch-zip} ${unzip} ${curl} ''; meta = with stdenv.lib; { @@ -30,4 +34,4 @@ stdenv.mkDerivation { # Quicker to build than to download, I hope hydraPlatforms = []; }; -} \ No newline at end of file +} diff --git a/pkgs/tools/package-management/nix-prefetch-scripts/nix-prefetch-zip b/pkgs/tools/package-management/nix-prefetch-scripts/nix-prefetch-zip new file mode 100755 index 000000000000..b1c71bbff332 --- /dev/null +++ b/pkgs/tools/package-management/nix-prefetch-scripts/nix-prefetch-zip @@ -0,0 +1,147 @@ +#! /bin/sh -e + +usage(){ + echo >&2 "syntax: nix-prefetch-zip [options] + +Options: + --url name The url of the archive to fetch. + --name name The name to use for the store path (defaults to \`basename \$url\`). + --hash name The hash of unpacked archive. + --hash-type hash Use the specified cryptographic hash algorithm, which can be one of md5, sha1, and sha256. + --base32 Print/accept the hash in a base-32 representation rather than hexadecimal. + --leave-root Keep the root directory of the archive. + --help Show this help text. +" + exit 1 +} + + +argi=0 +argfun="" +for arg; do + if test -z "$argfun"; then + case $arg in + --url) argfun=set_url;; + --name) argfun=set_name;; + --hash) argfun=set_expHash;; + --hash-type) argfun=set_hashType;; + --base32) hashFormat="--base32";; + --leave-root) leaveRoot=true;; + --help) usage;; + *) + echo "Unexpected argument: $arg" >&2 + usage + ;; + esac + else + case $argfun in + set_*) + var=$(echo $argfun | sed 's,^set_,,') + eval "$var=\$arg" + ;; + esac + argfun="" + fi +done + +if [ -z "$url" ]; then + echo "Error: No --url flag given" >&2 + usage +fi + +if [ -z "$name" ]; then + name=$(basename "$url") +fi + +if test -z "$hashType"; then + hashType=sha256 +fi + +tmp=$(mktemp -d 2>/dev/null || mktemp -d -t "$$") +trap "rm -rf \"\$tmp\"" EXIT + +TMPDIR=$tmp/unpacked/$name +mkdir -p $TMPDIR +downloadedFile=$tmp/$name + +unpackFile() { + local curSrc="$1" + + case "$curSrc" in + *.tar.xz | *.tar.lzma) + # Don't rely on tar knowing about .xz. + xz -d < $curSrc | tar xf - + ;; + *.tar | *.tar.* | *.tgz | *.tbz2) + # GNU tar can automatically select the decompression method + # (info "(tar) gzip"). + tar xf $curSrc + ;; + *.zip) + unzip -qq $curSrc + ;; + *) + echo "source archive $curSrc has unknown type" >&2 + exit 1 + ;; + esac +} + +# If the hash was given, a file with that hash may already be in the +# store. +if test -n "$expHash"; then + finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$name") + if ! nix-store --check-validity "$finalPath" 2> /dev/null; then + finalPath= + fi + hash=$expHash +fi + +# If we don't know the hash or a path with that hash doesn't exist, +# download the file and add it to the store. +if test -z "$finalPath"; then + curl="curl \ + --location --max-redirs 20 \ + --disable-epsv \ + --insecure" + + if ! $curl --fail "$url" --output "$downloadedFile"; then + echo "error: could not download $url" >&2 + exit 1 + fi + + cd $TMPDIR + unpackFile "$downloadedFile" + + # FIXME: handle zip files that contain a single regular file. + if [ -z "$leaveRoot" ]; then + shopt -s dotglob + if [ $(ls -d $TMPDIR/* | wc -l) != 1 ]; then + echo "error: zip file must contain a single directory." + exit 1 + fi + fn=$(cd "$TMPDIR" && echo *) + mv $TMPDIR/$fn/* "$TMPDIR/" + rmdir "$TMPDIR/$fn" + fi + + # Compute the hash. + hash=$(nix-hash --type $hashType $hashFormat $TMPDIR) + if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi + + # Add the downloaded file to the Nix store. + finalPath=$(nix-store --add-fixed --recursive "$hashType" $TMPDIR) + + if test -n "$expHash" -a "$expHash" != "$hash"; then + echo "hash mismatch for URL \`$url'" + exit 1 + fi +fi + +if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi + +echo $hash + +if test -n "$PRINT_PATH"; then + echo $finalPath +fi From 400b94e120462f9d6b503e38baa3e4d52917986d Mon Sep 17 00:00:00 2001 From: Charles Strahan Date: Sun, 22 Jun 2014 05:05:50 -0400 Subject: [PATCH 2/3] fix bzr prefetcher on darwin --- .../package-management/nix-prefetch-scripts/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/tools/package-management/nix-prefetch-scripts/default.nix b/pkgs/tools/package-management/nix-prefetch-scripts/default.nix index 8f5e7c4a544c..bb0d717a5a2e 100644 --- a/pkgs/tools/package-management/nix-prefetch-scripts/default.nix +++ b/pkgs/tools/package-management/nix-prefetch-scripts/default.nix @@ -1,4 +1,4 @@ -{ stdenv, makeWrapper, git, subversion, mercurial, bazaar, cvs, unzip, curl }: +{ stdenv, makeWrapper, git, subversion, mercurial, bazaar, cvs, unzip, curl, gnused }: stdenv.mkDerivation { name = "nix-prefetch-scripts"; @@ -14,7 +14,7 @@ stdenv.mkDerivation { cp $src $out/bin/$name; for dep in ''${@:3}; do local exe=$dep/bin; - local wrapArgs="$wrapArgs --suffix PATH : $exe" + local wrapArgs="$wrapArgs --prefix PATH : $exe" done wrapProgram $out/bin/$name $wrapArgs } @@ -22,7 +22,7 @@ stdenv.mkDerivation { copyScript "hg" ${../../../build-support/fetchhg/nix-prefetch-hg} ${mercurial} copyScript "git" ${../../../build-support/fetchgit/nix-prefetch-git} ${git} copyScript "svn" ${../../../build-support/fetchsvn/nix-prefetch-svn} ${subversion} - copyScript "bzr" ${../../../build-support/fetchbzr/nix-prefetch-bzr} ${bazaar} + copyScript "bzr" ${../../../build-support/fetchbzr/nix-prefetch-bzr} ${bazaar} ${gnused} copyScript "cvs" ${../../../build-support/fetchcvs/nix-prefetch-cvs} ${cvs} copyScript "zip" ${./nix-prefetch-zip} ${unzip} ${curl} ''; From 9bd8dffd77c8bdd0f73df14d56707250e7240857 Mon Sep 17 00:00:00 2001 From: Charles Strahan Date: Sun, 22 Jun 2014 05:42:34 -0400 Subject: [PATCH 3/3] require gnused for all prefetch scripts --- .../fetchzip}/nix-prefetch-zip | 48 +++++++++++-------- .../nix-prefetch-scripts/default.nix | 9 ++-- 2 files changed, 32 insertions(+), 25 deletions(-) rename pkgs/{tools/package-management/nix-prefetch-scripts => build-support/fetchzip}/nix-prefetch-zip (75%) diff --git a/pkgs/tools/package-management/nix-prefetch-scripts/nix-prefetch-zip b/pkgs/build-support/fetchzip/nix-prefetch-zip similarity index 75% rename from pkgs/tools/package-management/nix-prefetch-scripts/nix-prefetch-zip rename to pkgs/build-support/fetchzip/nix-prefetch-zip index b1c71bbff332..76255ab36747 100755 --- a/pkgs/tools/package-management/nix-prefetch-scripts/nix-prefetch-zip +++ b/pkgs/build-support/fetchzip/nix-prefetch-zip @@ -1,14 +1,13 @@ #! /bin/sh -e usage(){ - echo >&2 "syntax: nix-prefetch-zip [options] + echo >&2 "syntax: nix-prefetch-zip [OPTIONS] [URL [EXPECTED-HASH]] Options: - --url name The url of the archive to fetch. + --url url The url of the archive to fetch. --name name The name to use for the store path (defaults to \`basename \$url\`). - --hash name The hash of unpacked archive. - --hash-type hash Use the specified cryptographic hash algorithm, which can be one of md5, sha1, and sha256. - --base32 Print/accept the hash in a base-32 representation rather than hexadecimal. + --hash hash The hash of unpacked archive. + --hash-type type Use the specified cryptographic hash algorithm, which can be one of md5, sha1, and sha256. --leave-root Keep the root directory of the archive. --help Show this help text. " @@ -25,13 +24,18 @@ for arg; do --name) argfun=set_name;; --hash) argfun=set_expHash;; --hash-type) argfun=set_hashType;; - --base32) hashFormat="--base32";; --leave-root) leaveRoot=true;; --help) usage;; - *) - echo "Unexpected argument: $arg" >&2 - usage - ;; + *) argi=$(($argi + 1)) + case $argi in + 1) url=$arg;; + 2) rev=$arg;; + 3) expHash=$arg;; + *) echo "Unexpected argument: $arg" >&2 + usage + ;; + esac + ;; esac else case $argfun in @@ -57,11 +61,13 @@ if test -z "$hashType"; then hashType=sha256 fi -tmp=$(mktemp -d 2>/dev/null || mktemp -d -t "$$") -trap "rm -rf \"\$tmp\"" EXIT +hashFormat="--base32" -TMPDIR=$tmp/unpacked/$name -mkdir -p $TMPDIR +tmp=$(mktemp -d 2>/dev/null || mktemp -d -t "$$") +trap "rm -rf '$tmp'" EXIT + +unpackDir=$tmp/unpacked/$name +mkdir -p $unpackDir downloadedFile=$tmp/$name unpackFile() { @@ -110,27 +116,27 @@ if test -z "$finalPath"; then exit 1 fi - cd $TMPDIR + cd $unpackDir unpackFile "$downloadedFile" # FIXME: handle zip files that contain a single regular file. if [ -z "$leaveRoot" ]; then shopt -s dotglob - if [ $(ls -d $TMPDIR/* | wc -l) != 1 ]; then + if [ $(ls -d $unpackDir/* | wc -l) != 1 ]; then echo "error: zip file must contain a single directory." exit 1 fi - fn=$(cd "$TMPDIR" && echo *) - mv $TMPDIR/$fn/* "$TMPDIR/" - rmdir "$TMPDIR/$fn" + fn=$(cd "$unpackDir" && echo *) + mv $unpackDir/$fn/* "$unpackDir/" + rmdir "$unpackDir/$fn" fi # Compute the hash. - hash=$(nix-hash --type $hashType $hashFormat $TMPDIR) + hash=$(nix-hash --type $hashType $hashFormat $unpackDir) if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi # Add the downloaded file to the Nix store. - finalPath=$(nix-store --add-fixed --recursive "$hashType" $TMPDIR) + finalPath=$(nix-store --add-fixed --recursive "$hashType" $unpackDir) if test -n "$expHash" -a "$expHash" != "$hash"; then echo "hash mismatch for URL \`$url'" diff --git a/pkgs/tools/package-management/nix-prefetch-scripts/default.nix b/pkgs/tools/package-management/nix-prefetch-scripts/default.nix index bb0d717a5a2e..84439136d49e 100644 --- a/pkgs/tools/package-management/nix-prefetch-scripts/default.nix +++ b/pkgs/tools/package-management/nix-prefetch-scripts/default.nix @@ -11,20 +11,21 @@ stdenv.mkDerivation { function copyScript { local name=nix-prefetch-$1; local src=$2; + local wrapArgs="" cp $src $out/bin/$name; for dep in ''${@:3}; do - local exe=$dep/bin; - local wrapArgs="$wrapArgs --prefix PATH : $exe" + wrapArgs="$wrapArgs --prefix PATH : $dep/bin" done + wrapArgs="$wrapArgs --prefix PATH : ${gnused}/bin" wrapProgram $out/bin/$name $wrapArgs } copyScript "hg" ${../../../build-support/fetchhg/nix-prefetch-hg} ${mercurial} copyScript "git" ${../../../build-support/fetchgit/nix-prefetch-git} ${git} copyScript "svn" ${../../../build-support/fetchsvn/nix-prefetch-svn} ${subversion} - copyScript "bzr" ${../../../build-support/fetchbzr/nix-prefetch-bzr} ${bazaar} ${gnused} + copyScript "bzr" ${../../../build-support/fetchbzr/nix-prefetch-bzr} ${bazaar} copyScript "cvs" ${../../../build-support/fetchcvs/nix-prefetch-cvs} ${cvs} - copyScript "zip" ${./nix-prefetch-zip} ${unzip} ${curl} + copyScript "zip" ${../../../build-support/fetchzip/nix-prefetch-zip} ${unzip} ${curl} ''; meta = with stdenv.lib; {