From 50321b735be9e326dd5b2ab4cc69ae1971ac9594 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Thu, 23 Aug 2007 15:22:30 +0000
Subject: [PATCH] * Basic transparent mirror support in fetchurl (NIXPKGS-70).
When fetching a file with hash HASH of type TYPE, we first try to
download <base-url>/<type>/<hash>, where <base-url> is one of a list of
mirrors. For instance, given
src = fetchurl {
url = http://releases.mozilla.org/pub/mozilla.org/firefox/releases/2.0.0.6/source/firefox-2.0.0.6-source.tar.bz2;
sha1 = "eb72f55e4a8bf08e8c6ef227c0ade3d068ba1082";
};
and the mirror list [http://nix.cs.uu.nl/dist/tarballs], we first
try to download
http://nix.cs.uu.nl/dist/tarballs/sha1/eb72f55e4a8bf08e8c6ef227c0ade3d068ba1082
and if that fails, we use the original URL.
The list of mirrors is not yet user-configurable.
* `fetchurl' now also accepts an argument `urls' instead of `url' for
a list of alternative download locations, which fetchurl will try in
sequence.
svn path=/nixpkgs/trunk/; revision=9190
---
pkgs/build-support/fetchurl/builder.sh | 81 ++++++++++++++++++++-----
pkgs/build-support/fetchurl/default.nix | 37 +++++++++--
2 files changed, 100 insertions(+), 18 deletions(-)
diff --git a/pkgs/build-support/fetchurl/builder.sh b/pkgs/build-support/fetchurl/builder.sh
index 19514ac86675..c6ccf708855f 100644
--- a/pkgs/build-support/fetchurl/builder.sh
+++ b/pkgs/build-support/fetchurl/builder.sh
@@ -1,20 +1,73 @@
source $stdenv/setup
-header "downloading $out from $url"
+header "downloading file $name with $outputHashAlgo hash $outputHash..."
-curl --fail --location --max-redirs 20 --disable-epsv \
- --cookie-jar cookies "$url" > "$out"
+# Curl flags to handle redirects, not use EPSV, handle cookies for
+# servers to need them during redirects, and work on SSL without a
+# certificate (this isn't a security problem because we check the
+# cryptographic hash of the output anyway).
+curl="curl \
+ --location --max-redirs 20 \
+ --disable-epsv \
+ --cookie-jar cookies \
+ --insecure"
-if test "$NIX_OUTPUT_CHECKED" != "1"; then
- if test "$outputHashAlgo" != "md5"; then
- echo "hashes other than md5 are unsupported in Nix <= 0.7, upgrade to Nix 0.8"
- exit 1
+
+tryDownload() {
+ local url="$1"
+ echo
+ header "trying $url"
+ success=
+ if $curl --fail "$url" --output "$out"; then
+ success=1
fi
- actual=$(md5sum -b "$out" | cut -c1-32)
- if test "$actual" != "$id"; then
- echo "hash is $actual, expected $id"
- exit 1
- fi
-fi
+ stopNest
+}
-stopNest
+
+finish() {
+ # On old versions of Nix, verify the hash of the output. On newer
+ # versions, Nix verifies the hash itself.
+ if test "$NIX_OUTPUT_CHECKED" != "1"; then
+ if test "$outputHashAlgo" != "md5"; then
+ echo "hashes other than md5 are unsupported in Nix <= 0.7, upgrade to Nix 0.8"
+ exit 1
+ fi
+ actual=$(md5sum -b "$out" | cut -c1-32)
+ if test "$actual" != "$id"; then
+ echo "hash is $actual, expected $id"
+ exit 1
+ fi
+ fi
+
+ stopNest
+ exit 0
+}
+
+
+for mirror in $hashedMirrors; do
+ url="$mirror/$outputHashAlgo/$outputHash"
+ if $curl --fail --silent --show-error --head "$url" \
+ --write-out "%{http_code}" --output /dev/null > code 2> log; then
+ tryDownload "$url"
+ if test -n "$success"; then finish; fi
+ else
+ # Be quiet about 404 errors, which we interpret as the file
+ # not being present on this particular mirror.
+ if test "$(cat code)" != 404; then
+ echo "error checking the existence of $url:"
+ cat log
+ fi
+ fi
+done
+
+
+success=
+for url in $urls; do
+ tryDownload "$url"
+ if test -n "$success"; then finish; fi
+done
+
+
+echo "error: cannot download $name from any mirror"
+exit 1
diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix
index b6eec09c13bf..c79f3d214c64 100644
--- a/pkgs/build-support/fetchurl/default.nix
+++ b/pkgs/build-support/fetchurl/default.nix
@@ -3,16 +3,47 @@
{stdenv, curl}: # Note that `curl' may be `null', in case of the native stdenv.
-{name ? "", url, outputHash ? "", outputHashAlgo ? "", md5 ? "", sha1 ? "", sha256 ? ""}:
+{ # URL to fetch.
+ url ? ""
+
+, # Alternatively, a list of URLs specifying alternative download
+ # locations. They are tried in order.
+ urls ? []
+
+, # Name of the file. If empty, use the basename of `url' (or of the
+ # first element of `urls').
+ name ? ""
+
+ # Different ways of specifying the hash.
+, outputHash ? ""
+, outputHashAlgo ? ""
+, md5 ? ""
+, sha1 ? ""
+, sha256 ? ""
+}:
+
+assert urls != [] -> url == "";
+assert url != "" -> urls == [];
assert (outputHash != "" && outputHashAlgo != "")
|| md5 != "" || sha1 != "" || sha256 != "";
+let urls_ = if urls != [] then urls else [url]; in
+
stdenv.mkDerivation {
- name = if name != "" then name else baseNameOf (toString url);
+ name =
+ if name != "" then name
+ else baseNameOf (toString (builtins.head urls_));
builder = ./builder.sh;
buildInputs = [curl];
+ urls = urls_;
+
+ # The content-addressable mirrors.
+ hashedMirrors = [
+ http://nix.cs.uu.nl/dist/tarballs
+ ];
+
# Compatibility with Nix <= 0.7.
id = md5;
@@ -22,8 +53,6 @@ stdenv.mkDerivation {
outputHash = if outputHash != "" then outputHash else
if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5;
- inherit url;
-
# We borrow these environment variables from the caller to allow
# easy proxy configuration. This is impure, but a fixed-output
# derivation like fetchurl is allowed to do so since its result is