From 903129f770307954936c85fb5f8b0645540922b7 Mon Sep 17 00:00:00 2001 From: Arthur Noel Date: Wed, 27 Jan 2016 19:48:04 +0000 Subject: [PATCH] dockerTools: private registry support * authorization token is optional * registry url is taken from X-Docker-Endpoints header * pull.sh correctly resumes partial layer downloads * detjson.py does not fail on missing keys --- doc/functions.xml | 5 ++- pkgs/build-support/docker/detjson.py | 4 ++- pkgs/build-support/docker/pull.nix | 9 ++--- pkgs/build-support/docker/pull.sh | 53 +++++++++++++++++----------- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/doc/functions.xml b/doc/functions.xml index 5a350a23e0ad..7d250824f785 100644 --- a/doc/functions.xml +++ b/doc/functions.xml @@ -489,7 +489,6 @@ c = lib.makeOverridable f { a = 1; b = 2; } sha256 = "1bhw5hkz6chrnrih0ymjbmn69hyfriza2lr550xyvpdrnbzr4gk2"; indexUrl = "https://index.docker.io"; - registryUrl = "https://registry-1.docker.io"; registryVersion = "v1"; } @@ -534,8 +533,8 @@ c = lib.makeOverridable f { a = 1; b = 2; } - In the above example the default values are shown for the variables indexUrl, - registryUrl and registryVersion. + In the above example the default values are shown for the variables + indexUrl and registryVersion. Hence by default the Docker.io registry is used to pull the images. diff --git a/pkgs/build-support/docker/detjson.py b/pkgs/build-support/docker/detjson.py index ba2c20a475a9..439c21313878 100644 --- a/pkgs/build-support/docker/detjson.py +++ b/pkgs/build-support/docker/detjson.py @@ -24,9 +24,11 @@ SAFEDELS["container_config"] = SAFEDELS["config"] def makedet(j, safedels): for k,v in safedels.items(): + if k not in j: + continue if type(v) == dict: makedet(j[k], v) - elif k in j and j[k] == v: + elif j[k] == v: del j[k] def main(): diff --git a/pkgs/build-support/docker/pull.nix b/pkgs/build-support/docker/pull.nix index 7115a83df426..a5e7acaf159f 100644 --- a/pkgs/build-support/docker/pull.nix +++ b/pkgs/build-support/docker/pull.nix @@ -8,13 +8,14 @@ { imageName, imageTag ? "latest", imageId ? null , sha256, name ? "${imageName}-${imageTag}" , indexUrl ? "https://index.docker.io" -, registryUrl ? "https://registry-1.docker.io" , registryVersion ? "v1" , curlOpts ? "" }: +assert registryVersion == "v1"; + let layer = stdenv.mkDerivation { inherit name imageName imageTag imageId - indexUrl registryUrl registryVersion curlOpts; + indexUrl registryVersion curlOpts; builder = ./pull.sh; detjson = ./detjson.py; @@ -34,10 +35,6 @@ let layer = stdenv.mkDerivation { # This variable allows the user to pass additional options to curl "NIX_CURL_FLAGS" - - # This variable allows overriding the timeout for connecting to - # the hashed mirrors. - "NIX_CONNECT_TIMEOUT" ]; # Doing the download on a remote machine just duplicates network diff --git a/pkgs/build-support/docker/pull.sh b/pkgs/build-support/docker/pull.sh index 8a0782780afc..7ba146e9de09 100644 --- a/pkgs/build-support/docker/pull.sh +++ b/pkgs/build-support/docker/pull.sh @@ -6,17 +6,20 @@ source $stdenv/setup # servers to need them during redirects, and work on SSL without a # certificate (this isn't a security problem because we check the # cryptographic hash of the output anyway). -curl="curl \ - --location --max-redirs 20 \ - --retry 3 \ - --fail \ - --disable-epsv \ - --cookie-jar cookies \ - --insecure \ - $curlOpts \ - $NIX_CURL_FLAGS" - -baseUrl="$registryUrl/$registryVersion" +curl=$(command -v curl) +curl() { + [[ -n ${token:-} ]] && set -- -H "Authorization: Token $token" "$@" + $curl \ + --location --max-redirs 20 \ + --retry 3 \ + --fail \ + --disable-epsv \ + --cookie-jar cookies \ + --insecure \ + $curlOpts \ + $NIX_CURL_FLAGS \ + "$@" +} fetchLayer() { local url="$1" @@ -26,7 +29,7 @@ fetchLayer() { # if we get error code 18, resume partial download while [ $curlexit -eq 18 ]; do # keep this inside an if statement, since on failure it doesn't abort the script - if $curl -H "Authorization: Token $token" "$url" --output "$dest"; then + if curl -C - "$url" --output "$dest"; then return 0 else curlexit=$?; @@ -36,17 +39,25 @@ fetchLayer() { return $curlexit } -token="$($curl -o /dev/null -D- -H 'X-Docker-Token: true' "$indexUrl/$registryVersion/repositories/$imageName/images" | grep X-Docker-Token | tr -d '\r' | cut -d ' ' -f 2)" +headers=$(curl -o /dev/null -D- -H 'X-Docker-Token: true' \ + "$indexUrl/$registryVersion/repositories/$imageName/images") -if [ -z "$token" ]; then - echo "error: registry returned no token" - exit 1 +header() { + grep $1 <<< "$headers" | tr -d '\r' | cut -d ' ' -f 2 +} + +# this only takes the first endpoint, more may be provided +# https://docs.docker.com/v1.6/reference/api/docker-io_api/ +if ! registryUrl=$(header X-Docker-Endpoints); then + echo "error: index returned no endpoint" + exit 1 fi +baseUrl="https://$registryUrl/$registryVersion" -# token="${token//\"/\\\"}" +token="$(header X-Docker-Token || true)"; if [ -z "$imageId" ]; then - imageId="$($curl -H "Authorization: Token $token" "$baseUrl/repositories/$imageName/tags/$imageTag")" + imageId="$(curl "$baseUrl/repositories/$imageName/tags/$imageTag")" imageId="${imageId//\"/}" if [ -z "$imageId" ]; then echo "error: no image ID found for ${imageName}:${imageTag}" @@ -62,7 +73,7 @@ jshon -n object \ -n object -s "$imageId" -i "$imageTag" \ -i "$imageName" > $out/repositories -$curl -H "Authorization: Token $token" "$baseUrl/images/$imageId/ancestry" -o ancestry.json +curl "$baseUrl/images/$imageId/ancestry" -o ancestry.json layerIds=$(jshon -a -u < ancestry.json) for layerId in $layerIds; do @@ -70,6 +81,6 @@ for layerId in $layerIds; do mkdir "$out/$layerId" echo '1.0' > "$out/$layerId/VERSION" - $curl -H "Authorization: Token $token" "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json" + curl "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json" fetchLayer "$baseUrl/images/$layerId/layer" "$out/$layerId/layer.tar" -done \ No newline at end of file +done