alioth/nixpkgs
3
0
Fork 0
forked from mirrors/nixpkgs
Code Releases Activity

Merge pull request #31333 from ElvishJerricco/unextracted-all-cabal-hashes

Browse source 
Unextracted all cabal hashes
This commit is contained in:
Peter Simons 2017-11-08 16:25:16 +01:00 committed by GitHub
parent bb0b0822ef a38a3e2129
commit c7203efb7d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 13 additions and 238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
pkgs/data/misc/hackage/default.nix
View file

@ -1,34 +1,6 @@
{ fetchFromGitHub
, stdenv
, pkgconfig
, libarchive
, glib
, # Override this to use a different revision
src-spec ?
{ owner = "commercialhaskell";
repo = "all-cabal-hashes";
rev = "5e87c40f2cd96bd5dd953758e82f302107c7895e";
sha256 = "12rw5fld64s0a2zjsdijfs0dv6vc6z7gcf24h4m2dmymzms4namg";
}
, lib
}:
{ fetchurl }:
# Use builtins.fetchTarball "https://github.com/commercialhaskell/all-cabal-hashes/archive/hackage.tar.gz"
# instead if you want the latest Hackage automatically at the price of frequent re-downloads.
let partition-all-cabal-hashes = stdenv.mkDerivation
{ name = "partition-all-cabal-hashes";
src = ./partition-all-cabal-hashes.c;
unpackPhase = "true";
buildInputs = [ libarchive glib ];
nativeBuildInputs = [ pkgconfig ];
buildPhase =
"cc -O3 $(pkg-config --cflags --libs libarchive glib-2.0) $src -o partition-all-cabal-hashes";
installPhase =
''
mkdir -p $out/bin
install -m755 partition-all-cabal-hashes $out/bin
'';
};
in fetchFromGitHub (src-spec //
{ postFetch = "${partition-all-cabal-hashes}/bin/partition-all-cabal-hashes $downloadedFile $out";
})
fetchurl {
url = "https://github.com/commercialhaskell/all-cabal-hashes/archive/5e87c40f2cd96bd5dd953758e82f302107c7895e.tar.gz";
sha256 = "0hjkddda9mdm21nb9bkhr9n5r9jllisif1qmzha91a9cps5w1mx5";
}

190
pkgs/data/misc/hackage/partition-all-cabal-hashes.c
View file

@ -1,190 +0,0 @@
#include <stdio.h>
#include <ctype.h>
#include <glib.h>
#include <string.h>
#include <locale.h>
#include <archive.h>
#include <archive_entry.h>
static char * case_normalize(char * str) {
for (char * iter = str; *iter; ++iter) {
*iter = tolower(*iter);
}
return str;
}
static gint compare_str(const void * a, const void * b, void * _) {
return strcmp(a, b);
}
int main(int argc, char ** argv) {
if (argc != 3) {
fprintf(stderr, "Usage: %s TARBALL OUTPUT\n", argv[0]);
return 1;
}
size_t output_len = strlen(argv[2]);
/* Switch to standard locale to ensure consistency in case-folding.
*/
setlocale(LC_CTYPE, "C");
/* Map from case-normalized package name to a sorted sequence of
* package names in the equivalence class defined by
* case-normalization.
*/
GHashTable * equivalence_classes =
g_hash_table_new(g_str_hash, g_str_equal);
/* Open up the tarball.
*/
struct archive * ar = archive_read_new();
if (!ar) {
perror("Allocating archive structure");
return 1;
}
archive_read_support_filter_gzip(ar);
archive_read_support_format_tar(ar);
if (archive_read_open_filename( ar
, argv[1]
, 10240
) == ARCHIVE_FATAL) {
fprintf( stderr
, "Error opening %s: %s\n"
, argv[0]
, archive_error_string(ar)
);
return 1;
}
/* Extract the length of the output directory that prefixes all
* tarball entries from the first entry in the tarball.
*/
struct archive_entry * ent;
int err = archive_read_next_header(ar, &ent);
if (err != ARCHIVE_OK) {
if (err == ARCHIVE_EOF) {
fprintf( stderr
, "No entries in %s, surely this is an error!\n"
, argv[1]
);
} else {
fprintf( stderr
, "Error reading entry from %s: %s\n"
, argv[1]
, archive_error_string(ar)
);
}
return 1;
}
const char * path = archive_entry_pathname(ent);
/* Number of characters from the start of the path name until after
* the slash after the leading directory.
*/
size_t prefix_len = strchr(path, '/') - path + 1;
/* Extract each entry to the right partition.
*/
do {
path = archive_entry_pathname(ent) + prefix_len;
const char * pkg_end = strchr(path, '/');
if (!pkg_end)
/* If there is no second slash, then this is either just the entry
* corresponding to the root or some non-package file (e.g.
* travis.yml). In either case, we don't care.
*/
continue;
/* Find our package in the equivalence class map.
*/
char * pkg_name = g_strndup(path, pkg_end - path);
char * pkg_normalized =
case_normalize(g_strndup(path, pkg_end - path));
GSequence * pkg_class =
g_hash_table_lookup(equivalence_classes, pkg_normalized);
gint partition_num;
if (!pkg_class) {
/* We haven't seen any packages with this normalized name yet,
* so we need to initialize the sequence and add it to the map.
*/
pkg_class = g_sequence_new(NULL);
g_sequence_append(pkg_class, pkg_name);
g_hash_table_insert( equivalence_classes
, pkg_normalized
, pkg_class
);
partition_num = 1;
} else {
g_free(pkg_normalized);
/* Find the package name in the equivalence class */
GSequenceIter * pkg_iter =
g_sequence_search( pkg_class
, pkg_name
, compare_str
, NULL
);
if (!g_sequence_iter_is_end(pkg_iter)) {
/* If there are any packages after this one in the list, bail
* out. In principle we could solve this by moving them up to
* the next partition, but so far I've never seen any github
* tarballs out of order so let's save ourselves the work
* until we know we need it.
*/
fprintf( stderr
, "Out of order github tarball: %s is after %s\n"
, pkg_name
, (char *) g_sequence_get(pkg_iter)
);
return 1;
}
pkg_iter = g_sequence_iter_prev(pkg_iter);
if (strcmp( g_sequence_get(pkg_iter)
, pkg_name
) != 0) {
/* This package doesn't have the same name as the one right
* before where it should be in the sequence, which means it's
* new and needs to be added to the sequence.
*
* !!! We need to change this to use g_sequence_insert_before
* if we ever get an out-of-order github tarball, see comment
* after the check for !g_sequence_iter_is_end(pkg_iter).
*/
pkg_iter = g_sequence_append(pkg_class, pkg_name);
} else {
g_free(pkg_name);
}
/* Get the partition number, starting with 1.
*/
partition_num = g_sequence_iter_get_position(pkg_iter) + 1;
}
/* Set the destination path.
* The 3 below is for the length of /#/, the partition number part
* of the path. If we have more than 9 partitions, we deserve to
* segfault. The 1 at the end is for the trailing null.
*/
char * dest_path = g_malloc(output_len + 3 + strlen(path) + 1);
sprintf(dest_path, "%s/%d/%s", argv[2], partition_num, path);
archive_entry_set_pathname(ent, dest_path);
if (archive_read_extract(ar, ent, 0) != ARCHIVE_OK) {
fprintf( stderr
, "Error extracting entry %s from %s: %s\n"
, dest_path
, argv[1]
, archive_error_string(ar)
);
return 1;
}
} while ((err = archive_read_next_header(ar, &ent)) == ARCHIVE_OK);
if (err != ARCHIVE_EOF) {
fprintf( stderr
, "Error reading entry from %s: %s\n"
, argv[1]
, archive_error_string(ar)
);
return 1;
}
return 0;
}

23
pkgs/development/haskell-modules/make-package-set.nix
View file

@ -118,23 +118,16 @@ let
'';
};
all-cabal-hashes-component = name: import (pkgs.runCommand "all-cabal-hashes-component-${name}.nix" {}
''
set +o pipefail
for component in ${all-cabal-hashes}/*; do
if ls $component | grep -q "^${name}$"; then
echo "builtins.storePath $component" > $out
exit 0
fi
done
echo "${name} not found in any all-cabal-hashes component, are you sure it's in hackage?" >&2
exit 1
'');
all-cabal-hashes-component = name: version: pkgs.runCommand "all-cabal-hashes-component-${name}-${version}" {} ''
tar --wildcards -xzvf ${all-cabal-hashes} \*/${name}/${version}/${name}.{json,cabal}
mkdir -p $out
mv */${name}/${version}/${name}.{json,cabal} $out
'';
hackage2nix = name: version: let component = all-cabal-hashes-component name; in self.haskellSrc2nix {
hackage2nix = name: version: let component = all-cabal-hashes-component name version; in self.haskellSrc2nix {
name = "${name}-${version}";
sha256 = ''$(sed -e 's/.*"SHA256":"//' -e 's/".*$//' "${component}/${name}/${version}/${name}.json")'';
src = "${component}/${name}/${version}/${name}.cabal";
sha256 = ''$(sed -e 's/.*"SHA256":"//' -e 's/".*$//' "${component}/${name}.json")'';
src = "${component}/${name}.cabal";
};
in package-set { inherit pkgs stdenv callPackage; } self // {