From 1393d2262bfb01ba47c7f78a23322de8de17e7d7 Mon Sep 17 00:00:00 2001 From: Alexey Vakhrenev Date: Fri, 17 Feb 2023 10:13:05 +0400 Subject: [PATCH] redpanda: 22.3.11 -> 22.3.13 Build rpk from source. Add redpanda server. --- maintainers/maintainer-list.nix | 6 ++ pkgs/servers/redpanda/base64.nix | 26 +++++ pkgs/servers/redpanda/default.nix | 74 +++++++------ pkgs/servers/redpanda/hdr-histogram.nix | 28 +++++ pkgs/servers/redpanda/rapidjson.nix | 32 ++++++ pkgs/servers/redpanda/redpanda.patch | 73 +++++++++++++ pkgs/servers/redpanda/seastar-fixes.patch | 13 +++ pkgs/servers/redpanda/seastar.nix | 84 +++++++++++++++ pkgs/servers/redpanda/server.nix | 123 ++++++++++++++++++++++ pkgs/top-level/all-packages.nix | 2 + 10 files changed, 426 insertions(+), 35 deletions(-) create mode 100644 pkgs/servers/redpanda/base64.nix create mode 100644 pkgs/servers/redpanda/hdr-histogram.nix create mode 100644 pkgs/servers/redpanda/rapidjson.nix create mode 100644 pkgs/servers/redpanda/redpanda.patch create mode 100644 pkgs/servers/redpanda/seastar-fixes.patch create mode 100644 pkgs/servers/redpanda/seastar.nix create mode 100644 pkgs/servers/redpanda/server.nix diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix index 6e9e3faf60e6..36762361d104 100644 --- a/maintainers/maintainer-list.nix +++ b/maintainers/maintainer-list.nix @@ -1417,6 +1417,12 @@ githubId = 12958979; name = "Mika Naylor"; }; + avakhrenev = { + email = "avakhrenev@gmail.com"; + github = "avakhrenev"; + githubId = 1060224; + name = "Alexey Vakhrenev"; + }; avaq = { email = "nixpkgs@account.avaq.it"; github = "Avaq"; diff --git a/pkgs/servers/redpanda/base64.nix b/pkgs/servers/redpanda/base64.nix new file mode 100644 index 000000000000..305456c55ccf --- /dev/null +++ b/pkgs/servers/redpanda/base64.nix @@ -0,0 +1,26 @@ +{ clangStdenv +, cmake +, fetchFromGitHub +, lib +}: +let + pname = "base64"; + version = "0.5.0"; +in +clangStdenv.mkDerivation { + inherit pname version; + src = fetchFromGitHub { + owner = "aklomp"; + repo = "base64"; + rev = "v${version}"; + sha256 = "sha256-2HNI9ycT9f+NLwLElEuR61qmTguOsI+kNxv01ipxSqQ="; + }; + nativeBuildInputs = [ cmake ]; + meta = with lib; { + description = "Fast Base64 stream encoder/decoder in C99, with SIMD acceleration"; + license = licenses.bsd2; + homepage = "https://github.com/aklomp/base64"; + maintainers = with maintainers; [ avakhrenev ]; + platforms = platforms.unix; + }; +} diff --git a/pkgs/servers/redpanda/default.nix b/pkgs/servers/redpanda/default.nix index e2a265c0f5ef..c59aa621cc4d 100644 --- a/pkgs/servers/redpanda/default.nix +++ b/pkgs/servers/redpanda/default.nix @@ -1,48 +1,52 @@ -{ lib, stdenv, fetchzip }: - +{ buildGoModule +, callPackage +, doCheck ? !stdenv.isDarwin # Can't start localhost test server in MacOS sandbox. +, fetchFromGitHub +, installShellFiles +, lib +, stdenv +}: let - version = "22.3.11"; - platform = if stdenv.isLinux then "linux" else "darwin"; - arch = if stdenv.isAarch64 then "arm" else "amd"; - sha256s = { - darwin.amd = "sha256-kwAKxFg7BSNInvsQvFqgtpq8EEwSnmDeDyaF5b8L8SQ="; - darwin.arm = "sha256-kH5Ii672SeAIiRcWuAO3oVJVSBWp+r78RmTiR3BaDbg="; - linux.amd = "sha256-EKgkRKBrM4+X2YGoP2LpWRHL+fdHu44LYwCZ+O+c5ZY="; - linux.arm = "sha256-9b4oerRXjUVUYoswJWtnMBJSQDoCKClf673VjDQFUAw="; + version = "22.3.13"; + src = fetchFromGitHub { + owner = "redpanda-data"; + repo = "redpanda"; + rev = "v${version}"; + sha256 = "sha256-cUQFDXWnQYSLcfKFYg6BLrxF77iX+Yx3hcul4tMxdoc="; }; -in stdenv.mkDerivation rec { - pname = "redpanda"; - inherit version; + server = callPackage ./server.nix { inherit src version; }; +in +buildGoModule rec { + pname = "redpanda-rpk"; + inherit doCheck src version; + modRoot = "./src/go/rpk"; + runVend = false; + vendorSha256 = "sha256-JVZuHRh3gavIGArxDkqUQsL5oBjz35EKGsC75Sy+cMo="; - src = fetchzip { - url = "https://github.com/redpanda-data/redpanda/releases/download/v${version}/rpk-${platform}-${arch}64.zip"; - sha256 = sha256s.${platform}.${arch}; - }; + ldflags = [ + ''-X "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/version.version=${version}"'' + ''-X "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/version.rev=v${version}"'' + ''-X "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/container/common.tag=v${version}"'' + ]; - installPhase = '' - runHook preInstall + nativeBuildInputs = [ installShellFiles ]; - mkdir -p $out/bin - cp rpk $out/bin - - ${lib.optionalString stdenv.isLinux '' - patchelf \ - --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" \ - $out/bin/rpk - ''} - - runHook postInstall + postInstall = '' + for shell in bash fish zsh; do + $out/bin/rpk generate shell-completion $shell > rpk.$shell + installShellCompletion rpk.$shell + done ''; - # stripping somehow completely breaks it - dontStrip = true; + passthru = { + inherit server; + }; meta = with lib; { - description = "Redpanda is a streaming data platform for developers. Kafka API compatible. 10x faster. No ZooKeeper. No JVM! "; - sourceProvenance = with sourceTypes; [ binaryNativeCode ]; - license = licenses.bsl11; + description = "Redpanda client"; homepage = "https://redpanda.com/"; - maintainers = with maintainers; [ happysalada ]; + license = licenses.bsl11; + maintainers = with maintainers; [ avakhrenev happysalada ]; platforms = platforms.all; }; } diff --git a/pkgs/servers/redpanda/hdr-histogram.nix b/pkgs/servers/redpanda/hdr-histogram.nix new file mode 100644 index 000000000000..91000649e130 --- /dev/null +++ b/pkgs/servers/redpanda/hdr-histogram.nix @@ -0,0 +1,28 @@ +{ clangStdenv +, cmake +, fetchFromGitHub +, lib +, zlib +}: +let + pname = "HdrHistogram_c"; + version = "0.11.5"; +in +clangStdenv.mkDerivation { + inherit pname version; + src = fetchFromGitHub { + owner = "HdrHistogram"; + repo = "HdrHistogram_c"; + rev = version; + sha256 = "sha256-29if+0H8wdpQBN48lt0ylGgtUCv/tJYZnG5LzcIqXDs="; + }; + nativeBuildInputs = [ cmake ]; + buildInputs = [ zlib ]; + meta = with lib; { + description = "C port of the HdrHistogram"; + license = licenses.bsd2; + homepage = "https://github.com/HdrHistogram/HdrHistogram_c"; + maintainers = with maintainers; [ avakhrenev ]; + platforms = platforms.unix; + }; +} diff --git a/pkgs/servers/redpanda/rapidjson.nix b/pkgs/servers/redpanda/rapidjson.nix new file mode 100644 index 000000000000..99ada8c05f88 --- /dev/null +++ b/pkgs/servers/redpanda/rapidjson.nix @@ -0,0 +1,32 @@ +# rapidjson used in nixpkgs is too old. Although it is technically a latest release, it was made in 2016. +# Redpanda uses its own version +{ clangStdenv +, cmake +, fetchFromGitHub +, lib +, pkg-config +}: + +clangStdenv.mkDerivation rec { + pname = "rapidjson"; + version = "1.1.1"; + + src = fetchFromGitHub { + owner = "redpanda-data"; + repo = "rapidjson"; + rev = "27c3a8dc0e2c9218fe94986d249a12b5ed838f1d"; + sha256 = "sha256-wggyCL5uEsnJDxkYAUsXOjoO1MNQBGB05E6aSpsNcl0="; + }; + + nativeBuildInputs = [ pkg-config cmake ]; + + doCheck = false; + + meta = with lib; { + description = "Fast JSON parser/generator for C++ with both SAX/DOM style API"; + homepage = "http://rapidjson.org/"; + maintainers = with maintainers; [ avakhrenev ]; + license = licenses.mit; + platforms = platforms.unix; + }; +} diff --git a/pkgs/servers/redpanda/redpanda.patch b/pkgs/servers/redpanda/redpanda.patch new file mode 100644 index 000000000000..c3ee76d3c9ae --- /dev/null +++ b/pkgs/servers/redpanda/redpanda.patch @@ -0,0 +1,73 @@ +diff --git a/cmake/main.cmake b/cmake/main.cmake +index 8c60c4214..194f33a21 100644 +--- a/cmake/main.cmake ++++ b/cmake/main.cmake +@@ -15,15 +15,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_UNITY_BUILD_BATCH_SIZE 10) + set(CMAKE_CXX_EXTENSIONS OFF) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) +-list(APPEND BASE_LD_FLAGS_LIST +- -L${REDPANDA_DEPS_INSTALL_DIR}/lib +- -L${REDPANDA_DEPS_INSTALL_DIR}/lib64 +- -fuse-ld=lld) +-set(PKG_CONFIG_PATH_LIST +- ${REDPANDA_DEPS_INSTALL_DIR}/lib64/pkgconfig +- ${REDPANDA_DEPS_INSTALL_DIR}/share/pkgconfig +- ${REDPANDA_DEPS_INSTALL_DIR}/lib/pkgconfig +- ) + + list(APPEND BASE_CXX_FLAGS_LIST -fPIC) + list(APPEND BASE_C_FLAGS_LIST -fPIC) +diff --git a/cmake/testing.cmake b/cmake/testing.cmake +index 7f149dc82..7c57aa3dd 100644 +--- a/cmake/testing.cmake ++++ b/cmake/testing.cmake +@@ -24,6 +24,7 @@ message(STATUS "RP_ENABLE_BENCHMARK_TESTS=${RP_ENABLE_BENCHMARK_TESTS}") + message(STATUS "RP_ENABLE_HONEY_BADGER_TESTS=${RP_ENABLE_HONEY_BADGER_TESTS}") + + function (rp_test) ++ return() + set(options + INTEGRATION_TEST UNIT_TEST BENCHMARK_TEST HBADGER_TEST) + set(oneValueArgs BINARY_NAME TIMEOUT PREPARE_COMMAND POST_COMMAND) +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index eecd145ed..b9efa89b5 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -5,4 +5,5 @@ find_package(Boost REQUIRED + unit_test_framework) + find_package(absl REQUIRED) + add_subdirectory(v) +-add_subdirectory(go/kreq-gen) ++# Don't build kafka-request-generator, it is needed only for tests ++# add_subdirectory(go/kreq-gen) +diff --git a/src/v/CMakeLists.txt b/src/v/CMakeLists.txt +index 075da485e..af7ede2bc 100644 +--- a/src/v/CMakeLists.txt ++++ b/src/v/CMakeLists.txt +@@ -58,7 +58,9 @@ else() + if(${ENABLE_GIT_HASH}) + message(FATAL_ERROR "ENABLE_GIT_HASH cannot be 'on' when ENABLE_GIT_VERSION is 'off'") + endif() +- set(GIT_VER "no_version") ++ if(NOT GIT_VER) ++ set(GIT_VER "no_version") ++ endif() + endif() + + if(${ENABLE_GIT_HASH}) +@@ -71,8 +73,12 @@ if(${ENABLE_GIT_HASH}) + set(GIT_CLEAN_DIRTY "") + endif() + else() +- set(GIT_SHA1 "000") +- set(GIT_CLEAN_DIRTY "-dev") ++ if(NOT GIT_SHA1) ++ set(GIT_SHA1 "000") ++ endif() ++ if(NOT GIT_CLEAN_DIRTY) ++ set(GIT_CLEAN_DIRTY "-dev") ++ endif() + endif() + + configure_file(version.h.in version.h @ONLY) diff --git a/pkgs/servers/redpanda/seastar-fixes.patch b/pkgs/servers/redpanda/seastar-fixes.patch new file mode 100644 index 000000000000..c6b12b88c149 --- /dev/null +++ b/pkgs/servers/redpanda/seastar-fixes.patch @@ -0,0 +1,13 @@ +diff --git a/include/seastar/core/std-coroutine.hh b/include/seastar/core/std-coroutine.hh +index ea364bee..57474529 100644 +--- a/include/seastar/core/std-coroutine.hh ++++ b/include/seastar/core/std-coroutine.hh +@@ -87,7 +87,7 @@ class coroutine_handle { + + explicit operator bool() const noexcept { return _pointer; } + +- static coroutine_handle from_address(void* ptr) noexcept { ++ static constexpr coroutine_handle from_address(void* ptr) noexcept { + coroutine_handle hndl; + hndl._pointer = ptr; + return hndl; diff --git a/pkgs/servers/redpanda/seastar.nix b/pkgs/servers/redpanda/seastar.nix new file mode 100644 index 000000000000..3c7120e6c876 --- /dev/null +++ b/pkgs/servers/redpanda/seastar.nix @@ -0,0 +1,84 @@ +{ boost175 +, c-ares +, cmake +, cryptopp +, fetchFromGitHub +, fmt_8 +, gnutls +, hwloc +, lib +, libsystemtap +, libtasn1 +, liburing +, libxfs +, lksctp-tools +, llvmPackages_14 +, lz4 +, ninja +, numactl +, openssl +, pkg-config +, python3 +, ragel +, valgrind +, yaml-cpp +}: +let + pname = "seastar"; + version = "22.11.0"; +in +llvmPackages_14.stdenv.mkDerivation { + inherit pname version; + strictDeps = true; + src = fetchFromGitHub { + owner = "redpanda-data"; + repo = "seastar"; + rev = "30d3a28bde08d2228b4e560c173b89fdd94c3f05"; + sha256 = "sha256-Xzu7AJMkvE++BGEqluod3fwMEIpDnbCczmlEad0/4v4="; + }; + nativeBuildInputs = [ + cmake + ninja + openssl + pkg-config + python3 + ragel + ]; + buildInputs = [ + libsystemtap + libxfs + ]; + propagatedBuildInputs = [ + boost175 + c-ares + gnutls + cryptopp + fmt_8 + hwloc + libtasn1 + liburing + lksctp-tools + lz4 + numactl + valgrind + yaml-cpp + ]; + patches = [ + ./seastar-fixes.patch + ]; + postPatch = '' + patchShebangs ./scripts/seastar-json2code.py + ''; + cmakeFlags = [ + "-DSeastar_EXCLUDE_DEMOS_FROM_ALL=ON" + "-DSeastar_EXCLUDE_TESTS_FROM_ALL=ON" + ]; + doCheck = false; + meta = with lib; { + description = "High performance server-side application framework."; + license = licenses.asl20; + homepage = "https://seastar.io/"; + maintainers = with maintainers; [ avakhrenev ]; + platforms = platforms.unix; + }; +} diff --git a/pkgs/servers/redpanda/server.nix b/pkgs/servers/redpanda/server.nix new file mode 100644 index 000000000000..9cb050eb5f45 --- /dev/null +++ b/pkgs/servers/redpanda/server.nix @@ -0,0 +1,123 @@ +{ abseil-cpp +, avro-cpp +, callPackage +, ccache +, cmake +, crc32c +, croaring +, ctre +, curl +, dpdk +, git +, lib +, llvmPackages_14 +, llvm_14 +, ninja +, p11-kit +, pkg-config +, procps +, protobuf3_21 +, python3 +, snappy +, src +, unzip +, version +, writeShellScriptBin +, xxHash +, zip +, zstd +}: +let + pname = "redpanda"; + pythonPackages = p: with p; [ jinja2 ]; + seastar = callPackage ./seastar.nix { }; + base64 = callPackage ./base64.nix { }; + hdr-histogram = callPackage ./hdr-histogram.nix { }; + kafka-codegen-venv = python3.withPackages (ps: [ + ps.jinja2 + ps.jsonschema + ]); + rapidjson = callPackage ./rapidjson.nix { }; +in +llvmPackages_14.stdenv.mkDerivation rec { + inherit pname version src; + + preConfigure = '' + # setup sccache + export CCACHE_DIR=$TMPDIR/sccache-redpanda + mkdir -p $CCACHE_DIR + ''; + patches = [ + ./redpanda.patch + ]; + postPatch = '' + # Fix 'error: use of undeclared identifier 'roaring'; did you mean 'Roaring + # qualified reference to 'Roaring' is a constructor name rather than a type in this context' + substituteInPlace \ + ./src/v/storage/compacted_offset_list.h \ + ./src/v/storage/compaction_reducers.cc \ + ./src/v/storage/compaction_reducers.h \ + ./src/v/storage/segment_utils.h \ + ./src/v/storage/segment_utils.cc \ + --replace 'roaring::Roaring' 'Roaring' + + patchShebangs ./src/v/rpc/rpc_compiler.py + ''; + + doCheck = false; + + nativeBuildInputs = [ + (python3.withPackages pythonPackages) + (writeShellScriptBin "kafka-codegen-venv" "exec -a $0 ${kafka-codegen-venv}/bin/python3 $@") + ccache + cmake + curl + git + llvm_14 + ninja + pkg-config + procps + seastar + unzip + zip + ]; + + cmakeFlags = [ + "-DREDPANDA_DEPS_SKIP_BUILD=ON" + "-DRP_ENABLE_TESTS=OFF" + "-Wno-dev" + "-DGIT_VER=${version}" + "-DGIT_CLEAN_DIRTY=\"\"" + ]; + + buildInputs = [ + abseil-cpp + avro-cpp + base64 + crc32c + croaring + ctre + dpdk + hdr-histogram + p11-kit + protobuf3_21 + rapidjson + seastar + snappy + xxHash + zstd + ]; + + meta = with lib; { + description = "Kafka-compatible streaming platform."; + license = licenses.bsl11; + longDescription = '' + Redpanda is a Kafka-compatible streaming data platform that is + proven to be 10x faster and 6x lower in total costs. It is also JVM-free, + ZooKeeper-free, Jepsen-tested and source available. + ''; + homepage = "https://redpanda.com/"; + maintainers = with maintainers; [ avakhrenev happysalada ]; + platforms = platforms.linux; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 64d46f1399b3..ed6900f8f3db 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -11397,6 +11397,8 @@ with pkgs; redpanda = callPackage ../servers/redpanda { }; + redpanda-server = redpanda.server; + redsocks = callPackage ../tools/networking/redsocks { }; renpy = callPackage ../development/interpreters/renpy { };