From 00b0824635f1ab030234124d79af4fbea1d0053c Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Sun, 3 Sep 2023 21:39:04 +0200 Subject: [PATCH 1/4] maintainers/haskell/hydra-report.hs: allow disabling log requesting Since every failure in the jobset means one request to get the log when generating the list of newly broken packages, we need to add an option to disable log requesting in case a lot of new breakage needs to be entered. --- maintainers/scripts/haskell/hydra-report.hs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/maintainers/scripts/haskell/hydra-report.hs b/maintainers/scripts/haskell/hydra-report.hs index 5f7e40a28bcd..3dbd66f2e4c2 100755 --- a/maintainers/scripts/haskell/hydra-report.hs +++ b/maintainers/scripts/haskell/hydra-report.hs @@ -151,15 +151,18 @@ data Build = Build } deriving (Generic, ToJSON, FromJSON, Show) +data RequestLogsFlag = RequestLogs | NoRequestLogs + main :: IO () main = do args <- getArgs case args of ["get-report"] -> getBuildReports ["ping-maintainers"] -> printMaintainerPing - ["mark-broken-list"] -> printMarkBrokenList + ["mark-broken-list", "--no-request-logs"] -> printMarkBrokenList NoRequestLogs + ["mark-broken-list"] -> printMarkBrokenList RequestLogs ["eval-info"] -> printEvalInfo - _ -> putStrLn "Usage: get-report | ping-maintainers | mark-broken-list | eval-info" + _ -> putStrLn "Usage: get-report | ping-maintainers | mark-broken-list [--no-request-logs] | eval-info" reportFileName :: IO FilePath reportFileName = getXdgDirectory XdgCache "haskell-updates-build-report.json" @@ -775,16 +778,20 @@ printMaintainerPing = do textBuildSummary = printBuildSummary eval fetchTime buildSum topBrokenRdeps Text.putStrLn textBuildSummary -printMarkBrokenList :: IO () -printMarkBrokenList = do +printMarkBrokenList :: RequestLogsFlag -> IO () +printMarkBrokenList reqLogs = do (_, fetchTime, buildReport) <- readBuildReports runReq defaultHttpConfig $ forM_ buildReport \build@Build{job, id} -> case (getBuildState build, Text.splitOn "." $ unJobName job) of (Failed, ["haskellPackages", name, "x86_64-linux"]) -> do - -- Fetch build log from hydra to figure out the cause of the error. - build_log <- ByteString.lines <$> hydraPlainQuery ["build", showT id, "nixlog", "1", "raw"] -- We use the last probable error cause found in the build log file. - let error_message = fromMaybe " failure " $ safeLast $ mapMaybe probableErrorCause build_log + error_message <- fromMaybe "failure" <$> + case reqLogs of + NoRequestLogs -> pure Nothing + RequestLogs -> do + -- Fetch build log from hydra to figure out the cause of the error. + build_log <- ByteString.lines <$> hydraPlainQuery ["build", showT id, "nixlog", "1", "raw"] + pure $ safeLast $ mapMaybe probableErrorCause build_log liftIO $ putStrLn $ " - " <> Text.unpack name <> " # " <> error_message <> " in job https://hydra.nixos.org/build/" <> show id <> " at " <> formatTime defaultTimeLocale "%Y-%m-%d" fetchTime _ -> pure () From 76dc15354424f88c38124d5467b18dd5476ecc40 Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Sun, 3 Sep 2023 21:54:03 +0200 Subject: [PATCH 2/4] maintainers/haskell/hydra-report.hs: work around bulk status timeout This change adds a flag --slow to hydra-report.sh get-report which causes it to fetch the cheap evaluation overview endpoint (which only contains build ids and meta data). The gathered information is then used to request each build's status individually instead of in bulk which is very slow, but useful as a last resort if the bulk endpoint times out. --- maintainers/scripts/haskell/hydra-report.hs | 26 +++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/maintainers/scripts/haskell/hydra-report.hs b/maintainers/scripts/haskell/hydra-report.hs index 3dbd66f2e4c2..68ba75452d43 100755 --- a/maintainers/scripts/haskell/hydra-report.hs +++ b/maintainers/scripts/haskell/hydra-report.hs @@ -30,7 +30,7 @@ Because step 1) is quite expensive and takes roughly ~5 minutes the result is ca {-# OPTIONS_GHC -Wall #-} {-# LANGUAGE DataKinds #-} -import Control.Monad (forM_, (<=<)) +import Control.Monad (forM_, forM, (<=<)) import Control.Monad.Trans (MonadIO (liftIO)) import Data.Aeson ( FromJSON, @@ -108,6 +108,7 @@ newtype JobsetEvalInputs = JobsetEvalInputs {nixpkgs :: Nixpkgs} data Eval = Eval { id :: Int , jobsetevalinputs :: JobsetEvalInputs + , builds :: Seq Int } deriving (Generic, ToJSON, FromJSON, Show) @@ -151,18 +152,20 @@ data Build = Build } deriving (Generic, ToJSON, FromJSON, Show) +data HydraSlownessWorkaroundFlag = HydraSlownessWorkaround | NoHydraSlownessWorkaround data RequestLogsFlag = RequestLogs | NoRequestLogs main :: IO () main = do args <- getArgs case args of - ["get-report"] -> getBuildReports + ["get-report", "--slow"] -> getBuildReports HydraSlownessWorkaround + ["get-report"] -> getBuildReports NoHydraSlownessWorkaround ["ping-maintainers"] -> printMaintainerPing ["mark-broken-list", "--no-request-logs"] -> printMarkBrokenList NoRequestLogs ["mark-broken-list"] -> printMarkBrokenList RequestLogs ["eval-info"] -> printEvalInfo - _ -> putStrLn "Usage: get-report | ping-maintainers | mark-broken-list [--no-request-logs] | eval-info" + _ -> putStrLn "Usage: get-report [--slow] | ping-maintainers | mark-broken-list [--no-request-logs] | eval-info" reportFileName :: IO FilePath reportFileName = getXdgDirectory XdgCache "haskell-updates-build-report.json" @@ -170,18 +173,27 @@ reportFileName = getXdgDirectory XdgCache "haskell-updates-build-report.json" showT :: Show a => a -> Text showT = Text.pack . show -getBuildReports :: IO () -getBuildReports = runReq defaultHttpConfig do +getBuildReports :: HydraSlownessWorkaroundFlag -> IO () +getBuildReports opt = runReq defaultHttpConfig do evalMay <- Seq.lookup 0 . evals <$> hydraJSONQuery mempty ["jobset", "nixpkgs", "haskell-updates", "evals"] eval@Eval{id} <- maybe (liftIO $ fail "No Evalution found") pure evalMay liftIO . putStrLn $ "Fetching evaluation " <> show id <> " from Hydra. This might take a few minutes..." - buildReports :: Seq Build <- hydraJSONQuery (responseTimeout 600000000) ["eval", showT id, "builds"] + buildReports <- getEvalBuilds opt id liftIO do fileName <- reportFileName putStrLn $ "Finished fetching all builds from Hydra, saving report as " <> fileName now <- getCurrentTime encodeFile fileName (eval, now, buildReports) +getEvalBuilds :: HydraSlownessWorkaroundFlag -> Int -> Req (Seq Build) +getEvalBuilds NoHydraSlownessWorkaround id = + hydraJSONQuery (responseTimeout 600000000) ["eval", showT id, "builds"] +getEvalBuilds HydraSlownessWorkaround id = do + Eval{builds} <- hydraJSONQuery mempty [ "eval", showT id ] + forM builds $ \buildId -> do + liftIO $ putStrLn $ "Querying build " <> show buildId + hydraJSONQuery mempty [ "build", showT buildId ] + hydraQuery :: HttpResponse a => Proxy a -> Option 'Https -> [Text] -> Req (HttpResponseBody a) hydraQuery responseType option query = responseBody @@ -190,7 +202,7 @@ hydraQuery responseType option query = (foldl' (/:) (https "hydra.nixos.org") query) NoReqBody responseType - (header "User-Agent" "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell)" <> option) + (header "User-Agent" "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell) pls fix https://github.com/NixOS/nixos-org-configurations/issues/270" <> option) hydraJSONQuery :: FromJSON a => Option 'Https -> [Text] -> Req a hydraJSONQuery = hydraQuery jsonResponse From 71cbb538a5b9cb6b69cea6f0c386f29b5f5a5f2e Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Sun, 3 Sep 2023 21:56:38 +0200 Subject: [PATCH 3/4] maintainers/haskell/hydra-report.hs: increase timeout to 15min This seems to be the server side Hydra timeout as well, so it makes sense waiting as long as Hydra will try to give a response. --- maintainers/scripts/haskell/hydra-report.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/scripts/haskell/hydra-report.hs b/maintainers/scripts/haskell/hydra-report.hs index 68ba75452d43..0b8ca6bd1f46 100755 --- a/maintainers/scripts/haskell/hydra-report.hs +++ b/maintainers/scripts/haskell/hydra-report.hs @@ -187,7 +187,7 @@ getBuildReports opt = runReq defaultHttpConfig do getEvalBuilds :: HydraSlownessWorkaroundFlag -> Int -> Req (Seq Build) getEvalBuilds NoHydraSlownessWorkaround id = - hydraJSONQuery (responseTimeout 600000000) ["eval", showT id, "builds"] + hydraJSONQuery (responseTimeout 900000000) ["eval", showT id, "builds"] getEvalBuilds HydraSlownessWorkaround id = do Eval{builds} <- hydraJSONQuery mempty [ "eval", showT id ] forM builds $ \buildId -> do From 5a835f67429e95871a3502d6bfd7055754ca1508 Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Sun, 3 Sep 2023 22:15:15 +0200 Subject: [PATCH 4/4] maintainers/haskell/mark-broken.sh: allow passing --no-request-logs This flag needs to be passed through to hydra-report.hs. --- maintainers/scripts/haskell/mark-broken.sh | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/maintainers/scripts/haskell/mark-broken.sh b/maintainers/scripts/haskell/mark-broken.sh index 2624311aac9d..9aa9433b8023 100755 --- a/maintainers/scripts/haskell/mark-broken.sh +++ b/maintainers/scripts/haskell/mark-broken.sh @@ -10,6 +10,24 @@ set -euo pipefail +do_commit=false +mark_broken_list_flags="" + +for arg in "$@"; do + case "$arg" in + --do-commit) + do_commit=true + ;; + --no-request-logs) + mark_broken_list_flags="$mark_broken_list_flags $arg" + ;; + *) + echo "$0: unknown flag: $arg" + exit 100 + ;; + esac +done + broken_config="pkgs/development/haskell-modules/configuration-hackage2nix/broken.yaml" tmpfile=$(mktemp) @@ -17,7 +35,7 @@ trap "rm ${tmpfile}" 0 echo "Remember that you need to manually run 'maintainers/scripts/haskell/hydra-report.hs get-report' sometime before running this script." echo "Generating a list of broken builds and displaying for manual confirmation ..." -maintainers/scripts/haskell/hydra-report.hs mark-broken-list | sort -i > "$tmpfile" +maintainers/scripts/haskell/hydra-report.hs mark-broken-list $mark_broken_list_flags | sort -i > "$tmpfile" $EDITOR "$tmpfile" @@ -34,7 +52,7 @@ clear="env -u HOME -u NIXPKGS_CONFIG" $clear maintainers/scripts/haskell/regenerate-hackage-packages.sh evalline=$(maintainers/scripts/haskell/hydra-report.hs eval-info) -if [[ "${1:-}" == "--do-commit" ]]; then +if $do_commit; then git add $broken_config git add pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml git add pkgs/development/haskell-modules/hackage-packages.nix