From 58a7606c8d2b4889668129282c01364aa9afc5a0 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Thu, 25 Jan 2024 00:59:25 -0500 Subject: [PATCH] Nix.Derivation: vendor parser from sorki's Haskell-Nix-Derivation-Library fork All credit goes to @sorki and @rickynils for their work on: - https://github.com/Gabriella439/Haskell-Nix-Derivation-Library/pull/26 - https://github.com/Gabriella439/Haskell-Nix-Derivation-Library/pull/24 and of course to @Gabriella439 for the entire library! This commit takes the changes relevant to `Nix.Derivation` and vendors them in-tree. Co-Authored-By: Gabriella Gonzalez Co-Authored-By: sorki Co-Authored-By: Rickard Nilsson --- default.nix | 28 +-- lib-nix-derivation/LICENSE | 24 +++ lib-nix-derivation/Nix/Derivation.hs | 22 +++ lib-nix-derivation/Nix/Derivation/Parser.hs | 194 ++++++++++++++++++++ lib-nix-derivation/Nix/Derivation/Types.hs | 79 ++++++++ lib/NOM/Update.hs | 13 +- lib/NOM/Update/Monad.hs | 2 +- nix-output-monitor.cabal | 76 +++++--- 8 files changed, 393 insertions(+), 45 deletions(-) create mode 100644 lib-nix-derivation/LICENSE create mode 100644 lib-nix-derivation/Nix/Derivation.hs create mode 100644 lib-nix-derivation/Nix/Derivation/Parser.hs create mode 100644 lib-nix-derivation/Nix/Derivation/Types.hs diff --git a/default.nix b/default.nix index f79e4b5..832cec8 100644 --- a/default.nix +++ b/default.nix @@ -1,9 +1,9 @@ { mkDerivation, ansi-terminal, async, attoparsec, base, bytestring -, cassava, containers, data-default, directory, extra, filepath -, hermes-json, HUnit, lib, lock-file, MemoTrie, nix-derivation -, optics, random, relude, safe, stm, streamly-core, strict -, strict-types, terminal-size, text, time, transformers -, typed-process, unix, word8 +, cassava, containers, data-default, deepseq, directory, extra +, filepath, hermes-json, HUnit, lib, lock-file, MemoTrie, optics +, random, relude, safe, stm, streamly-core, strict, strict-types +, terminal-size, text, time, transformers, typed-process, unix +, vector, word8 }: mkDerivation { pname = "nix-output-monitor"; @@ -13,24 +13,24 @@ mkDerivation { isExecutable = true; libraryHaskellDepends = [ ansi-terminal async attoparsec base bytestring cassava containers - data-default directory extra filepath hermes-json lock-file - MemoTrie nix-derivation optics relude safe stm streamly-core strict - strict-types terminal-size text time transformers word8 + data-default deepseq directory extra filepath hermes-json lock-file + MemoTrie optics relude safe stm streamly-core strict strict-types + terminal-size text time transformers vector word8 ]; executableHaskellDepends = [ ansi-terminal async attoparsec base bytestring cassava containers data-default directory extra filepath hermes-json lock-file - MemoTrie nix-derivation optics relude safe stm streamly-core strict - strict-types terminal-size text time transformers typed-process - unix word8 + MemoTrie optics relude safe stm streamly-core strict strict-types + terminal-size text time transformers typed-process unix word8 ]; testHaskellDepends = [ ansi-terminal async attoparsec base bytestring cassava containers data-default directory extra filepath hermes-json HUnit lock-file - MemoTrie nix-derivation optics random relude safe stm streamly-core - strict strict-types terminal-size text time transformers - typed-process word8 + MemoTrie optics random relude safe stm streamly-core strict + strict-types terminal-size text time transformers typed-process + word8 ]; + doHaddock = false; homepage = "https://github.com/maralorn/nix-output-monitor"; description = "Processes output of Nix commands to show helpful and pretty information"; license = lib.licenses.agpl3Plus; diff --git a/lib-nix-derivation/LICENSE b/lib-nix-derivation/LICENSE new file mode 100644 index 0000000..2850819 --- /dev/null +++ b/lib-nix-derivation/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2017 Gabriella Gonzalez +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Gabriella Gonzalez nor the names of other contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib-nix-derivation/Nix/Derivation.hs b/lib-nix-derivation/Nix/Derivation.hs new file mode 100644 index 0000000..45e6fe9 --- /dev/null +++ b/lib-nix-derivation/Nix/Derivation.hs @@ -0,0 +1,22 @@ +module Nix.Derivation ( + -- * Types + Derivation (..), + DerivationInputs (..), + DerivationOutput (..), + + -- * Parse derivations + parseDerivation, + parseDerivationWith, + textParser, +) where + +import Nix.Derivation.Parser ( + parseDerivation, + parseDerivationWith, + textParser, + ) +import Nix.Derivation.Types ( + Derivation (..), + DerivationInputs (..), + DerivationOutput (..), + ) diff --git a/lib-nix-derivation/Nix/Derivation/Parser.hs b/lib-nix-derivation/Nix/Derivation/Parser.hs new file mode 100644 index 0000000..ac8da3b --- /dev/null +++ b/lib-nix-derivation/Nix/Derivation/Parser.hs @@ -0,0 +1,194 @@ +{-# LANGUAGE MultiWayIf #-} +{-# OPTIONS_GHC -fno-warn-unused-do-bind #-} + +-- | Parsing logic +module Nix.Derivation.Parser ( + -- * Parser + parseDerivation, + parseDerivationWith, + textParser, +) where + +import Control.Applicative (Applicative (pure)) +import Control.Monad (MonadFail (fail)) +import Data.Attoparsec.Text qualified +import Data.Attoparsec.Text.Lazy (Parser) +import Data.Attoparsec.Text.Lazy qualified +import Data.Bool (Bool (..), not, otherwise, (&&), (||)) +import Data.Eq (Eq (..)) +import Data.Functor ((<$>)) +import Data.Map (Map) +import Data.Map qualified +import Data.Maybe (Maybe (Just)) +import Data.Monoid (Monoid (mempty)) +import Data.Ord (Ord) +import Data.Semigroup (Semigroup ((<>))) +import Data.Set (Set) +import Data.Set qualified +import Data.Text (Text) +import Data.Text qualified +import Data.Vector (Vector) +import Data.Vector qualified +import Nix.Derivation.Types ( + Derivation (..), + DerivationInputs (..), + DerivationOutput (..), + ) +import System.FilePath (FilePath) +import System.FilePath qualified + +listOf :: Parser a -> Parser [a] +listOf element = do + "[" + es <- Data.Attoparsec.Text.Lazy.sepBy element "," + "]" + pure es + +-- | Parse a derivation +parseDerivation :: + Parser + ( Derivation + FilePath + Text + Text + DerivationOutput + DerivationInputs + ) +parseDerivation = + parseDerivationWith + textParser + textParser + (parseDerivationOutputWith filepathParser) + (parseDerivationInputsWith filepathParser textParser) + +{- | Parse a derivation using custom +parsers for filepaths, texts, outputNames and derivation inputs/outputs +-} +parseDerivationWith :: + ( Ord txt + , Ord outputName + ) => + Parser txt -> + Parser outputName -> + Parser (drvOutput fp) -> + Parser (drvInputs fp outputName) -> + Parser (Derivation fp txt outputName drvOutput drvInputs) +parseDerivationWith string outputName parseOutput parseInputs = do + "Derive(" + let keyValue0 = do + "(" + key <- outputName + "," + out <- parseOutput + ")" + pure (key, out) + outputs <- mapOf keyValue0 + "," + inputs <- parseInputs + "," + platform <- string + "," + builder <- string + "," + args <- vectorOf string + "," + let keyValue1 = do + "(" + key <- string + "," + value <- string + ")" + pure (key, value) + env <- mapOf keyValue1 + ")" + pure Derivation{..} + +-- | Parse a derivation output +parseDerivationOutputWith :: + ( Eq fp + , Monoid fp + ) => + Parser fp -> + Parser (DerivationOutput fp) +parseDerivationOutputWith filepath = do + path <- filepath + "," + hashAlgo <- textParser + "," + hash <- textParser + if + | path /= mempty && hashAlgo == mempty && hash == mempty -> + pure DerivationOutput{..} + | path /= mempty && hashAlgo /= mempty && hash /= mempty -> + pure FixedDerivationOutput{..} + | path == mempty && hashAlgo /= mempty && hash == mempty -> + pure ContentAddressedDerivationOutput{..} + | otherwise -> + fail "bad output in derivation" + +-- | Parse a derivation inputs +parseDerivationInputsWith :: + ( Ord fp + , Ord outputName + ) => + Parser fp -> + Parser outputName -> + Parser (DerivationInputs fp outputName) +parseDerivationInputsWith filepath outputName = do + let keyValue = do + "(" + key <- filepath + "," + value <- setOf outputName + ")" + pure (key, value) + drvs <- mapOf keyValue + "," + srcs <- setOf filepath + pure DerivationInputs{..} + +textParser :: Parser Text +textParser = do + "\"" + let predicate c = not (c == '"' || c == '\\') + let loop = do + text0 <- Data.Attoparsec.Text.takeWhile predicate + char0 <- Data.Attoparsec.Text.anyChar + case char0 of + '"' -> do + pure [text0] + _ -> do + char1 <- Data.Attoparsec.Text.anyChar + char2 <- case char1 of + 'n' -> pure '\n' + 'r' -> pure '\r' + 't' -> pure '\t' + _ -> pure char1 + textChunks <- loop + pure (text0 : Data.Text.singleton char2 : textChunks) + Data.Text.concat <$> loop + +filepathParser :: Parser FilePath +filepathParser = do + text <- textParser + let str = Data.Text.unpack text + case (Data.Text.uncons text, System.FilePath.isValid str) of + (Just ('/', _), True) -> do + pure str + _ -> do + fail ("bad path ‘" <> Data.Text.unpack text <> "’ in derivation") + +setOf :: (Ord a) => Parser a -> Parser (Set a) +setOf element = do + es <- listOf element + pure (Data.Set.fromList es) + +vectorOf :: Parser a -> Parser (Vector a) +vectorOf element = do + es <- listOf element + pure (Data.Vector.fromList es) + +mapOf :: (Ord k) => Parser (k, v) -> Parser (Map k v) +mapOf keyValue = do + keyValues <- listOf keyValue + pure (Data.Map.fromList keyValues) diff --git a/lib-nix-derivation/Nix/Derivation/Types.hs b/lib-nix-derivation/Nix/Derivation/Types.hs new file mode 100644 index 0000000..78a25e3 --- /dev/null +++ b/lib-nix-derivation/Nix/Derivation/Types.hs @@ -0,0 +1,79 @@ +{-# OPTIONS_GHC -Wno-partial-fields #-} + +-- | Shared types +module Nix.Derivation.Types ( + -- * Types + Derivation (..), + DerivationInputs (..), + DerivationOutput (..), +) where + +import Control.DeepSeq (NFData) +import Data.Eq (Eq) +import Data.Map (Map) +import Data.Ord (Ord) +import Data.Set (Set) +import Data.Text (Text) +import Data.Vector (Vector) +import GHC.Generics (Generic) +import Text.Show (Show) + +-- | A Nix derivation +data Derivation fp txt outputName drvOutput drvInputs = Derivation + { outputs :: Map outputName (drvOutput fp) + -- ^ Outputs produced by this derivation where keys are output names + , inputs :: drvInputs fp outputName + -- ^ Inputs (sources and derivations) + , platform :: txt + -- ^ Platform required for this derivation + , builder :: txt + -- ^ Code to build the derivation, which can be a path or a builtin function + , args :: Vector txt + -- ^ Arguments passed to the executable used to build to derivation + , env :: Map txt txt + -- ^ Environment variables provided to the executable used to build the + -- derivation + } + deriving stock (Eq, Generic, Ord, Show) + +instance + ( NFData fp + , NFData txt + , NFData outputName + , NFData (drvOutput fp) + , NFData (drvInputs fp outputName) + ) => + NFData (Derivation fp txt outputName drvOutput drvInputs) + +data DerivationInputs fp drvOutput = DerivationInputs + { drvs :: Map fp (Set drvOutput) + -- ^ Inputs that are derivations where keys specify derivation paths and + -- values specify which output names are used by this derivation + , srcs :: Set fp + -- ^ Inputs that are sources + } + deriving stock (Eq, Generic, Ord, Show) + +instance (NFData a, NFData b) => NFData (DerivationInputs a b) + +-- | An output of a Nix derivation +data DerivationOutput fp + = DerivationOutput + { path :: fp + -- ^ Path where the output will be saved + } + | FixedDerivationOutput + { path :: fp + -- ^ Path where the output will be saved + , hashAlgo :: Text + -- ^ Hash used for expected hash computation + , hash :: Text + -- ^ Expected hash + } + | ContentAddressedDerivationOutput + { hashAlgo :: Text + -- ^ Hash used for expected hash computation + } + deriving stock (Eq, Generic, Ord, Show) + +instance (NFData a) => NFData (DerivationOutput a) diff --git a/lib/NOM/Update.hs b/lib/NOM/Update.hs index bdf60aa..2843737 100644 --- a/lib/NOM/Update.hs +++ b/lib/NOM/Update.hs @@ -362,7 +362,7 @@ lookupDerivationInfos drvName = do drvId <- lookupDerivation drvName getDerivationInfos drvId -insertDerivation :: Nix.Derivation FilePath Text -> DerivationId -> ProcessingT m () +insertDerivation :: Nix.Derivation FilePath Text Text Nix.DerivationOutput Nix.DerivationInputs -> DerivationId -> ProcessingT m () insertDerivation derivation drvId = do -- We need to be really careful in this function. The Nix.Derivation keeps the -- read-in derivation file in memory. When using Texts from it we must make @@ -371,12 +371,17 @@ insertDerivation derivation drvId = do outputs <- derivation.outputs & Map.mapKeys (parseOutputName . Text.copy) & Map.traverseMaybeWithKey \_ path -> - parseStorePath (toText (Nix.path path)) & mapM \pathName -> do + let + storePath = case path of + Nix.ContentAddressedDerivationOutput{} -> error "God help us all it's a content-addressed output" + _ -> path.path + in + parseStorePath (toText storePath) & mapM \pathName -> do pathId <- getStorePathId pathName modify' (gfield @"storePathInfos" %~ CMap.adjust (gfield @"producer" .~ Strict.Just drvId) pathId) pure pathId inputSources <- - derivation.inputSrcs & flip foldlM mempty \acc path -> do + derivation.inputs.srcs & flip foldlM mempty \acc path -> do pathIdMay <- parseStorePath (toText path) & mapM \pathName -> do pathId <- getStorePathId pathName @@ -384,7 +389,7 @@ insertDerivation derivation drvId = do pure pathId pure $ maybe id CSet.insert pathIdMay acc inputDerivationsList <- - derivation.inputDrvs & Map.toList & mapMaybeM \(drvPath, outputs_of_input) -> do + derivation.inputs.drvs & Map.toList & mapMaybeM \(drvPath, outputs_of_input) -> do depIdMay <- parseDerivation (toText drvPath) & mapM \depName -> do depId <- lookupDerivation depName diff --git a/lib/NOM/Update/Monad.hs b/lib/NOM/Update/Monad.hs index 9f515f9..a0d2fa9 100644 --- a/lib/NOM/Update/Monad.hs +++ b/lib/NOM/Update/Monad.hs @@ -38,7 +38,7 @@ instance (MonadNow m) => MonadNow (WriterT a m) where getNow = lift getNow class (Monad m) => MonadReadDerivation m where - getDerivation :: Derivation -> m (Either NOMError (Nix.Derivation FilePath Text)) + getDerivation :: Derivation -> m (Either NOMError (Nix.Derivation FilePath Text Text Nix.DerivationOutput Nix.DerivationInputs)) instance MonadReadDerivation IO where getDerivation = diff --git a/nix-output-monitor.cabal b/nix-output-monitor.cabal index 512a281..79bd23e 100644 --- a/nix-output-monitor.cabal +++ b/nix-output-monitor.cabal @@ -34,32 +34,11 @@ source-repository head type: git location: https://git.maralorn.de/nix-output-monitor -common common-config - default-extensions: - AllowAmbiguousTypes - BlockArguments - DataKinds - DeriveAnyClass - DerivingStrategies - DuplicateRecordFields - ImportQualifiedPost - LambdaCase - NoFieldSelectors - NoImplicitPrelude - OverloadedRecordDot - OverloadedStrings - RecordWildCards - StrictData - TypeFamilies - TypeOperators - UnicodeSyntax - ViewPatterns - +common common-deps build-depends: , ansi-terminal , async , attoparsec - , base >=4.10 && <5.0 , bytestring , cassava , containers @@ -67,10 +46,9 @@ common common-config , directory , extra , filepath - , hermes-json >=0.6.0.0 + , hermes-json >=0.6.0.0 , lock-file , MemoTrie - , nix-derivation , optics , relude , safe @@ -84,6 +62,28 @@ common common-config , transformers , word8 +common common-config + default-extensions: + AllowAmbiguousTypes + BlockArguments + DataKinds + DeriveAnyClass + DerivingStrategies + DuplicateRecordFields + ImportQualifiedPost + LambdaCase + NoFieldSelectors + NoImplicitPrelude + OverloadedRecordDot + OverloadedStrings + RecordWildCards + StrictData + TypeFamilies + TypeOperators + UnicodeSyntax + ViewPatterns + + build-depends: base >=4.10 && <5.0 default-language: GHC2021 ghc-options: -Weverything -Wno-missing-import-lists @@ -94,8 +94,27 @@ common common-config -fno-show-valid-hole-fits -fexpose-all-unfoldings -fshow-warning-groups -library +library lib-nix-derivation import: common-config + hs-source-dirs: lib-nix-derivation + exposed-modules: + Nix.Derivation + Nix.Derivation.Parser + Nix.Derivation.Types + + build-depends: + , attoparsec + , containers + , deepseq + , filepath + , text + , vector + +library + import: + , common-config + , common-deps + hs-source-dirs: lib exposed-modules: Data.Sequence.Strict @@ -124,8 +143,13 @@ library NOM.Update.Monad.CacheBuildReports NOM.Util + build-depends: lib-nix-derivation + common exes - import: common-config + import: + , common-config + , common-deps + ghc-options: -threaded -Wno-unused-packages -with-rtsopts=-maxN2 executable nom