WIP: ANE-2123, trying to add AllFilters to walkDir via a Has handle

src/App/Fossa/BinaryDeps.hs

@@ -22,12 +22,20 @@ import Path.Extra (tryMakeRelative)
 import Srclib.Converter qualified as Srclib
 import Srclib.Types (AdditionalDepData (..), SourceUnit (..), SourceUserDefDep (..))
 import Types (DiscoveredProjectType (BinaryDepsProjectType), GraphBreadth (Complete))
+import Control.Carrier.Reader (Reader)
 
 -- | Binary detection is sufficiently different from other analysis types that it cannot be just another strategy.
 -- Instead, binary detection is run separately over the entire scan directory, outputting its own source unit.
 -- The goal of this feature is to enable a FOSSA user to flag all vendored binaries (as defined by git) in the project as dependencies.
 -- Users may then use standard FOSSA UX flows to ignore or add license information to the detected binaries.
-analyzeBinaryDeps :: (Has (Lift IO) sig m, Has Diagnostics sig m, Has Logger sig m, Has ReadFS sig m) => Path Abs Dir -> AllFilters -> m (Maybe SourceUnit)
+analyzeBinaryDeps ::
+  (Has (Lift IO) sig m
+  , Has Diagnostics sig m
+  , Has Logger sig m
+  , Has ReadFS sig m
+  , Has (Reader AllFilters) sig m
+  ) =>
+  Path Abs Dir -> AllFilters -> m (Maybe SourceUnit)
 analyzeBinaryDeps dir filters = do
   binaryPaths <- findBinaries (toPathFilters dir filters) dir
   if null binaryPaths
@@ -43,10 +51,22 @@ analyzeBinaryDeps dir filters = do
 -- The @Path Abs Dir@ provided is used to render the name of the resulting dependency:
 -- if we fallback to a plain "unknown binary" strategy its name is reported as the relative path between the provided @Path Abs Dir@ and the @Path Abs File@.
 -- If the path can't be made relative, the dependency name is the absolute path of the binary.
-analyzeSingleBinary :: (Has (Lift IO) sig m, Has Logger sig m, Has ReadFS sig m, Has Diagnostics sig m) => Path Abs Dir -> Path Abs File -> m SourceUserDefDep
+analyzeSingleBinary ::
+  (Has (Lift IO) sig m
+  , Has Logger sig m
+  , Has ReadFS sig m
+  , Has Diagnostics sig m
+  , Has (Reader AllFilters) sig m
+  ) =>
+  Path Abs Dir -> Path Abs File -> m SourceUserDefDep
 analyzeSingleBinary root file = context ("Analyzing " <> toText file) $ resolveBinary strategies root file
 
-findBinaries :: (Has (Lift IO) sig m, Has Diagnostics sig m, Has ReadFS sig m) => PathFilters -> Path Abs Dir -> m [Path Abs File]
+findBinaries ::
+  (Has (Lift IO) sig m
+  , Has Diagnostics sig m
+  , Has ReadFS sig m
+  , Has (Reader AllFilters) sig m
+  ) => PathFilters -> Path Abs Dir -> m [Path Abs File]
 findBinaries filters = walk' $ \dir _ files -> do
   if shouldFingerprintDir dir filters
     then do
@@ -100,7 +120,14 @@ resolveBinary (resolve : remainingStrategies) = \root file -> do
 resolveBinary [] = strategyRawFingerprint
 
 -- | Functions which may be able to resolve a binary to a dependency.
-strategies :: (Has (Lift IO) sig m, Has Diagnostics sig m, Has Logger sig m, Has ReadFS sig m) => [(Path Abs Dir -> Path Abs File -> m (Maybe SourceUserDefDep))]
+strategies ::
+  (Has (Lift IO) sig m
+  , Has Diagnostics sig m
+  , Has Logger sig m
+  , Has ReadFS sig m
+  , Has (Reader AllFilters) sig m
+  ) =>
+  [(Path Abs Dir -> Path Abs File -> m (Maybe SourceUserDefDep))]
 strategies =
   [resolveJar]
 

src/App/Fossa/BinaryDeps/Jar.hs

@@ -40,6 +40,8 @@ import Strategy.Maven.Pom.PomFile (
   pomLicenseName,
   validatePom,
  )
+import Control.Carrier.Reader (Reader)
+import Discovery.Filters (AllFilters)
 
 data JarMetadata = JarMetadata
   { jarName :: Text
@@ -53,7 +55,13 @@ data JarMetadata = JarMetadata
 --   2. Search inside for a file named `pom.xml`; if there are multiple pick the one with the shortest path.
 --      If a representative pom.xml was found, parse it and return metadata derived from it.
 --   3. Attempt to open `META-INF/MANIFEST.MF`, parse it, and return metadata derived from it.
-resolveJar :: (Has (Lift IO) sig m, Has Diagnostics sig m, Has Logger sig m, Has ReadFS sig m) => Path Abs Dir -> Path Abs File -> m (Maybe SourceUserDefDep)
+resolveJar ::
+  (Has (Lift IO) sig m
+  , Has Diagnostics sig m
+  , Has Logger sig m
+  , Has ReadFS sig m
+  , Has (Reader AllFilters) sig m
+  ) => Path Abs Dir -> Path Abs File -> m (Maybe SourceUserDefDep)
 resolveJar _ file | not $ fileHasSuffix file [".jar", ".aar"] = pure Nothing
 resolveJar root file = do
   let fileDescription = toText file
@@ -101,7 +109,13 @@ metaInfManifestToMeta manifest =
     <*> fromMaybeText "Missing implementation version" (Map.lookup "Implementation-Version" manifest)
     <*> pure "" -- Don't attempt to use Bundle-License; it's a URL and we don't parse it on the backend
 
-tacticPom :: (Has (Lift IO) sig m, Has Diagnostics sig m, Has Logger sig m, Has ReadFS sig m) => Path Abs Dir -> m JarMetadata
+tacticPom ::
+  (Has (Lift IO) sig m
+  , Has Diagnostics sig m
+  , Has Logger sig m
+  , Has ReadFS sig m
+  , Has (Reader AllFilters) sig m
+  ) => Path Abs Dir -> m JarMetadata
 tacticPom archive = context ("Parse representative pom.xml in " <> toText archive) $ do
   poms <- context "Find pom.xml files" $ walk' (collectFilesNamed "pom.xml") (archive </> $(mkRelDir "META-INF"))
   when (length poms > 1) $

src/App/Fossa/LicenseScanner.hs

@@ -81,6 +81,8 @@ import Srclib.Types (
   Locator (..),
  )
 import Types (LicenseScanPathFilters (licenseScanPathFilterFileExclude))
+import Control.Carrier.Reader (Reader)
+import Discovery.Filters (AllFilters)
 
 data LicenseScanErr
   = NoSuccessfulScans SourceLocation
@@ -110,6 +112,7 @@ runLicenseScanOnDir ::
   , Has (Lift IO) sig m
   , Has Exec sig m
   , Has ReadFS sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   Text ->
   Maybe LicenseScanPathFilters ->
@@ -130,6 +133,7 @@ recursivelyScanArchives ::
   , Has Diagnostics sig m
   , Has Finally sig m
   , Has Exec sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   Text ->
   Maybe LicenseScanPathFilters ->
@@ -202,6 +206,7 @@ scanAndUploadVendoredDep ::
   , Has Exec sig m
   , Has ReadFS sig m
   , Has FossaApiClient sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   Path Abs Dir ->
   Maybe LicenseScanPathFilters ->
@@ -220,6 +225,7 @@ scanVendoredDep ::
   , Has StickyLogger sig m
   , Has Exec sig m
   , Has ReadFS sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   Path Abs Dir ->
   Maybe LicenseScanPathFilters ->
@@ -247,6 +253,7 @@ scanArchive ::
   , Has (Lift IO) sig m
   , Has Exec sig m
   , Has ReadFS sig m
+  , Has (Reader AllFilters) sig m
   , Has StickyLogger sig m
   ) =>
   Path Abs Dir ->
@@ -271,6 +278,7 @@ scanDirectory ::
   , Has ReadFS sig m
   , Has Diagnostics sig m
   , Has (Lift IO) sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   Maybe ScannableArchive ->
   Text ->
@@ -287,6 +295,7 @@ scanDirectory origin pathPrefix licenseScanPathFilters uploadKind path = do
 hasAnyFiles ::
   ( Has Diagnostics sig m
   , Has ReadFS sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   Path Abs Dir ->
   m Bool
@@ -306,6 +315,7 @@ scanNonEmptyDirectory ::
   , Has (Lift IO) sig m
   , Has Diagnostics sig m
   , Has ReadFS sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   Text ->
   Maybe LicenseScanPathFilters ->
@@ -349,6 +359,7 @@ licenseScanSourceUnit ::
   , Has Logger sig m
   , Has Exec sig m
   , Has ReadFS sig m
+  , Has (Reader AllFilters) sig m
   , Has FossaApiClient sig m
   ) =>
   VendoredDependencyScanMode ->

src/App/Fossa/Reachability/Maven.hs

@@ -23,6 +23,8 @@ import Strategy.Maven.Pom.PomFile (
   PomBuild (PomBuild),
  )
 import Text.Pretty.Simple (pShow)
+import Control.Carrier.Reader (Reader)
+import Discovery.Filters (AllFilters)
 
 -- | Discovers the JAR files associated with the project at the provided path,
 -- then returns the parsed results of analyzing these JARs.
@@ -32,6 +34,7 @@ mavenJarCallGraph ::
   , Has Diagnostics sig m
   , Has Exec sig m
   , Has (Lift IO) sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   Path Abs Dir ->
   m CallGraphAnalysis
@@ -44,6 +47,7 @@ getJarsByBuild ::
   ( Has Logger sig m
   , Has ReadFS sig m
   , Has Diagnostics sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   Path Abs Dir ->
   m [Path Abs File]

src/App/Fossa/Reachability/Upload.hs

@@ -48,6 +48,7 @@ import Srclib.Types (
   SourceUnitDependency (..),
  )
 import Types (DiscoveredProjectType (..), GraphBreadth (..))
+import Discovery.Filters (AllFilters)
 
 analyzeForReachability ::
   ( Has Logger sig m
@@ -57,6 +58,7 @@ analyzeForReachability ::
   , Has (Lift IO) sig m
   , Has Debug sig m
   , Has (Reader ReachabilityConfig) sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   [DiscoveredProjectScan] ->
   m [SourceUnitReachabilityAttempt]
@@ -107,6 +109,7 @@ callGraphOf ::
   , Has (Lift IO) sig m
   , Has Debug sig m
   , Has (Reader ReachabilityConfig) sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   DiscoveredProjectScan ->
   m SourceUnitReachabilityAttempt

src/App/Fossa/VSI/Analyze.hs

@@ -38,6 +38,7 @@ import Effect.Logger (Color (..), Logger, Severity (SevError, SevInfo, SevWarn),
 import Effect.ReadFS (ReadFS)
 import Path (Abs, Dir, File, Path, Rel, isProperPrefixOf, (</>))
 import Path qualified as P
+import Control.Carrier.Reader (Reader)
 
 runVsiAnalysis ::
   ( Has (Lift IO) sig m
@@ -228,6 +229,7 @@ discover ::
   , Has Finally sig m
   , Has TaskPool sig m
   , Has Logger sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   TBMChan (Path Rel File, Combined) ->
   -- | Filters used to exclude scanned directories

src/App/Fossa/VSI/DynLinked/Internal/Resolve.hs

@@ -35,6 +35,8 @@ import Text.Megaparsec (Parsec, empty, eof, many, takeWhile1P)
 import Text.Megaparsec.Char (char, space1)
 import Text.Megaparsec.Char.Lexer qualified as L
 import Types (DiscoveredProjectType (VsiProjectType), GraphBreadth (Complete))
+import Control.Carrier.Reader (Reader)
+import Discovery.Filters (AllFilters)
 
 -- | Resolves a set of dynamic dependencies into a @SourceUnit@.
 -- Any @DynamicDependency@ that isn't resolved to a Linux package dependency is converted to an unknown binary dependency.
@@ -43,6 +45,7 @@ toSourceUnit ::
   , Has Logger sig m
   , Has ReadFS sig m
   , Has Diagnostics sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   Path Abs Dir ->
   LinuxDistro ->

src/App/Fossa/VSI/Fingerprint.hs

@@ -29,6 +29,8 @@ import Data.Word8 (isSpace)
 import Discovery.Walk (WalkStep (..), walk')
 import Effect.ReadFS (ReadFS, contentIsBinary)
 import Path (Abs, Dir, File, Path, toFilePath)
+import Control.Carrier.Reader (Reader)
+import Discovery.Filters (AllFilters)
 
 -- | Fingerprint deterministically idenfies a file and is derived from its content.
 --
@@ -99,7 +101,7 @@ fingerprintRaw file = context "raw" $ contentIsBinary file >>= doFingerprint
       fp <- hasher $ toFilePath file
       pure $ encodeFingerprint fp
 
-fingerprintContentsRaw :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Lift IO) sig m) => Path Abs Dir -> m [Fingerprint Raw]
+fingerprintContentsRaw :: (Has ReadFS sig m, Has Diagnostics sig m, Has (Lift IO) sig m, Has (Reader AllFilters) sig m) => Path Abs Dir -> m [Fingerprint Raw]
 fingerprintContentsRaw = walk' $ \_ _ files -> do
   fps <- traverse fingerprintRaw files
   pure (fps, WalkContinue)

src/App/Fossa/VSI/IAT/AssertRevisionBinaries.hs

@@ -11,12 +11,15 @@ import Effect.Logger (Logger, logInfo)
 import Effect.ReadFS (ReadFS)
 import Path (Abs, Dir, Path)
 import Srclib.Types (Locator)
+import Control.Carrier.Reader (Reader)
+import Discovery.Filters (AllFilters)
 
 assertRevisionBinaries ::
   ( Has Diagnostics sig m
   , Has ReadFS sig m
   , Has (Lift IO) sig m
   , Has Logger sig m
+  , Has (Reader AllFilters) sig m
   , Has API.FossaApiClient sig m
   ) =>
   Path Abs Dir ->

src/App/Fossa/VSI/IAT/AssertUserDefinedBinaries.hs

@@ -22,7 +22,10 @@ import Control.Effect.Lift (Lift)
 import Control.Monad (void)
 import Effect.Logger (Logger, logInfo)
 import Effect.ReadFS (ReadFS)
+import Control.Carrier.Reader (Reader)
+import Discovery.Filters (AllFilters)
 
+-- TODO: pass a `Has (Reader AllFilters) sig m` effect here somehow?
 linkBinsSubCommand :: SubCommand LinkUserBinsOpts LinkUserBinsConfig
 linkBinsSubCommand = mkSubCommand assertUserDefinedBinaries
 
@@ -31,6 +34,7 @@ assertUserDefinedBinaries ::
   , Has ReadFS sig m
   , Has (Lift IO) sig m
   , Has Logger sig m
+  , Has (Reader AllFilters) sig m
   ) =>
   LinkUserBinsConfig ->
   m ()

src/App/Fossa/VSIDeps.hs

@@ -32,6 +32,7 @@ import Path (Abs, Dir, Path, toFilePath)
 import Srclib.Converter qualified as Srclib
 import Srclib.Types (AdditionalDepData (..), SourceUnit (..), SourceUserDefDep, textToOriginPath)
 import Types (DiscoveredProjectType (..), GraphBreadth (Complete))
+import Control.Carrier.Reader (Reader)
 
 -- | VSI analysis is sufficiently different from other analysis types that it cannot be just another strategy.
 -- Instead, VSI analysis is run separately over the entire scan directory, outputting its own source unit.

src/Discovery/Archive.hs

@@ -51,6 +51,8 @@ import Path qualified as P
 import Path.IO qualified as PIO
 import Prettyprinter (Pretty (pretty), hsep, viaShow, vsep)
 import Prelude hiding (zip)
+import Control.Carrier.Reader (Reader)
+import Discovery.Filters (AllFilters)
 
 data ArchiveUnpackFailure = ArchiveUnpackFailure (Path Abs File) SomeException
 newtype UnsupportedArchiveErr = UnsupportedArchiveErr (Path Abs File)
@@ -84,7 +86,13 @@ convertArchiveToDir file = do
 
 -- | Given a function to run over unarchived contents, recursively unpack archives
 discover ::
-  (Has (Lift IO) sig m, Has ReadFS sig m, Has Diagnostics sig m, Has Finally sig m, Has TaskPool sig m) =>
+  (Has (Lift IO) sig m
+  , Has ReadFS sig m
+  , Has Diagnostics sig m
+  , Has Finally sig m
+  , Has TaskPool sig m
+  , Has (Reader AllFilters) sig m
+  ) =>
   -- | Callback to run on the discovered file
   (Path Abs Dir -> Maybe FileAncestry -> m ()) ->
   -- | Path to the archive