Skip to content

Commit

Permalink
Merge pull request #69 from trailofbits/ww/tweak-podman-discovery
Browse files Browse the repository at this point in the history 
Tweak Podman discovery
  • Loading branch information
@ESultanik
ESultanik authored Jul 29, 2022
2 parents 3803d03 + ef3a766 commit 66fc2ee
Show file tree
Hide file tree
Showing 20 changed files with 1,293 additions and 562 deletions.
45 changes: 26 additions & 19 deletions it_depends/audit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,24 @@ class OSVVulnerability(Vulnerability):

"""Additional keys available from the OSV Vulnerability db."""
EXTRA_KEYS = [
"published", "modified", "withdrawn", "related", "package",
"details", "affects", "affected", "references", "severity",
"database_specific", "ecosystem_specific"]
"published",
"modified",
"withdrawn",
"related",
"package",
"details",
"affects",
"affected",
"references",
"severity",
"database_specific",
"ecosystem_specific",
]

def __init__(self, osv_dict: Dict):
# Get the first available information as summary (N/A if none)
summary = osv_dict.get("summary", "") or osv_dict.get("details", "")\
or "N/A"
super().__init__(
osv_dict["id"], osv_dict.get("aliases", []), summary)
summary = osv_dict.get("summary", "") or osv_dict.get("details", "") or "N/A"
super().__init__(osv_dict["id"], osv_dict.get("aliases", []), summary)

# Inherit all other attributes
for k in OSVVulnerability.EXTRA_KEYS:
Expand All @@ -37,27 +45,25 @@ def from_osv_dict(cls, d: Dict):

class VulnerabilityProvider(ABC):
"""Interface of a vulnerability provider."""
def query(self, pkg: Package) ->\
Iterable[Vulnerability]:

def query(self, pkg: Package) -> Iterable[Vulnerability]:
"""Queries the vulnerability provider for vulnerabilities in pkg"""
raise NotImplementedError()


class OSVProject(VulnerabilityProvider):
"""OSV project vulnerability provider"""

QUERY_URL = "https://api.osv.dev/v1/query"

def query(self, pkg: Package) ->\
Iterable[OSVVulnerability]:
def query(self, pkg: Package) -> Iterable[OSVVulnerability]:
"""Queries the OSV project for vulnerabilities in Package pkg"""
q = {"version": str(pkg.version), "package": {"name": pkg.name}}
r = post(OSVProject.QUERY_URL, json=q).json()
return map(OSVVulnerability.from_osv_dict, r.get("vulns", []))


def vulnerabilities(repo: PackageRepository, nworkers=None) -> \
PackageRepository:

def vulnerabilities(repo: PackageRepository, nworkers=None) -> PackageRepository:
def _get_vulninfo(pkg: Package) -> Tuple[Package, FrozenSet[Vulnerability]]:
"""Enrich a Package with vulnerability information"""
ret = OSVProject().query(pkg)
Expand All @@ -66,9 +72,9 @@ def _get_vulninfo(pkg: Package) -> Tuple[Package, FrozenSet[Vulnerability]]:
# thread handle the updates.
return (pkg, frozenset({vuln: vuln for vuln in ret}))

with ThreadPoolExecutor(max_workers=nworkers) as executor, \
tqdm(desc="Checking for vulnerabilities", leave=False,
unit=" packages") as t:
with ThreadPoolExecutor(max_workers=nworkers) as executor, tqdm(
desc="Checking for vulnerabilities", leave=False, unit=" packages"
) as t:
futures = {executor.submit(_get_vulninfo, pkg): pkg for pkg in repo}
t.total = len(futures)

Expand All @@ -77,8 +83,9 @@ def _get_vulninfo(pkg: Package) -> Tuple[Package, FrozenSet[Vulnerability]]:
t.update(1)
pkg, vulns = future.result()
except Exception as exc:
logger.error("Failed to retrieve vulnerability information. "
"Exception: {}".format(exc))
logger.error(
"Failed to retrieve vulnerability information. " "Exception: {}".format(exc)
)
else:
pkg.update_vulnerabilities(vulns)

Expand Down
115 changes: 81 additions & 34 deletions it_depends/autotools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,29 +11,39 @@
from it_depends.ubuntu.apt import cached_file_to_package as file_to_package

from .dependencies import (
Dependency, DependencyResolver, PackageCache, ResolverAvailability, SimpleSpec, SourcePackage, SourceRepository,
Version
Dependency,
DependencyResolver,
PackageCache,
ResolverAvailability,
SimpleSpec,
SourcePackage,
SourceRepository,
Version,
)

logger = logging.getLogger(__name__)


class AutotoolsResolver(DependencyResolver):
""" This attempts to parse configure.ac in an autotool based repo.
"""This attempts to parse configure.ac in an autotool based repo.
It supports the following macros:
AC_INIT, AC_CHECK_HEADER, AC_CHECK_LIB, PKG_CHECK_MODULES
BUGS:
does not handle boost deps
assumes ubuntu host
"""

name = "autotools"
description = "classifies the dependencies of native/autotools packages parsing configure.ac"

def is_available(self) -> ResolverAvailability:
if shutil.which("autoconf") is None:
return ResolverAvailability(False, "`autoconf` does not appear to be installed! "
"Make sure it is installed and in the PATH.")
return ResolverAvailability(
False,
"`autoconf` does not appear to be installed! "
"Make sure it is installed and in the PATH.",
)
return ResolverAvailability(True)

def can_resolve_from_source(self, repo: SourceRepository) -> bool:
Expand All @@ -47,11 +57,10 @@ def _ac_check_header(header_file, file_to_package_cache=None):
https://www.gnu.org/software/autoconf/manual/autoconf-2.67/html_node/Generic-Headers.html
"""
logger.info(f"AC_CHECK_HEADER {header_file}")
package_name = file_to_package(f"{re.escape(header_file)}", file_to_package_cache=file_to_package_cache)
return Dependency(package=package_name,
semantic_version=SimpleSpec("*"),
source="ubuntu"
)
package_name = file_to_package(
f"{re.escape(header_file)}", file_to_package_cache=file_to_package_cache
)
return Dependency(package=package_name, semantic_version=SimpleSpec("*"), source="ubuntu")

@staticmethod
def _ac_check_lib(function, file_to_package_cache=None):
Expand All @@ -62,7 +71,10 @@ def _ac_check_lib(function, file_to_package_cache=None):
"""
lib_file, function_name = function.split(".")
logger.info(f"AC_CHECK_LIB {lib_file}")
package_name = file_to_package(f"lib{re.escape(lib_file)}(.a|.so)", file_to_package_cache=file_to_package_cache)
package_name = file_to_package(
f"lib{re.escape(lib_file)}(.a|.so)",
file_to_package_cache=file_to_package_cache,
)
return Dependency(package=package_name, semantic_version=SimpleSpec("*"), source="ubuntu")

@staticmethod
Expand All @@ -78,19 +90,23 @@ def _pkg_check_modules(module_name, version=None, file_to_package_cache=None):
module_file = re.escape(module_name + ".pc")
logger.info(f"PKG_CHECK_MODULES {module_file}, {version}")
package_name = file_to_package(module_file, file_to_package_cache=file_to_package_cache)
return Dependency(package=package_name, semantic_version=SimpleSpec(version), source="ubuntu")
return Dependency(
package=package_name, semantic_version=SimpleSpec(version), source="ubuntu"
)

@staticmethod
@functools.lru_cache(maxsize=128)
def _replace_variables(token: str, configure: str):
"""
Search all variable occurrences in token and then try to find
bindings for them in the configure script.
Search all variable occurrences in token and then try to find
bindings for them in the configure script.
"""
if "$" not in token:
return token
variable_list = re.findall(r"\$([a-zA-Z_0-9]+)|\${([_a-zA-Z0-9]+)}", token)
variables = set(var for var in itertools.chain(*variable_list) if var) # remove dups and empty
variables = set(
var for var in itertools.chain(*variable_list) if var
) # remove dups and empty
for var in variables:
logger.info(f"Trying to find bindings for {var} in configure")

Expand All @@ -100,23 +116,28 @@ def _replace_variables(token: str, configure: str):
# For example:
# for var in THIS THAT ;
# TODO/CHALLENGE Merge this two \/
solutions = re.findall(f"{var}=\\s*\"([^\"]*)\"", configure)
solutions = re.findall(f'{var}=\\s*"([^"]*)"', configure)
solutions += re.findall(f"{var}=\\s*'([^']*)'", configure)
if len(solutions) > 1:
logger.warning(f"Found several solutions for {var}: {solutions}")
if len(solutions) == 0:
logger.warning(f"No solution found for binding {var}")
continue
logger.info(f"Found a solution {solutions}")
sol = (solutions+[None, ])[0]
sol = (
solutions
+ [
None,
]
)[0]
if sol is not None:
token = token.replace(f"${var}", sol).replace(f"${{{var}}}", sol)
if "$" in token:
raise ValueError(f"Could not find a binding for variable/s in {token}")
return token

def resolve_from_source(
self, repo: SourceRepository, cache: Optional[PackageCache] = None
self, repo: SourceRepository, cache: Optional[PackageCache] = None
) -> Optional[SourcePackage]:
if not self.can_resolve_from_source(repo):
return None
Expand All @@ -125,19 +146,31 @@ def resolve_from_source(
# builds a temporary copy of configure.ac containing aclocal env
subprocess.check_output(("aclocal", f"--output={tmp.name}"), cwd=repo.path)
with open(tmp.name, "ab") as tmp2:
with open(repo.path/"configure.ac", "rb") as conf:
with open(repo.path / "configure.ac", "rb") as conf:
tmp2.write(conf.read())

trace = subprocess.check_output(
("autoconf", "-t", 'AC_CHECK_HEADER:$n:$1',
"-t", 'AC_CHECK_LIB:$n:$1.$2',
"-t", 'PKG_CHECK_MODULES:$n:$2',
"-t", 'PKG_CHECK_MODULES_STATIC:$n', tmp.name), cwd=repo.path).decode("utf8")
configure = subprocess.check_output(["autoconf", tmp.name], cwd=repo.path).decode("utf8")
(
"autoconf",
"-t",
"AC_CHECK_HEADER:$n:$1",
"-t",
"AC_CHECK_LIB:$n:$1.$2",
"-t",
"PKG_CHECK_MODULES:$n:$2",
"-t",
"PKG_CHECK_MODULES_STATIC:$n",
tmp.name,
),
cwd=repo.path,
).decode("utf8")
configure = subprocess.check_output(["autoconf", tmp.name], cwd=repo.path).decode(
"utf8"
)

file_to_package_cache: List[Tuple[str]] = []
deps = []
for macro in trace.split('\n'):
for macro in trace.split("\n"):
logger.debug(f"Handling: {macro}")
macro, *arguments = macro.split(":")
try:
Expand All @@ -147,28 +180,42 @@ def resolve_from_source(
continue
try:
if macro == "AC_CHECK_HEADER":
deps.append(self._ac_check_header(header_file=arguments[0], file_to_package_cache=file_to_package_cache))
deps.append(
self._ac_check_header(
header_file=arguments[0],
file_to_package_cache=file_to_package_cache,
)
)
elif macro == "AC_CHECK_LIB":
deps.append(self._ac_check_lib(function=arguments[0], file_to_package_cache=file_to_package_cache))
deps.append(
self._ac_check_lib(
function=arguments[0],
file_to_package_cache=file_to_package_cache,
)
)
elif macro == "PKG_CHECK_MODULES":
module_name, *version = arguments[0].split(" ")
deps.append(self._pkg_check_modules(module_name=module_name,
version="".join(version),
file_to_package_cache=file_to_package_cache))
deps.append(
self._pkg_check_modules(
module_name=module_name,
version="".join(version),
file_to_package_cache=file_to_package_cache,
)
)
else:
logger.error("Macro not supported %r", macro)
except Exception as e:
logger.error(str(e))
continue

'''
"""
# Identity of this package.
PACKAGE_NAME='Bitcoin Core'
PACKAGE_TARNAME='bitcoin'
PACKAGE_VERSION='21.99.0'
PACKAGE_STRING='Bitcoin Core 21.99.0'
PACKAGE_BUGREPORT='https://github.com/bitcoin/bitcoin/issues'
PACKAGE_URL='https://bitcoincore.org/'''
PACKAGE_URL='https://bitcoincore.org/"""
try:
package_name = self._replace_variables("$PACKAGE_NAME", configure)
except ValueError as e:
Expand All @@ -180,11 +227,11 @@ def resolve_from_source(
except ValueError as e:
logger.error(str(e))
package_version = "0.0.0"

return SourcePackage(
name=package_name,
version=Version.coerce(package_version),
source=self.name,
dependencies=deps,
source_repo=repo
source_repo=repo,
)
Loading

0 comments on commit 66fc2ee

Please sign in to comment.