fix(kernel_crawler): support debian rpi.

Signed-off-by: Federico Di Pierro <[email protected]>

kernel_crawler/deb.py

@@ -74,8 +74,8 @@ def scan_packages(cls, stream):
                 if key in ('Provides', 'Depends'):
                     value = value.split(', ')
             except ValueError:
-                print(line)
-                raise
+                # Just skip the line if it is malformed
+                continue
             current_package[key] = value
 
         if current_package:

kernel_crawler/debian.py

@@ -31,6 +31,7 @@ def __init__(self, arch):
         mirrors = [
             deb.DebMirror('http://mirrors.edge.kernel.org/debian/', arch, repo_filter),
             deb.DebMirror('http://security.debian.org/', arch, repo_filter),
+            deb.DebMirror('http://archive.raspberrypi.com/debian/', arch, repo_filter),
         ]
         super(DebianMirror, self).__init__(mirrors, arch)
 
@@ -58,7 +59,8 @@ def to_driverkit_config(self, release, deps):
         headers = []
         headers_rt = []
         headers_cloud = []
-        # Magic to obtain `rt`, `cloud` and normal headers:
+        headers_rpi = []
+        # Magic to obtain `rt`, `cloud`, `rpi` and normal headers:
         # List is like this one:
         # "http://security.debian.org/pool/updates/main/l/linux/linux-headers-4.19.0-23-common_4.19.269-1_all.deb",
         # "http://security.debian.org/pool/updates/main/l/linux/linux-headers-4.19.0-23-rt-amd64_4.19.269-1_amd64.deb",
@@ -67,14 +69,16 @@ def to_driverkit_config(self, release, deps):
         # "http://security.debian.org/pool/updates/main/l/linux/linux-headers-4.19.0-23-cloud-amd64_4.19.269-1_amd64.deb",
         # "http://security.debian.org/pool/updates/main/l/linux/linux-headers-4.19.0-23-amd64_4.19.269-1_amd64.deb"
         # So:
-        # * common is split in `common-rt` and `common` (for cloud and normal)
+        # * common is split in `common-rt`, `common-rpi` and `common` (for cloud and normal)
         # * kbuild is the same across all flavors
         # * headers are split between `rt`, `cloud` and normal
         for dep in deps:
             if dep.find("headers") != -1:
                 if dep.find("common") != -1:
                     if dep.find("-rt") != -1:
                         headers_rt.append(dep)
+                    elif dep.find("-rpi") != -1:
+                        headers_rpi.append(dep)
                     else:
                         headers.append(dep)
                         headers_cloud.append(dep)
@@ -83,13 +87,23 @@ def to_driverkit_config(self, release, deps):
                         headers_rt.append(dep)
                     elif dep.find("-cloud") != -1:
                         headers_cloud.append(dep)
+                    elif dep.find("-rpi") != -1:
+                        headers_rpi.append(dep)
                     else:
                         headers.append(dep)
             if dep.find("kbuild") != -1:
                 headers.append(dep)
                 headers_rt.append(dep)
                 headers_cloud.append(dep)
+                headers_rpi.append(dep)
 
-        return [repo.DriverKitConfig(release + "-" + self.arch, "debian", headers),
-                      repo.DriverKitConfig(release + "-rt-" + self.arch, "debian", headers_rt),
-                      repo.DriverKitConfig(release + "-cloud-" + self.arch, "debian", headers_cloud)]
+        final = []
+        if len(headers) >= 3:
+            final.append(repo.DriverKitConfig(release + "-" + self.arch, "debian", headers))
+        if len(headers_rt) >= 3:
+            final.append(repo.DriverKitConfig(release + "-rt-" + self.arch, "debian", headers_rt))
+        if len(headers_cloud) >= 3:
+            final.append(repo.DriverKitConfig(release + "-cloud-" + self.arch, "debian", headers_cloud))
+        if len(headers_rpi) >= 3:
+            final.append(repo.DriverKitConfig(release + "-rpi-" + self.arch, "debian", headers_rpi))
+        return final

kernel_crawler/utils/download.py

@@ -34,9 +34,13 @@ def get_url(url):
 
 def get_first_of(urls):
     last_exc = Exception('Empty url list')
-    for url in urls:
+    for idx, url in enumerate(urls):
         try:
-            return get_url(url)
+            content = get_url(url)
+            # If content is None and we got elements after this one,
+            # try the next ones.
+            if content is not None or idx == len(urls) - 1:
+                return content
         except Exception as exc:
             last_exc = exc
     raise last_exc