Skip to content

Commit

Permalink
chore(kernel_crawler): avoid failing if archlinux GET fails.
Browse files Browse the repository at this point in the history
Signed-off-by: Federico Di Pierro <[email protected]>
  • Loading branch information
FedeDP committed Apr 5, 2024
1 parent 2fecb92 commit 11ba396
Showing 1 changed file with 12 additions and 9 deletions.
21 changes: 12 additions & 9 deletions kernel_crawler/archlinux.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import requests
from bs4 import BeautifulSoup
import re

Expand Down Expand Up @@ -44,14 +44,17 @@ def parse_kernel_release(self, kernel_package):
def get_package_tree(self, filter=''):
packages = {}

soup = BeautifulSoup(get_url(self.base_url), features='lxml')
for a in soup.find_all('a', href=True):
package = a['href']
# skip .sig and .. links
if not package.endswith('.sig') and package != '../':
parsed_kernel_release = self.parse_kernel_release(package)

packages.setdefault(parsed_kernel_release, set()).add(self.base_url + package)
try:
soup = BeautifulSoup(get_url(self.base_url), features='lxml')
for a in soup.find_all('a', href=True):
package = a['href']
# skip .sig and .. links
if not package.endswith('.sig') and package != '../':
parsed_kernel_release = self.parse_kernel_release(package)

packages.setdefault(parsed_kernel_release, set()).add(self.base_url + package)
except requests.HTTPError:
pass

return packages

Expand Down

0 comments on commit 11ba396

Please sign in to comment.