From 56a44ec7e63e10f772a1048976f9fbfe45081633 Mon Sep 17 00:00:00 2001 From: thebigg Date: Mon, 15 Feb 2021 18:49:21 -0600 Subject: [PATCH] -Provider names are used as a prefix for job ids now. Should fix PaulMcInnis/JobFunnel#123. Partially addresses PaulMcInnis/JobFunnel#133. --- jobfunnel/backend/scrapers/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jobfunnel/backend/scrapers/base.py b/jobfunnel/backend/scrapers/base.py index ed473661..6b446da9 100644 --- a/jobfunnel/backend/scrapers/base.py +++ b/jobfunnel/backend/scrapers/base.py @@ -343,6 +343,9 @@ def scrape_job(self, job_soup: BeautifulSoup, delay: float, if job and not invalid_job: try: job.validate() + # Prefix the id with the scraper name to avoid key conflicts + new_key_id = job.provider + '_' + job.key_id + job.key_id = new_key_id except Exception as err: # Bad job scrapes can't take down execution! # NOTE: desc too short etc, usually indicates that the job