-Provider names are used as a prefix for job ids now. Should fix #123.…

… Partially addresses #133.
PaulMcInnis · Feb 16, 2021 · 56a44ec · 56a44ec
1 parent e509ef4
commit 56a44ec
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/jobfunnel/backend/scrapers/base.py b/jobfunnel/backend/scrapers/base.py
@@ -343,6 +343,9 @@ def scrape_job(self, job_soup: BeautifulSoup, delay: float,
         if job and not invalid_job:
             try:
                 job.validate()
+                # Prefix the id with the scraper name to avoid key conflicts
+                new_key_id = job.provider + '_' + job.key_id
+                job.key_id = new_key_id
             except Exception as err:
                 # Bad job scrapes can't take down execution!
                 # NOTE: desc too short etc, usually indicates that the job