-Provider names are used as a prefix for job ids now. Should fix Paul…

…McInnis/JobFunnel#123. Partially addresses PaulMcInnis/JobFunnel#133.
EmersonCosta0915 · Feb 16, 2021 · 203c036 · 203c036
1 parent 799ddaf
commit 203c036
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/jobfunnel/backend/scrapers/base.py b/jobfunnel/backend/scrapers/base.py
@@ -343,6 +343,9 @@ def scrape_job(self, job_soup: BeautifulSoup, delay: float,
         if job and not invalid_job:
             try:
                 job.validate()
+                # Prefix the id with the scraper name to avoid key conflicts
+                new_key_id = job.provider + '_' + job.key_id
+                job.key_id = new_key_id
             except Exception as err:
                 # Bad job scrapes can't take down execution!
                 # NOTE: desc too short etc, usually indicates that the job