Skip to content

Commit

Permalink
added filtering to subpage loop too
Browse files Browse the repository at this point in the history
  • Loading branch information
jakopako committed Apr 30, 2024
1 parent 39188b5 commit 9a1e758
Showing 1 changed file with 6 additions and 1 deletion.
7 changes: 6 additions & 1 deletion scraper/scraper.go
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,8 @@ func (c Scraper) GetItems(globalConfig *GlobalConfig, rawDyn bool) ([]map[string
// to speed things up we check the filter after each field.
// Like that we safe time if we already know for sure that
// we want to filter out a certain item. Especially, if
// certain elements would need to be fetched from subpages
// certain elements would need to be fetched from subpages.
// filter fast!
if !c.filterItem(currentItem) {
return
}
Expand Down Expand Up @@ -347,6 +348,10 @@ func (c Scraper) GetItems(globalConfig *GlobalConfig, rawDyn bool) ([]map[string
scrLogger.Error(fmt.Sprintf("error while parsing field %s: %v. Skipping item %v.", f.Name, err, currentItem))
return
}
// filter fast!
if !c.filterItem(currentItem) {
return
}
}
}
}
Expand Down

0 comments on commit 9a1e758

Please sign in to comment.