diff options
Diffstat (limited to 'gallery_dl/job.py')
| -rw-r--r-- | gallery_dl/job.py | 69 |
1 files changed, 23 insertions, 46 deletions
diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 4e185d0..97a8d3f 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -11,7 +11,6 @@ import json import time import errno import logging -import operator import functools import collections from . import extractor, downloader, postprocessor @@ -201,7 +200,6 @@ class DownloadJob(Job): def __init__(self, url, parent=None): Job.__init__(self, url, parent) self.log = self.get_logger("download") - self.blacklist = None self.fallback = None self.archive = None self.sleep = None @@ -209,6 +207,7 @@ class DownloadJob(Job): self.downloaders = {} self.out = output.select() self.visited = parent.visited if parent else set() + self._extractor_filter = None self._skipcnt = 0 def handle_url(self, url, kwdict): @@ -297,9 +296,9 @@ class DownloadJob(Job): else: extr = extractor.find(url) if extr: - if self.blacklist is None: - self.blacklist = self._build_blacklist() - if extr.category in self.blacklist: + if self._extractor_filter is None: + self._extractor_filter = self._build_extractor_filter() + if not self._extractor_filter(extr): extr = None if extr: @@ -444,22 +443,20 @@ class DownloadJob(Job): self.hooks = collections.defaultdict(list) pp_log = self.get_logger("postprocessor") pp_list = [] - category = self.extractor.category - basecategory = self.extractor.basecategory pp_conf = config.get((), "postprocessor") or {} for pp_dict in postprocessors: if isinstance(pp_dict, str): pp_dict = pp_conf.get(pp_dict) or {"name": pp_dict} - whitelist = pp_dict.get("whitelist") - if whitelist and category not in whitelist and \ - basecategory not in whitelist: - continue - - blacklist = pp_dict.get("blacklist") - if blacklist and ( - category in blacklist or basecategory in blacklist): + clist = pp_dict.get("whitelist") + if clist is not None: + negate = False + else: + clist = pp_dict.get("blacklist") + negate = True + if clist and not util.build_extractor_filter( + clist, negate)(self.extractor): continue name = pp_dict.get("name") @@ -500,38 +497,18 @@ class DownloadJob(Job): if condition(pathfmt.kwdict): callback(pathfmt) - def _build_blacklist(self): - wlist = self.extractor.config("whitelist") - if wlist is not None: - if isinstance(wlist, str): - wlist = wlist.split(",") - - # build a set of all categories - blist = set() - add = blist.add - update = blist.update - get = operator.itemgetter(0) - - for extr in extractor._list_classes(): - category = extr.category - if category: - add(category) - else: - update(map(get, extr.instances)) - - # remove whitelisted categories - blist.difference_update(wlist) - return blist - - blist = self.extractor.config("blacklist") - if blist is not None: - if isinstance(blist, str): - blist = blist.split(",") - blist = set(blist) + def _build_extractor_filter(self): + clist = self.extractor.config("whitelist") + if clist is not None: + negate = False else: - blist = {self.extractor.category} - blist |= util.SPECIAL_EXTRACTORS - return blist + clist = self.extractor.config("blacklist") + negate = True + if clist is None: + clist = (self.extractor.category,) + + return util.build_extractor_filter( + clist, negate, util.SPECIAL_EXTRACTORS) class SimulationJob(DownloadJob): |