From 143723944033d7a6593d57bd1cf6ae97713b6ce7 Mon Sep 17 00:00:00 2001
From: Unit 193 <unit193@unit193.net>
Date: Mon, 30 Nov 2020 21:12:25 -0500
Subject: New upstream version 1.15.4.

---
 gallery_dl/extractor/2chan.py       |   9 +--
 gallery_dl/extractor/hentainexus.py |  37 ++++++++++-
 gallery_dl/extractor/mangadex.py    | 128 +++++++++++++++++++-----------------
 gallery_dl/extractor/mangapanda.py  |  10 +--
 gallery_dl/extractor/mangoxo.py     |   9 ++-
 gallery_dl/extractor/reddit.py      |  19 ++++--
 gallery_dl/version.py               |   2 +-
 7 files changed, 136 insertions(+), 78 deletions(-)

(limited to 'gallery_dl')

diff --git a/gallery_dl/extractor/2chan.py b/gallery_dl/extractor/2chan.py
index 51e461e..9813f2b 100644
--- a/gallery_dl/extractor/2chan.py
+++ b/gallery_dl/extractor/2chan.py
@@ -21,9 +21,9 @@ class _2chanThreadExtractor(Extractor):
     archive_fmt = "{board}_{thread}_{tim}"
     url_fmt = "https://{server}.2chan.net/{board}/src/{filename}"
     pattern = r"(?:https?://)?([^.]+)\.2chan\.net/([^/]+)/res/(\d+)"
-    test = ("http://dec.2chan.net/70/res/11048.htm", {
-        "url": "2ecf919139bd5d915930530b3576d67c388a2a49",
-        "keyword": "8def4ec98a89fd4fff8bbcbae603604dcb4a3bb9",
+    test = ("http://dec.2chan.net/70/res/4752.htm", {
+        "url": "1c2d2ce8aea0fc71d94735cfc30009d628f33548",
+        "keyword": "f508d6841ea2cb19ed799aac9dc580263ca50651",
     })
 
     def __init__(self, match):
@@ -70,7 +70,8 @@ class _2chanThreadExtractor(Extractor):
         data = self._extract_post(post)
         if data["name"]:
             data["name"] = data["name"].strip()
-        if '<a href="/' in post:
+        path = text.extract(post, '<a href="/', '"')[0]
+        if path and not path.startswith("bin/jump"):
             self._extract_image(post, data)
             data["tim"], _, data["extension"] = data["filename"].partition(".")
             data["time"] = data["tim"][:-3]
diff --git a/gallery_dl/extractor/hentainexus.py b/gallery_dl/extractor/hentainexus.py
index 49c1a98..a6db0d5 100644
--- a/gallery_dl/extractor/hentainexus.py
+++ b/gallery_dl/extractor/hentainexus.py
@@ -10,6 +10,7 @@
 
 from .common import GalleryExtractor, Extractor, Message
 from .. import text, util
+import binascii
 import json
 
 
@@ -57,10 +58,40 @@ class HentainexusGalleryExtractor(GalleryExtractor):
         return data
 
     def images(self, _):
-        url = "{}/read/{}".format(self.root, self.gallery_id)
+        url = "{}/read/{}/001".format(self.root, self.gallery_id)
         page = self.request(url).text
-        urls = text.extract(page, "initReader(", "]")[0] + "]"
-        return [(url, None) for url in json.loads(urls)]
+
+        data = json.loads(self._decode(text.extract(
+            page, 'initReader("', '"')[0]))
+        base = data["b"] + data["r"]
+        gid = data["i"]
+
+        return [
+            ("{}{}/{}/{}".format(base, page["h"], gid, page["p"]), None)
+            for page in data["f"]
+        ]
+
+    @staticmethod
+    def _decode(data):
+        # https://hentainexus.com/static/js/reader.min.js?r=6
+        blob = binascii.a2b_base64(data)
+        key = blob[0:64]
+        indices = list(range(256))
+        result = ""
+
+        x = 0
+        for i in range(256):
+            x = (x + indices[i] + key[i % len(key)]) % 256
+            indices[i], indices[x] = indices[x], indices[i]
+
+        x = i = 0
+        for n in range(64, len(blob)):
+            i = (i + 1) % 256
+            x = (x + indices[i]) % 256
+            indices[i], indices[x] = indices[x], indices[i]
+            result += chr(blob[n] ^ indices[(indices[i] + indices[x]) % 256])
+
+        return result
 
     @staticmethod
     def _join_title(data):
diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py
index d50e0f2..c91e9a8 100644
--- a/gallery_dl/extractor/mangadex.py
+++ b/gallery_dl/extractor/mangadex.py
@@ -28,14 +28,30 @@ class MangadexExtractor(Extractor):
 
     def chapter_data(self, chapter_id):
         """Request API results for 'chapter_id'"""
-        url = "{}/api/chapter/{}".format(self.root, chapter_id)
-        return self.request(url).json()
+        url = "{}/api/v2/chapter/{}".format(self.root, chapter_id)
+        return self.request(url).json()["data"]
 
     @memcache(keyarg=1)
     def manga_data(self, manga_id):
         """Request API results for 'manga_id'"""
-        url = "{}/api/manga/{}".format(self.root, manga_id)
-        return self.request(url).json()
+        url = "{}/api/v2/manga/{}".format(self.root, manga_id)
+        return self.request(url).json()["data"]
+
+    def manga_chapters(self, manga_id):
+        """Request chapter list for 'manga_id'"""
+        url = "{}/api/v2/manga/{}/chapters".format(self.root, manga_id)
+        data = self.request(url).json()["data"]
+
+        groups = {
+            group["id"]: group["name"]
+            for group in data["groups"]
+        }
+
+        for chapter in data["chapters"]:
+            cgroups = chapter["groups"]
+            for idx, group_id in enumerate(cgroups):
+                cgroups[idx] = groups[group_id]
+            yield chapter
 
 
 class MangadexChapterExtractor(MangadexExtractor):
@@ -50,61 +66,58 @@ class MangadexChapterExtractor(MangadexExtractor):
     pattern = r"(?:https?://)?(?:www\.)?mangadex\.(?:org|cc)/chapter/(\d+)"
     test = (
         ("https://mangadex.org/chapter/122094", {
-            "keyword": "ef1084c2845825979e150512fed8fdc209baf05a",
+            "keyword": "89d1b24b4baa1fb737d32711d9f2ade6ea426987",
             #  "content": "50383a4c15124682057b197d40261641a98db514",
         }),
         # oneshot
         ("https://mangadex.cc/chapter/138086", {
             "count": 64,
-            "keyword": "f3da80e57b1acfe1bede7d6ebe82a4bae3f9101a",
+            "keyword": "c53a0e4c12250578a4e630281085875e59532c03",
         }),
     )
 
     def __init__(self, match):
         MangadexExtractor.__init__(self, match)
         self.chapter_id = match.group(1)
-        self.data = None
 
     def items(self):
-        data = self.metadata()
-        imgs = self.images()
-        data["count"] = len(imgs)
-
-        yield Message.Version, 1
-        yield Message.Directory, data
-        for data["page"], url in enumerate(imgs, 1):
-            yield Message.Url, url, text.nameext_from_url(url, data)
-
-    def metadata(self):
-        """Return a dict with general metadata"""
         cdata = self.chapter_data(self.chapter_id)
-        mdata = self.manga_data(cdata["manga_id"])
-        self.data = cdata
+        mdata = self.manga_data(cdata["mangaId"])
 
         chapter, sep, minor = cdata["chapter"].partition(".")
-        return {
-            "manga"   : text.unescape(mdata["manga"]["title"]),
-            "manga_id": cdata["manga_id"],
-            "artist"  : text.unescape(mdata["manga"]["artist"]),
-            "author"  : text.unescape(mdata["manga"]["author"]),
+        lang = self.iso639_map.get(cdata["language"], cdata["language"])
+
+        base = cdata["server"] + cdata["hash"] + "/"
+        if base[0] == "/":
+            base = text.urljoin(self.root, base)
+
+        if "serverFallback" in cdata:
+            fallback = cdata["serverFallback"] + cdata["hash"] + "/"
+        else:
+            fallback = None
+
+        data = {
+            "manga"   : text.unescape(mdata["title"]),
+            "manga_id": mdata["id"],
+            "artist"  : mdata["artist"],
+            "author"  : mdata["author"],
             "title"   : text.unescape(cdata["title"]),
             "volume"  : text.parse_int(cdata["volume"]),
             "chapter" : text.parse_int(chapter),
             "chapter_minor": sep + minor,
             "chapter_id": cdata["id"],
-            "group"   : text.unescape(
-                mdata["chapter"][self.chapter_id]["group_name"]),
+            "group"   : [group["name"] for group in cdata["groups"]],
             "date"    : text.parse_timestamp(cdata["timestamp"]),
-            "lang"    : util.language_to_code(cdata["lang_name"]),
-            "language": cdata["lang_name"],
+            "lang"    : lang,
+            "language": util.code_to_language(lang),
+            "count"   : len(cdata["pages"]),
         }
 
-    def images(self):
-        """Return a list of all image URLs"""
-        base = self.data["server"] + self.data["hash"] + "/"
-        if base.startswith("/"):
-            base = text.urljoin(self.root, base)
-        return [base + page for page in self.data["page_array"]]
+        yield Message.Directory, data
+        for data["page"], page in enumerate(cdata["pages"], 1):
+            if fallback:
+                data["_fallback"] = (fallback + page,)
+            yield Message.Url, base + page, text.nameext_from_url(page, data)
 
 
 class MangadexMangaExtractor(MangadexExtractor):
@@ -117,16 +130,16 @@ class MangadexMangaExtractor(MangadexExtractor):
         ("https://mangadex.org/manga/2946/souten-no-koumori", {
             "pattern": r"https://mangadex.org/chapter/\d+",
             "keyword": {
-                "manga": "Souten no Koumori",
+                "manga"   : "Souten no Koumori",
                 "manga_id": 2946,
-                "title": "re:One[Ss]hot",
-                "volume": 0,
-                "chapter": 0,
+                "title"   : "re:One[Ss]hot",
+                "volume"  : 0,
+                "chapter" : 0,
                 "chapter_minor": "",
                 "chapter_id": int,
-                "group": str,
-                "date": "type:datetime",
-                "lang": str,
+                "group"   : list,
+                "date"    : "type:datetime",
+                "lang"    : str,
                 "language": str,
             },
         }),
@@ -140,7 +153,7 @@ class MangadexMangaExtractor(MangadexExtractor):
 
     def __init__(self, match):
         MangadexExtractor.__init__(self, match)
-        self.manga_id = text.parse_int(match.group(1))
+        self.manga_id = match.group(1)
 
     def items(self):
         yield Message.Version, 1
@@ -150,27 +163,24 @@ class MangadexMangaExtractor(MangadexExtractor):
 
     def chapters(self):
         """Return a sorted list of chapter-metadata dicts"""
-        data = self.manga_data(self.manga_id)
-        if "chapter" not in data:
-            return ()
-        manga = data["manga"]
-
+        manga = self.manga_data(self.manga_id)
         results = []
-        for chid, info in data["chapter"].items():
-            chapter, sep, minor = info["chapter"].partition(".")
-            lang = self.iso639_map.get(info["lang_code"], info["lang_code"])
+
+        for cdata in self.manga_chapters(self.manga_id):
+            chapter, sep, minor = cdata["chapter"].partition(".")
+            lang = self.iso639_map.get(cdata["language"], cdata["language"])
             results.append({
                 "manga"   : text.unescape(manga["title"]),
-                "manga_id": self.manga_id,
-                "artist"  : text.unescape(manga["artist"]),
-                "author"  : text.unescape(manga["author"]),
-                "title"   : text.unescape(info["title"]),
-                "volume"  : text.parse_int(info["volume"]),
+                "manga_id": text.parse_int(self.manga_id),
+                "artist"  : manga["artist"],
+                "author"  : manga["author"],
+                "title"   : text.unescape(cdata["title"]),
+                "volume"  : text.parse_int(cdata["volume"]),
                 "chapter" : text.parse_int(chapter),
                 "chapter_minor": sep + minor,
-                "chapter_id": text.parse_int(chid),
-                "group"   : text.unescape(info["group_name"]),
-                "date"    : text.parse_timestamp(info["timestamp"]),
+                "chapter_id": text.parse_int(cdata["id"]),
+                "group"   : cdata["groups"],
+                "date"    : text.parse_timestamp(cdata["timestamp"]),
                 "lang"    : lang,
                 "language": util.code_to_language(lang),
                 "_extractor": MangadexChapterExtractor,
diff --git a/gallery_dl/extractor/mangapanda.py b/gallery_dl/extractor/mangapanda.py
index 6067bd0..155a9b6 100644
--- a/gallery_dl/extractor/mangapanda.py
+++ b/gallery_dl/extractor/mangapanda.py
@@ -6,7 +6,7 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-"""Extractors for https://www.mangapanda.com/"""
+"""Extractors for http://www.mangapanda.com/"""
 
 from .common import ChapterExtractor, MangaExtractor
 from .. import text
@@ -15,7 +15,7 @@ from .. import text
 class MangapandaBase():
     """Base class for mangapanda extractors"""
     category = "mangapanda"
-    root = "https://www.mangapanda.com"
+    root = "http://www.mangapanda.com"
 
     @staticmethod
     def parse_page(page, data):
@@ -36,7 +36,7 @@ class MangapandaChapterExtractor(MangapandaBase, ChapterExtractor):
     """Extractor for manga-chapters from mangapanda.com"""
     archive_fmt = "{manga}_{chapter}_{page}"
     pattern = r"(?:https?://)?(?:www\.)?mangapanda\.com((/[^/?#]+)/(\d+))"
-    test = ("https://www.mangapanda.com/red-storm/2", {
+    test = ("http://www.mangapanda.com/red-storm/2", {
         "url": "1f633f776e950531ba9b1e81965316458e785261",
         "keyword": "b24df4b9cc36383fb6a44e06d32a3884a4dcb5fb",
     })
@@ -97,8 +97,8 @@ class MangapandaMangaExtractor(MangapandaBase, MangaExtractor):
     chapterclass = MangapandaChapterExtractor
     reverse = False
     pattern = r"(?:https?://)?(?:www\.)?mangapanda\.com(/[^/?#]+)/?$"
-    test = ("https://www.mangapanda.com/mushishi", {
-        "url": "357f965732371cac1990fee8b480f62e29141a42",
+    test = ("http://www.mangapanda.com/mushishi", {
+        "url": "50a1ba730b85426b904da256c80f68ba6a8a2566",
         "keyword": "031b3ea085921c552de017ecbb9b906e462229c9",
     })
 
diff --git a/gallery_dl/extractor/mangoxo.py b/gallery_dl/extractor/mangoxo.py
index 344dd56..ad2947a 100644
--- a/gallery_dl/extractor/mangoxo.py
+++ b/gallery_dl/extractor/mangoxo.py
@@ -89,6 +89,7 @@ class MangoxoAlbumExtractor(MangoxoExtractor):
                 "date": "dt:2019-03-22 14:42:00",
                 "description": str,
             },
+            "id": int,
             "num": int,
             "count": 65,
         },
@@ -107,8 +108,12 @@ class MangoxoAlbumExtractor(MangoxoExtractor):
 
         yield Message.Version, 1
         yield Message.Directory, data
-        for data["num"], image in enumerate(imgs, 1):
-            yield Message.Url, image, text.nameext_from_url(image, data)
+
+        data["extension"] = None
+        for data["num"], path in enumerate(imgs, 1):
+            data["id"] = text.parse_int(text.extract(path, "=", "&")[0])
+            url = self.root + "/external/" + path.rpartition("url=")[2]
+            yield Message.Url, url, text.nameext_from_url(url, data)
 
     def metadata(self, page):
         """Return general metadata"""
diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py
index 0be7f17..273ac05 100644
--- a/gallery_dl/extractor/reddit.py
+++ b/gallery_dl/extractor/reddit.py
@@ -118,18 +118,25 @@ class RedditExtractor(Extractor):
 
     def _extract_gallery(self, submission):
         if submission["gallery_data"] is None:
-            self.log.warning("gallery '%s' was deleted", submission["id"])
+            self.log.warning("gallery %s: deleted", submission["id"])
             return
 
         meta = submission["media_metadata"]
         for item in submission["gallery_data"]["items"]:
-            src = meta[item["media_id"]]["s"]
+            data = meta[item["media_id"]]
+            if data["status"] != "valid" or "s" not in data:
+                self.log.warning(
+                    "gallery %s: skipping item %s ('status: %s')",
+                    submission["id"], item["media_id"], data.get("status"))
+                continue
+            src = data["s"]
             url = src.get("u") or src.get("gif") or src.get("mp4")
             if url:
                 yield url.partition("?")[0].replace("/preview.", "/i.", 1)
             else:
-                self.log.error("Unable to get download URL for item '%s'",
-                               item["media_id"])
+                self.log.error(
+                    "gallery %s: unable to fetch download URL for item %s",
+                    submission["id"], item["media_id"])
                 self.log.debug(src)
 
 
@@ -211,6 +218,10 @@ class RedditSubmissionExtractor(RedditExtractor):
             "pattern": r"https://i\.redd\.it/\w+\.gif",
             "count": 2,
         }),
+        # "failed" gallery item (#1127)
+        ("https://www.reddit.com/r/cosplay/comments/jvwaqr", {
+            "count": 1,
+        }),
         ("https://old.reddit.com/r/lavaporn/comments/2a00np/"),
         ("https://np.reddit.com/r/lavaporn/comments/2a00np/"),
         ("https://m.reddit.com/r/lavaporn/comments/2a00np/"),
diff --git a/gallery_dl/version.py b/gallery_dl/version.py
index 605865f..0683276 100644
--- a/gallery_dl/version.py
+++ b/gallery_dl/version.py
@@ -6,4 +6,4 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-__version__ = "1.15.3"
+__version__ = "1.15.4"
-- 
cgit v1.2.3