New upstream version 1.27.5.upstream/1.27.5

1 files changed, 39 insertions, 5 deletions

diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py
index ea70b58..3686e1b 100644
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@@ -46,11 +46,13 @@ class DeviantartExtractor(Extractor):
         self.extra = self.config("extra", False)
         self.quality = self.config("quality", "100")
         self.original = self.config("original", True)
+        self.previews = self.config("previews", False)
         self.intermediary = self.config("intermediary", True)
         self.comments_avatars = self.config("comments-avatars", False)
         self.comments = self.comments_avatars or self.config("comments", False)
 
         self.api = DeviantartOAuthAPI(self)
+        self.eclipse_api = None
         self.group = False
         self._premium_cache = {}
 
@@ -76,6 +78,11 @@ class DeviantartExtractor(Extractor):
         else:
             self._update_content = self._update_content_default
 
+        if self.previews == "all":
+            self.previews_images = self.previews = True
+        else:
+            self.previews_images = False
+
         journals = self.config("journals", "html")
         if journals == "html":
             self.commit_journal = self._commit_journal_html
@@ -171,8 +178,19 @@ class DeviantartExtractor(Extractor):
 
             if self.commit_journal:
                 if "excerpt" in deviation:
-                    journal = self.api.deviation_content(
-                        deviation["deviationid"])
+                    #  journal = self.api.deviation_content(
+                    #      deviation["deviationid"])
+                    if not self.eclipse_api:
+                        self.eclipse_api = DeviantartEclipseAPI(self)
+                    content = self.eclipse_api.deviation_extended_fetch(
+                        deviation["index"],
+                        deviation["author"]["username"],
+                        "journal",
+                    )["deviation"]["textContent"]
+                    html = content["html"]["markup"]
+                    if html.startswith("{"):
+                        html = content["excerpt"].replace("\n", "<br />")
+                    journal = {"html": html}
                 elif "body" in deviation:
                     journal = {"html": deviation.pop("body")}
                 else:
@@ -197,6 +215,18 @@ class DeviantartExtractor(Extractor):
                     comment["_extractor"] = DeviantartAvatarExtractor
                     yield Message.Queue, url, comment
 
+            if self.previews and "preview" in deviation:
+                preview = deviation["preview"]
+                deviation["is_preview"] = True
+                if self.previews_images:
+                    yield self.commit(deviation, preview)
+                else:
+                    mtype = mimetypes.guess_type(
+                        "a." + deviation["extension"], False)[0]
+                    if mtype and not mtype.startswith("image/"):
+                        yield self.commit(deviation, preview)
+                del deviation["is_preview"]
+
             if not self.extra:
                 continue
 
@@ -284,6 +314,9 @@ class DeviantartExtractor(Extractor):
         html = journal["html"]
         shadow = SHADOW_TEMPLATE.format_map(thumbs[0]) if thumbs else ""
 
+        if not html:
+            self.log.warning("%s: Empty journal content", deviation["index"])
+
         if "css" in journal:
             css, cls = journal["css"], "withskin"
         elif html.startswith("<style"):
@@ -321,10 +354,11 @@ class DeviantartExtractor(Extractor):
         deviation["extension"] = "htm"
         return Message.Url, html, deviation
 
-    @staticmethod
-    def _commit_journal_text(deviation, journal):
+    def _commit_journal_text(self, deviation, journal):
         html = journal["html"]
-        if html.startswith("<style"):
+        if not html:
+            self.log.warning("%s: Empty journal content", deviation["index"])
+        elif html.startswith("<style"):
             html = html.partition("</style>")[2]
         head, _, tail = html.rpartition("<script")
         content = "\n".join(