New upstream version 1.19.1.upstream/1.19.1

1 files changed, 211 insertions, 79 deletions

diff --git a/docs/gallery-dl-example.conf b/docs/gallery-dl-example.conf
index d84febd..72e7465 100644
--- a/docs/gallery-dl-example.conf
+++ b/docs/gallery-dl-example.conf
@@ -2,91 +2,123 @@
     "extractor":
     {
         "base-directory": "~/gallery-dl/",
+
+        "#": "set global archive file for all extractors",
         "archive": "~/gallery-dl/archive.sqlite3",
-        "proxy": "http://10.10.1.10:3128",
 
+        "#": "replace invalid path characters with unicode alternatives",
+        "path-restrict": {
+            "\\": "⧹",
+            "/" : "⧸",
+            "|" : "￨",
+            ":" : "꞉",
+            "*" : "∗",
+            "?" : "？",
+            "\"": "″",
+            "<" : "﹤",
+            ">" : "﹥"
+        },
+
+        "#": "write tags for several *booru sites",
         "postprocessors": [
             {
-                "name": "ugoira",
-                "whitelist": ["pixiv", "danbooru"],
-                "ffmpeg-twopass": true,
-                "ffmpeg-args": ["-c:v", "libvpx", "-crf", "4", "-b:v", "5000k", "-an"]
-            },
-            {
                 "name": "metadata",
-                "whitelist": ["danbooru", "yandere", "sankaku"],
-                "mode": "tags"
+                "mode": "tags",
+                "whitelist": ["danbooru", "moebooru", "sankaku"]
             }
         ],
 
         "pixiv":
         {
+            "#": "override global archive setting for pixiv",
             "archive": "~/gallery-dl/archive-pixiv.sqlite3",
 
+            "#": "set custom directory and filename format strings for all pixiv downloads",
             "filename": "{id}{num}.{extension}",
             "directory": ["Pixiv", "Works", "{user[id]}"],
+            "refresh-token": "aBcDeFgHiJkLmNoPqRsTuVwXyZ01234567890-FedC9",
 
-            "username": "foo",
-            "password": "bar",
+            "#": "transform ugoira into lossless MKVs",
+            "ugoira": true,
+            "postprocessors": ["ugoira-copy"],
 
+            "#": "use special settings for favorites and bookmarks",
             "favorite":
             {
                 "directory": ["Pixiv", "Favorites", "{user[id]}"]
             },
-
             "bookmark":
             {
                 "directory": ["Pixiv", "My Bookmarks"],
-
-                "username": "foo123",
-                "password": "bar123"
+                "refresh-token": "01234567890aBcDeFgHiJkLmNoPqRsTuVwXyZ-ZyxW1"
             }
         },
 
+        "danbooru":
+        {
+            "ugoira": true,
+            "postprocessors": ["ugoira-webm"]
+        },
+
         "exhentai":
         {
+            "#": "use cookies instead of logging in with username and password",
             "cookies":
             {
                 "ipb_member_id": "12345",
                 "ipb_pass_hash": "1234567890abcdef",
                 "igneous"      : "123456789",
-                "hath_perks"   : "m1.m2.m3.a-123456789a"
+                "hath_perks"   : "m1.m2.m3.a-123456789a",
+                "sk"           : "n4m34tv3574m2c4e22c35zgeehiw",
+                "sl"           : "dm_2"
             },
 
-            "proxy":
-            {
-                "http": "http://10.10.1.10:8080",
-                "https": "https://10.10.1.10:443"
-            },
+            "#": "wait 2 to 4.8 seconds between HTTP requests",
+            "sleep-request": [2.0, 4.8],
 
             "filename": "{num:>04}_{name}.{extension}",
-            "directory": ["{category!c}", "{title}"],
-
-            "wait-min": 1.0,
-            "wait-max": 5.0
+            "directory": ["{category!c}", "{title}"]
         },
 
-        "mangadex":
+        "sankaku":
         {
-            "chapter-filter": "lang == 'en'",
-            "postprocessors": [{
-                "name": "zip",
-                "keep-files": false,
-                "compression": "zip"
-            }]
+            "#": "authentication with cookies is not possible for sankaku",
+            "username": "user",
+            "password": "#secret#"
+        },
+
+        "furaffinity": {
+            "#": "authentication with username and password is not possible due to CAPTCHA",
+            "cookies": {
+                "a": "01234567-89ab-cdef-fedc-ba9876543210",
+                "b": "fedcba98-7654-3210-0123-456789abcdef"
+            },
+
+            "descriptions": "html",
+            "postprocessors": ["content"]
         },
 
         "deviantart":
         {
+            "#": "download 'gallery' and 'scraps' images for user profile URLs",
             "include": "gallery,scraps",
+
+            "#": "use custom API credentials to avoid 429 errors",
+            "client-id": "98765",
+            "client-secret": "0123456789abcdef0123456789abcdef",
+            "refresh-token": "0123456789abcdef0123456789abcdef01234567",
+
+            "#": "put description texts into a separate directory",
             "metadata": true,
-            "postprocessors": [{
-                "name": "metadata",
-                "mode": "custom",
-                "directory"       : "Descriptions",
-                "content-format"  : "{description}\n",
-                "extension-format": "descr.txt"
-            }]
+            "postprocessors": [
+                {
+                    "name": "metadata",
+                    "mode": "custom",
+                    "directory"       : "Descriptions",
+                    "content-format"  : "{description}\n",
+                    "extension-format": "descr.txt"
+                }
+            ]
         },
 
         "flickr":
@@ -96,88 +128,126 @@
             "size-max": 1920
         },
 
+        "mangadex":
+        {
+            "#": "only download safe/suggestive chapters translated to English",
+            "lang": "en",
+            "ratings": ["safe", "suggestive"],
+
+            "#": "put chapters into '.cbz' archives",
+            "postprocessors": ["cbz"]
+        },
+
         "reddit":
         {
-            "morecomments": true,
-            "date-min": "2017-01",
-            "date-format": "%Y-%m",
-            "recursion": 1
+            "#": "only spawn child extractors for links to specific sites",
+            "whitelist": ["imgur", "redgifs", "gfycat"],
+
+            "#": "put files from child extractors into the reddit directory",
+            "parent-directory": true,
+
+            "#": "transfer metadata to any child extractor as '_reddit'",
+            "parent-metadata": "_reddit"
         },
 
-        "sankaku":
+        "imgur":
         {
-            "sleep": 2,
-            "wait-min": 5.0,
-            "wait-max": 5.0,
-            "cookies": "~/gallery-dl/cookies-sankaku.txt"
+            "#": "use different directory and filename formats when coming from a reddit post",
+            "directory":
+            {
+                "'_reddit' in locals()": []
+            },
+            "filename":
+            {
+                "'_reddit' in locals()": "{_reddit[id]} {id}.{extension}",
+                ""                     : "{id}.{extension}"
+            }
         },
 
         "tumblr":
         {
-            "posts": "all",
+            "posts"   : "all",
             "external": false,
-            "reblogs": false,
-            "inline": true,
+            "reblogs" : false,
+            "inline"  : true,
 
+            "#": "use special settings when downloading liked posts",
             "likes":
             {
-                "posts": "video,photo,link",
+                "posts"   : "video,photo,link",
                 "external": true,
-                "reblogs": true
+                "reblogs" : true
             }
         },
 
+        "twitter":
+        {
+            "#": "write text content for *all* tweets",
+            "postprocessors": ["content"],
+            "text-tweets": true
+        },
+
         "mastodon":
         {
-            "mastodon.xyz":
+            "#": "add 'tabletop.social' as recognized mastodon instance",
+            "#": "(run 'gallery-dl oauth:mastodon:tabletop.social to get an access token')",
+            "tabletop.social":
             {
-                "access-token": "cab65529..."
-            },
-            "tabletop.social": {
+                "root": "https://tabletop.social",
                 "access-token": "513a36c6..."
             },
 
+            "#": "set filename format strings for all 'mastodon' instances",
             "directory": ["mastodon", "{instance}", "{account[username]!l}"],
-            "filename": "{id}_{media[id]}.{extension}"
+            "filename" : "{id}_{media[id]}.{extension}"
         },
 
         "foolslide": {
-            "otscans":   {"root": "https://otscans.com/foolslide"},
+            "#": "add two more foolslide instances",
+            "otscans"  : {"root": "https://otscans.com/foolslide"},
             "helvetica": {"root": "https://helveticascans.com/r" }
         },
 
         "foolfuuka": {
+            "#": "add two other foolfuuka 4chan archives",
             "fireden-onion": {"root": "http://ydt6jy2ng3s3xg2e.onion"},
-            "scalearchive":  {"root": "https://archive.scaled.team"  }
-        },
-
-        "replace invalid path characters with unicode alternatives": null,
-        "path-restrict": {
-            "\\": "⧹",
-            "/" : "⧸",
-            "|" : "￨",
-            ":" : "꞉",
-            "*" : "∗",
-            "?" : "？",
-            "\"": "″",
-            "<" : "﹤",
-            ">" : "﹥"
+            "scalearchive" : {"root": "https://archive.scaled.team"  }
         }
     },
 
     "downloader":
     {
-        "part-directory": "/tmp/.download/",
+        "#": "restrict download speed to 1 MB/s",
         "rate": "1M",
+
+        "#": "show download progress indicator after 2 seconds",
+        "progress": 2.0,
+
+        "#": "retry failed downloads up to 3 times",
         "retries": 3,
-        "timeout": 8.5
+
+        "#": "consider a download 'failed' after 8 seconds of inactivity",
+        "timeout": 8.0,
+
+        "#": "write '.part' files into a special directory",
+        "part-directory": "/tmp/.download/",
+
+        "#": "do not update file modification times",
+        "mtime": false,
+
+        "ytdl":
+        {
+            "#": "use yt-dlp instead of youtube-dl",
+            "module": "yt_dlp"
+        }
     },
 
     "output":
     {
-        "mode": "terminal",
         "log": {
             "level": "info",
+
+            "#": "use different ANSI colors for each log level",
             "format": {
                 "debug"  : "\u001b[0;37m{name}: {message}\u001b[0m",
                 "info"   : "\u001b[1;37m{name}: {message}\u001b[0m",
@@ -185,11 +255,19 @@
                 "error"  : "\u001b[1;31m{name}: {message}\u001b[0m"
             }
         },
+
+        "#": "shorten filenames to fit into one terminal line",
+        "#": "while also considering wider East-Asian characters",
+        "shorten": "eaw",
+
+        "#": "write logging messages to a separate file",
         "logfile": {
             "path": "~/gallery-dl/log.txt",
             "mode": "w",
             "level": "debug"
         },
+
+        "#": "write unrecognized URLs to a separate file",
         "unsupportedfile": {
             "path": "~/gallery-dl/unsupported.txt",
             "mode": "a",
@@ -198,9 +276,63 @@
         }
     },
 
-    "cache": {
-        "file": "~/gallery-dl/cache.sqlite3"
+    "postprocessor":
+    {
+        "#": "write 'content' metadata into separate files",
+        "content":
+        {
+            "name" : "metadata",
+
+            "#": "write data for every post instead of each individual file",
+            "event": "post",
+            "filename": "{post_id|tweet_id|id}.txt",
+
+            "#": "write only the values for 'content' or 'description'",
+            "mode" : "custom",
+            "format": "{content|description}\n"
+        },
+
+        "#": "put files into a '.cbz' archive",
+        "cbz":
+        {
+            "name": "zip",
+            "extension": "cbz"
+        },
+
+        "#": "various ugoira post processor configurations to create different file formats",
+        "ugoira-webm":
+        {
+            "name": "ugoira",
+            "extension": "webm",
+            "ffmpeg-args": ["-c:v", "libvpx-vp9", "-an", "-b:v", "0", "-crf", "30"],
+            "ffmpeg-twopass": true,
+            "ffmpeg-demuxer": "image2"
+        },
+        "ugoira-mp4":
+        {
+            "name": "ugoira",
+            "extension": "mp4",
+            "ffmpeg-args": ["-c:v", "libx264", "-an", "-b:v", "4M", "-preset", "veryslow"],
+            "ffmpeg-twopass": true,
+            "libx264-prevent-odd": true
+        },
+        "ugoira-gif":
+        {
+            "name": "ugoira",
+            "extension": "gif",
+            "ffmpeg-args": ["-filter_complex", "[0:v] split [a][b];[a] palettegen [p];[b][p] paletteuse"]
+        },
+        "ugoira-copy": {
+            "name": "ugoira",
+            "extension": "mkv",
+            "ffmpeg-args": ["-c", "copy"],
+            "libx264-prevent-odd": false,
+            "repeat-last-frame": false
+        }
     },
 
-    "netrc": true
+    "#": "use a custom cache file location",
+    "cache": {
+        "file": "~/gallery-dl/cache.sqlite3"
+    }
 }