1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
|
# -*- coding: utf-8 -*-
# Copyright 2019-2022 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Write metadata to external files"""
from .common import PostProcessor
from .. import util, formatter
import sys
import os
class MetadataPP(PostProcessor):
def __init__(self, job, options):
PostProcessor.__init__(self, job)
mode = options.get("mode", "json")
if mode == "custom":
self.write = self._write_custom
cfmt = options.get("content-format") or options.get("format")
if isinstance(cfmt, list):
cfmt = "\n".join(cfmt) + "\n"
self._content_fmt = formatter.parse(cfmt).format_map
ext = "txt"
elif mode == "tags":
self.write = self._write_tags
ext = "txt"
else:
self.write = self._write_json
self.indent = options.get("indent", 4)
self.ascii = options.get("ascii", False)
ext = "json"
directory = options.get("directory")
if directory:
self._directory = self._directory_custom
sep = os.sep + (os.altsep or "")
self._metadir = util.expand_path(directory).rstrip(sep) + os.sep
filename = options.get("filename")
extfmt = options.get("extension-format")
if filename:
if filename == "-":
self.run = self._run_stdout
else:
self._filename = self._filename_custom
self._filename_fmt = formatter.parse(filename).format_map
elif extfmt:
self._filename = self._filename_extfmt
self._extension_fmt = formatter.parse(extfmt).format_map
else:
self.extension = options.get("extension", ext)
events = options.get("event")
if events is None:
events = ("file",)
elif isinstance(events, str):
events = events.split(",")
job.register_hooks({event: self.run for event in events}, options)
archive = options.get("archive")
if archive:
extr = job.extractor
archive = util.expand_path(archive)
archive_format = (
options.get("archive-prefix", extr.category) +
options.get("archive-format", "_MD_" + extr.archive_fmt))
try:
if "{" in archive:
archive = formatter.parse(archive).format_map(
job.pathfmt.kwdict)
self.archive = util.DownloadArchive(
archive, archive_format, "_archive_metadata")
except Exception as exc:
self.log.warning(
"Failed to open download archive at '%s' ('%s: %s')",
archive, exc.__class__.__name__, exc)
else:
self.log.debug("Using download archive '%s'", archive)
else:
self.archive = None
self.mtime = options.get("mtime")
def run(self, pathfmt):
archive = self.archive
if archive and archive.check(pathfmt.kwdict):
return
directory = self._directory(pathfmt)
path = directory + self._filename(pathfmt)
try:
with open(path, "w", encoding="utf-8") as fp:
self.write(fp, pathfmt.kwdict)
except FileNotFoundError:
os.makedirs(directory)
with open(path, "w", encoding="utf-8") as fp:
self.write(fp, pathfmt.kwdict)
if archive:
archive.add(pathfmt.kwdict)
if self.mtime:
mtime = pathfmt.kwdict.get("_mtime")
if mtime:
util.set_mtime(path, mtime)
def _run_stdout(self, pathfmt):
self.write(sys.stdout, pathfmt.kwdict)
def _directory(self, pathfmt):
return pathfmt.realdirectory
def _directory_custom(self, pathfmt):
return os.path.join(pathfmt.realdirectory, self._metadir)
def _filename(self, pathfmt):
return (pathfmt.filename or "metadata") + "." + self.extension
def _filename_custom(self, pathfmt):
return pathfmt.clean_path(pathfmt.clean_segment(
self._filename_fmt(pathfmt.kwdict)))
def _filename_extfmt(self, pathfmt):
kwdict = pathfmt.kwdict
ext = kwdict.get("extension")
kwdict["extension"] = pathfmt.extension
kwdict["extension"] = pathfmt.prefix + self._extension_fmt(kwdict)
filename = pathfmt.build_filename(kwdict)
kwdict["extension"] = ext
return filename
def _write_custom(self, fp, kwdict):
fp.write(self._content_fmt(kwdict))
def _write_tags(self, fp, kwdict):
tags = kwdict.get("tags") or kwdict.get("tag_string")
if not tags:
return
if isinstance(tags, str):
taglist = tags.split(", ")
if len(taglist) < len(tags) / 16:
taglist = tags.split(" ")
tags = taglist
elif isinstance(tags, dict):
taglists = tags.values()
tags = []
extend = tags.extend
for taglist in taglists:
extend(taglist)
tags.sort()
fp.write("\n".join(tags) + "\n")
def _write_json(self, fp, kwdict):
util.dump_json(util.filter_dict(kwdict), fp, self.ascii, self.indent)
__postprocessor__ = MetadataPP
|
