From 0f2c04e70a0ffdd0892d6970cafbcd952d221db5 Mon Sep 17 00:00:00 2001 From: Agustin Henze Date: Wed, 12 Dec 2012 20:15:48 -0300 Subject: Imported Upstream version 5 --- nikola/plugins/compile_rest/__init__.py | 79 ++++ .../compile_rest/pygments_code_block_directive.py | 401 +++++++++++++++++++++ nikola/plugins/compile_rest/youtube.py | 33 ++ 3 files changed, 513 insertions(+) create mode 100644 nikola/plugins/compile_rest/__init__.py create mode 100644 nikola/plugins/compile_rest/pygments_code_block_directive.py create mode 100644 nikola/plugins/compile_rest/youtube.py (limited to 'nikola/plugins/compile_rest') diff --git a/nikola/plugins/compile_rest/__init__.py b/nikola/plugins/compile_rest/__init__.py new file mode 100644 index 0000000..0a25a06 --- /dev/null +++ b/nikola/plugins/compile_rest/__init__.py @@ -0,0 +1,79 @@ +import codecs +import os + +import docutils.core +import docutils.io +from docutils.parsers.rst import directives + +from pygments_code_block_directive import ( + code_block_directive, + listings_directive) +directives.register_directive('code-block', code_block_directive) +directives.register_directive('listing', listings_directive) + +import pygments_code_block_directive +# Below is to make pyflakes happy (sigh) +pygments_code_block_directive +from youtube import youtube +directives.register_directive('youtube', youtube) + +from nikola.plugin_categories import PageCompiler + + +class CompileRest(PageCompiler): + """Compile reSt into HTML.""" + + name = "rest" + + def compile_html(self, source, dest): + """Compile reSt into HTML.""" + try: + os.makedirs(os.path.dirname(dest)) + except: + pass + error_level = 100 + with codecs.open(dest, "w+", "utf8") as out_file: + with codecs.open(source, "r", "utf8") as in_file: + data = in_file.read() + output, error_level = rst2html(data, + settings_overrides={'initial_header_level': 2}) + out_file.write(output) + if error_level < 3: + return True + else: + return False + + +def rst2html(source, source_path=None, source_class=docutils.io.StringInput, + destination_path=None, + reader=None, reader_name='standalone', + parser=None, parser_name='restructuredtext', + writer=None, writer_name='html', + settings=None, settings_spec=None, + settings_overrides=None, config_section=None, + enable_exit_status=None): + """ + Set up & run a `Publisher`, and return a dictionary of document parts. + Dictionary keys are the names of parts, and values are Unicode strings; + encoding is up to the client. For programmatic use with string I/O. + + For encoded string input, be sure to set the 'input_encoding' setting to + the desired encoding. Set it to 'unicode' for unencoded Unicode string + input. Here's how:: + + publish_parts(..., settings_overrides={'input_encoding': 'unicode'}) + + Parameters: see `publish_programmatically`. + """ + output, pub = docutils.core.publish_programmatically( + source=source, source_path=source_path, source_class=source_class, + destination_class=docutils.io.StringOutput, + destination=None, destination_path=destination_path, + reader=reader, reader_name=reader_name, + parser=parser, parser_name=parser_name, + writer=writer, writer_name=writer_name, + settings=settings, settings_spec=settings_spec, + settings_overrides=settings_overrides, + config_section=config_section, + enable_exit_status=enable_exit_status) + return pub.writer.parts['fragment'], pub.document.reporter.max_level diff --git a/nikola/plugins/compile_rest/pygments_code_block_directive.py b/nikola/plugins/compile_rest/pygments_code_block_directive.py new file mode 100644 index 0000000..ac91f3c --- /dev/null +++ b/nikola/plugins/compile_rest/pygments_code_block_directive.py @@ -0,0 +1,401 @@ +# -*- coding: utf-8 -*- +#$Date: 2012-02-28 21:07:21 -0300 (Tue, 28 Feb 2012) $ +#$Revision: 2443 $ + +# :Author: a Pygments author|contributor; Felix Wiemann; Guenter Milde +# :Date: $Date: 2012-02-28 21:07:21 -0300 (Tue, 28 Feb 2012) $ +# :Copyright: This module has been placed in the public domain. +# +# This is a merge of `Using Pygments in ReST documents`_ from the pygments_ +# documentation, and a `proof of concept`_ by Felix Wiemann. +# +# ========== =========================================================== +# 2007-06-01 Removed redundancy from class values. +# 2007-06-04 Merge of successive tokens of same type +# (code taken from pygments.formatters.others). +# 2007-06-05 Separate docutils formatter script +# Use pygments' CSS class names (like the html formatter) +# allowing the use of pygments-produced style sheets. +# 2007-06-07 Merge in the formatting of the parsed tokens +# (misnamed as docutils_formatter) as class DocutilsInterface +# 2007-06-08 Failsave implementation (fallback to a standard literal block +# if pygments not found) +# ========== =========================================================== +# +# :: + +"""Define and register a code-block directive using pygments""" + + +# Requirements +# ------------ +# :: + +import codecs +from copy import copy +import os +import urlparse + +from docutils import nodes, core +from docutils.parsers.rst import directives + +pygments = None +try: + import pygments + from pygments.lexers import get_lexer_by_name + from pygments.formatters.html import _get_ttype_class +except ImportError: + pass + + +# Customisation +# ------------- +# +# Do not insert inline nodes for the following tokens. +# (You could add e.g. Token.Punctuation like ``['', 'p']``.) :: + +unstyled_tokens = [''] + + +# DocutilsInterface +# ----------------- +# +# This interface class combines code from +# pygments.formatters.html and pygments.formatters.others. +# +# It does not require anything of docutils and could also become a part of +# pygments:: + +class DocutilsInterface(object): + """Parse `code` string and yield "classified" tokens. + + Arguments + + code -- string of source code to parse + language -- formal language the code is written in. + + Merge subsequent tokens of the same token-type. + + Yields the tokens as ``(ttype_class, value)`` tuples, + where ttype_class is taken from pygments.token.STANDARD_TYPES and + corresponds to the class argument used in pygments html output. + + """ + + def __init__(self, code, language, custom_args={}): + self.code = code + self.language = language + self.custom_args = custom_args + + def lex(self): + """Get lexer for language (use text as fallback)""" + try: + if self.language and unicode(self.language).lower() != 'none': + lexer = get_lexer_by_name(self.language.lower(), + **self.custom_args + ) + else: + lexer = get_lexer_by_name('text', **self.custom_args) + except ValueError: + # what happens if pygment isn't present ? + lexer = get_lexer_by_name('text') + return pygments.lex(self.code, lexer) + + def join(self, tokens): + """join subsequent tokens of same token-type + """ + tokens = iter(tokens) + (lasttype, lastval) = tokens.next() + for ttype, value in tokens: + if ttype is lasttype: + lastval += value + else: + yield(lasttype, lastval) + (lasttype, lastval) = (ttype, value) + yield(lasttype, lastval) + + def __iter__(self): + """parse code string and yield "clasified" tokens + """ + try: + tokens = self.lex() + except IOError: + yield ('', self.code) + return + + for ttype, value in self.join(tokens): + yield (_get_ttype_class(ttype), value) + + +# code_block_directive +# -------------------- +# :: + +def code_block_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + """Parse and classify content of a code_block.""" + if 'include' in options: + try: + if 'encoding' in options: + encoding = options['encoding'] + else: + encoding = 'utf-8' + content = codecs.open( + options['include'], 'r', encoding).read().rstrip() + except (IOError, UnicodeError): # no file or problem reading it + content = u'' + line_offset = 0 + if content: + # here we define the start-at and end-at options + # so that limit is included in extraction + # this is different than the start-after directive of docutils + # (docutils/parsers/rst/directives/misc.py L73+) + # which excludes the beginning + # the reason is we want to be able to define a start-at like + # def mymethod(self) + # and have such a definition included + + after_text = options.get('start-at', None) + if after_text: + # skip content in include_text before + # *and NOT incl.* a matching text + after_index = content.find(after_text) + if after_index < 0: + raise state_machine.reporter.severe( + 'Problem with "start-at" option of "%s" ' + 'code-block directive:\nText not found.' % + options['start-at']) + content = content[after_index:] + line_offset = len(content[:after_index].splitlines()) + + after_text = options.get('start-after', None) + if after_text: + # skip content in include_text before + # *and incl.* a matching text + after_index = content.find(after_text) + if after_index < 0: + raise state_machine.reporter.severe( + 'Problem with "start-after" option of "%s" ' + 'code-block directive:\nText not found.' % + options['start-after']) + line_offset = len(content[:after_index + + len(after_text)].splitlines()) + content = content[after_index + len(after_text):] + + # same changes here for the same reason + before_text = options.get('end-at', None) + if before_text: + # skip content in include_text after + # *and incl.* a matching text + before_index = content.find(before_text) + if before_index < 0: + raise state_machine.reporter.severe( + 'Problem with "end-at" option of "%s" ' + 'code-block directive:\nText not found.' % + options['end-at']) + content = content[:before_index + len(before_text)] + + before_text = options.get('end-before', None) + if before_text: + # skip content in include_text after + # *and NOT incl.* a matching text + before_index = content.find(before_text) + if before_index < 0: + raise state_machine.reporter.severe( + 'Problem with "end-before" option of "%s" ' + 'code-block directive:\nText not found.' % + options['end-before']) + content = content[:before_index] + + else: + content = u'\n'.join(content) + + if 'tabsize' in options: + tabw = options['tabsize'] + else: + tabw = int(options.get('tab-width', 8)) + + content = content.replace('\t', ' ' * tabw) + + withln = "linenos" in options + if not "linenos_offset" in options: + line_offset = 0 + + language = arguments[0] + # create a literal block element and set class argument + code_block = nodes.literal_block(classes=["code", language]) + + if withln: + lineno = 1 + line_offset + total_lines = content.count('\n') + 1 + line_offset + lnwidth = len(str(total_lines)) + fstr = "\n%%%dd " % lnwidth + code_block += nodes.inline(fstr[1:] % lineno, fstr[1:] % lineno, + classes=['linenumber']) + + # parse content with pygments and add to code_block element + content = content.rstrip() + if pygments is None: + code_block += nodes.Text(content, content) + else: + # The [:-1] is because pygments adds a trailing \n which looks bad + l = list(DocutilsInterface(content, language, options)) + if l[-1] == ('', u'\n'): + l = l[:-1] + for cls, value in l: + if withln and "\n" in value: + # Split on the "\n"s + values = value.split("\n") + # The first piece, pass as-is + code_block += nodes.Text(values[0], values[0]) + # On the second and later pieces, insert \n and linenos + linenos = range(lineno, lineno + len(values)) + for chunk, ln in zip(values, linenos)[1:]: + if ln <= total_lines: + code_block += nodes.inline(fstr % ln, fstr % ln, + classes=['linenumber']) + code_block += nodes.Text(chunk, chunk) + lineno += len(values) - 1 + + elif cls in unstyled_tokens: + # insert as Text to decrease the verbosity of the output. + code_block += nodes.Text(value, value) + else: + code_block += nodes.inline(value, value, classes=[cls]) + + return [code_block] + +# Custom argument validators +# -------------------------- +# :: +# +# Move to separated module?? + + +def string_list(argument): + """ + Converts a space- or comma-separated list of values into a python list + of strings. + (Directive option conversion function) + Based in positive_int_list of docutils.parsers.rst.directives + """ + if ',' in argument: + entries = argument.split(',') + else: + entries = argument.split() + return entries + + +def string_bool(argument): + """ + Converts True, true, False, False in python boolean values + """ + if argument is None: + msg = 'argument required but none supplied; choose "True" or "False"' + raise ValueError(msg) + + elif argument.lower() == 'true': + return True + elif argument.lower() == 'false': + return False + else: + raise ValueError('"%s" unknown; choose from "True" or "False"' + % argument) + + +def csharp_unicodelevel(argument): + return directives.choice(argument, ('none', 'basic', 'full')) + + +def lhs_litstyle(argument): + return directives.choice(argument, ('bird', 'latex')) + + +def raw_compress(argument): + return directives.choice(argument, ('gz', 'bz2')) + + +def listings_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + fname = arguments[0] + options['include'] = os.path.join('listings', fname) + target = urlparse.urlunsplit(("link", 'listing', fname, '', '')) + generated_nodes = [core.publish_doctree('`%s <%s>`_' % (fname, target))[0]] + generated_nodes += code_block_directive(name, [arguments[1]], + options, content, lineno, content_offset, block_text, + state, state_machine) + return generated_nodes + +code_block_directive.arguments = (1, 0, 1) +listings_directive.arguments = (2, 0, 1) +code_block_directive.content = 1 +listings_directive.content = 1 +code_block_directive.options = {'include': directives.unchanged_required, + 'start-at': directives.unchanged_required, + 'end-at': directives.unchanged_required, + 'start-after': directives.unchanged_required, + 'end-before': directives.unchanged_required, + 'linenos': directives.unchanged, + 'linenos_offset': directives.unchanged, + 'tab-width': directives.unchanged, + # generic + 'stripnl': string_bool, + 'stripall': string_bool, + 'ensurenl': string_bool, + 'tabsize': directives.positive_int, + 'encoding': directives.encoding, + # Lua + 'func_name_hightlighting': string_bool, + 'disabled_modules': string_list, + # Python Console + 'python3': string_bool, + # Delphi + 'turbopascal': string_bool, + 'delphi': string_bool, + 'freepascal': string_bool, + 'units': string_list, + # Modula2 + 'pim': string_bool, + 'iso': string_bool, + 'objm2': string_bool, + 'gm2ext': string_bool, + # CSharp + 'unicodelevel': csharp_unicodelevel, + # Literate haskell + 'litstyle': lhs_litstyle, + # Raw + 'compress': raw_compress, + # Rst + 'handlecodeblocks': string_bool, + # Php + 'startinline': string_bool, + 'funcnamehighlighting': string_bool, + 'disabledmodules': string_list, + } + +listings_directive.options = copy(code_block_directive.options) +listings_directive.options.pop('include') + +# .. _doctutils: http://docutils.sf.net/ +# .. _pygments: http://pygments.org/ +# .. _Using Pygments in ReST documents: http://pygments.org/docs/rstdirective/ +# .. _proof of concept: +# http://article.gmane.org/gmane.text.docutils.user/3689 +# +# Test output +# ----------- +# +# If called from the command line, call the docutils publisher to render the +# input:: + +if __name__ == '__main__': + from docutils.core import publish_cmdline, default_description + from docutils.parsers.rst import directives + directives.register_directive('code-block', code_block_directive) + description = "code-block directive test output" + default_description + try: + import locale + locale.setlocale(locale.LC_ALL, '') + except Exception: + pass + publish_cmdline(writer_name='html', description=description) diff --git a/nikola/plugins/compile_rest/youtube.py b/nikola/plugins/compile_rest/youtube.py new file mode 100644 index 0000000..584160b --- /dev/null +++ b/nikola/plugins/compile_rest/youtube.py @@ -0,0 +1,33 @@ +from docutils import nodes +from docutils.parsers.rst import directives + +CODE = """\ + +""" + + +def youtube(name, args, options, content, lineno, + contentOffset, blockText, state, stateMachine): + """ Restructured text extension for inserting youtube embedded videos """ + if len(content) == 0: + return + string_vars = { + 'yid': content[0], + 'width': 425, + 'height': 344, + 'extra': '' + } + extra_args = content[1:] # Because content[0] is ID + extra_args = [ea.strip().split("=") for ea in extra_args] # key=value + extra_args = [ea for ea in extra_args if len(ea) == 2] # drop bad lines + extra_args = dict(extra_args) + if 'width' in extra_args: + string_vars['width'] = extra_args.pop('width') + if 'height' in extra_args: + string_vars['height'] = extra_args.pop('height') + return [nodes.raw('', CODE % (string_vars), format='html')] +youtube.content = True +directives.register_directive('youtube', youtube) -- cgit v1.2.3