From 0c4dfdec5b55b6064dccc38bbfb0a7c0699c895a Mon Sep 17 00:00:00 2001 From: Agustin Henze Date: Thu, 30 May 2013 17:41:06 -0300 Subject: Imported Upstream version 5.4.4 --- nikola/plugins/command_check.py | 21 +- nikola/plugins/command_console.py | 76 +- nikola/plugins/command_deploy.py | 26 +- nikola/plugins/command_import_blogger.py | 17 +- nikola/plugins/command_import_wordpress.py | 32 +- nikola/plugins/command_install_theme.py | 4 + nikola/plugins/command_new_post.py | 36 +- nikola/plugins/command_planetoid.plugin | 9 + nikola/plugins/command_planetoid/__init__.py | 287 +++ nikola/plugins/compile_bbcode.py | 16 +- nikola/plugins/compile_html.py | 14 +- nikola/plugins/compile_ipynb.plugin | 10 + nikola/plugins/compile_ipynb/README.txt | 35 + nikola/plugins/compile_ipynb/__init__.py | 100 + nikola/plugins/compile_markdown/__init__.py | 51 +- nikola/plugins/compile_markdown/mdx_gist.py | 189 ++ nikola/plugins/compile_markdown/mdx_nikola.py | 56 + nikola/plugins/compile_markdown/mdx_podcast.py | 87 + nikola/plugins/compile_misaka.plugin | 10 + nikola/plugins/compile_misaka/__init__.py | 82 + nikola/plugins/compile_rest/__init__.py | 81 +- nikola/plugins/compile_rest/dummy.py | 44 + nikola/plugins/compile_rest/gist_directive.py | 2 +- nikola/plugins/compile_rest/listing.py | 121 ++ .../compile_rest/pygments_code_block_directive.py | 424 ---- nikola/plugins/compile_rest/slides.py | 79 +- nikola/plugins/compile_rest/soundcloud.py | 62 +- nikola/plugins/compile_rest/vimeo.py | 114 +- nikola/plugins/compile_rest/youtube.py | 57 +- nikola/plugins/compile_textile.py | 14 +- nikola/plugins/compile_txt2tags.py | 14 +- nikola/plugins/compile_wiki.py | 12 +- nikola/plugins/task_archive.py | 66 +- nikola/plugins/task_copy_assets.py | 25 + nikola/plugins/task_create_bundles.py | 46 +- nikola/plugins/task_indexes.py | 35 +- nikola/plugins/task_localsearch.plugin | 10 + nikola/plugins/task_localsearch/MIT-LICENSE.txt | 20 + nikola/plugins/task_localsearch/__init__.py | 102 + .../files/assets/css/img/expand.png | Bin 0 -> 424 bytes .../task_localsearch/files/assets/css/img/link.png | Bin 0 -> 463 bytes .../files/assets/css/img/loader.gif | Bin 0 -> 4178 bytes .../files/assets/css/img/search.gif | Bin 0 -> 208 bytes .../files/assets/css/tipuesearch.css | 232 +++ .../files/assets/js/tipuesearch.js | 426 ++++ .../files/assets/js/tipuesearch_set.js | 28 + .../task_localsearch/files/tipue_search.html | 31 + nikola/plugins/task_mustache.plugin | 10 + nikola/plugins/task_mustache/__init__.py | 197 ++ .../plugins/task_mustache/mustache-template.html | 29 + nikola/plugins/task_mustache/mustache.html | 36 + nikola/plugins/task_redirect.py | 2 +- nikola/plugins/task_render_galleries.py | 29 +- nikola/plugins/task_render_listings.py | 12 +- nikola/plugins/task_render_pages.py | 5 + nikola/plugins/task_render_posts.py | 84 +- nikola/plugins/task_render_rss.py | 11 +- nikola/plugins/task_render_sources.py | 21 +- nikola/plugins/task_render_tags.py | 20 +- nikola/plugins/task_sitemap/__init__.py | 94 +- nikola/plugins/task_sitemap/sitemap_gen.py | 2137 -------------------- 61 files changed, 2901 insertions(+), 2989 deletions(-) create mode 100644 nikola/plugins/command_planetoid.plugin create mode 100644 nikola/plugins/command_planetoid/__init__.py create mode 100644 nikola/plugins/compile_ipynb.plugin create mode 100644 nikola/plugins/compile_ipynb/README.txt create mode 100644 nikola/plugins/compile_ipynb/__init__.py create mode 100644 nikola/plugins/compile_markdown/mdx_gist.py create mode 100644 nikola/plugins/compile_markdown/mdx_nikola.py create mode 100644 nikola/plugins/compile_markdown/mdx_podcast.py create mode 100644 nikola/plugins/compile_misaka.plugin create mode 100644 nikola/plugins/compile_misaka/__init__.py create mode 100644 nikola/plugins/compile_rest/dummy.py create mode 100644 nikola/plugins/compile_rest/listing.py delete mode 100644 nikola/plugins/compile_rest/pygments_code_block_directive.py create mode 100644 nikola/plugins/task_localsearch.plugin create mode 100644 nikola/plugins/task_localsearch/MIT-LICENSE.txt create mode 100644 nikola/plugins/task_localsearch/__init__.py create mode 100755 nikola/plugins/task_localsearch/files/assets/css/img/expand.png create mode 100755 nikola/plugins/task_localsearch/files/assets/css/img/link.png create mode 100644 nikola/plugins/task_localsearch/files/assets/css/img/loader.gif create mode 100644 nikola/plugins/task_localsearch/files/assets/css/img/search.gif create mode 100755 nikola/plugins/task_localsearch/files/assets/css/tipuesearch.css create mode 100644 nikola/plugins/task_localsearch/files/assets/js/tipuesearch.js create mode 100644 nikola/plugins/task_localsearch/files/assets/js/tipuesearch_set.js create mode 100755 nikola/plugins/task_localsearch/files/tipue_search.html create mode 100644 nikola/plugins/task_mustache.plugin create mode 100644 nikola/plugins/task_mustache/__init__.py create mode 100644 nikola/plugins/task_mustache/mustache-template.html create mode 100644 nikola/plugins/task_mustache/mustache.html delete mode 100644 nikola/plugins/task_sitemap/sitemap_gen.py (limited to 'nikola/plugins') diff --git a/nikola/plugins/command_check.py b/nikola/plugins/command_check.py index a396f63..ea82703 100644 --- a/nikola/plugins/command_check.py +++ b/nikola/plugins/command_check.py @@ -24,6 +24,7 @@ from __future__ import print_function import os +import sys try: from urllib import unquote from urlparse import urlparse @@ -74,14 +75,17 @@ class CommandCheck(Command): print(self.help()) return False if options['links']: - scan_links(options['find_sources']) + failure = scan_links(options['find_sources']) if options['files']: - scan_files() + failure = scan_files() + if failure: + sys.exit(1) existing_targets = set([]) def analize(task, find_sources=False): + rv = False try: filename = task.split(":")[-1] d = lxml.html.fromstring(open(filename).read()) @@ -100,6 +104,7 @@ def analize(task, find_sources=False): if os.path.exists(target_filename): existing_targets.add(target_filename) else: + rv = True print("Broken link in {0}: ".format(filename), target) if find_sources: print("Possible sources:") @@ -109,17 +114,21 @@ def analize(task, find_sources=False): except Exception as exc: print("Error with:", filename, exc) + return rv def scan_links(find_sources=False): print("Checking Links:\n===============\n") + failure = False for task in os.popen('nikola list --all', 'r').readlines(): task = task.strip() if task.split(':')[0] in ('render_tags', 'render_archive', 'render_galleries', 'render_indexes', - 'render_pages', + 'render_pages' 'render_site') and '.html' in task: - analize(task, find_sources) + if analize(task, find_sources): + failure = True + return failure def scan_files(): @@ -127,6 +136,7 @@ def scan_files(): task_fnames = set([]) real_fnames = set([]) # First check that all targets are generated in the right places + failure = False for task in os.popen('nikola list --all', 'r').readlines(): task = task.strip() if 'output' in task and ':' in task: @@ -144,6 +154,7 @@ def scan_files(): print("\nFiles from unknown origins:\n") for f in only_on_output: print(f) + failure = True only_on_input = list(task_fnames - real_fnames) if only_on_input: @@ -151,3 +162,5 @@ def scan_files(): print("\nFiles not generated:\n") for f in only_on_input: print(f) + + return failure diff --git a/nikola/plugins/command_console.py b/nikola/plugins/command_console.py index 4af759f..f4d0295 100644 --- a/nikola/plugins/command_console.py +++ b/nikola/plugins/command_console.py @@ -29,35 +29,77 @@ import os from nikola.plugin_categories import Command -class Deploy(Command): +class Console(Command): """Start debugging console.""" name = "console" + shells = ['ipython', 'bpython', 'plain'] + doc_purpose = "Start an interactive python console with access to your site and configuration." - def _execute(self, options, args): - """Start the console.""" + def ipython(self): + """IPython shell.""" from nikola import Nikola try: import conf + except ImportError: + print("No configuration found, cannot run the console.") + else: + import IPython SITE = Nikola(**conf.__dict__) SITE.scan_posts() - print("You can now access your configuration as conf and your " - "site engine as SITE.") + IPython.embed(header='Nikola Console (conf = configuration, SITE ' + '= site engine)') + + def bpython(self): + """bpython shell.""" + from nikola import Nikola + try: + import conf except ImportError: - print("No configuration found.") - import code + print("No configuration found, cannot run the console.") + else: + import bpython + SITE = Nikola(**conf.__dict__) + SITE.scan_posts() + gl = {'conf': conf, 'SITE': SITE, 'Nikola': Nikola} + bpython.embed(banner='Nikola Console (conf = configuration, SITE ' + '= site engine)', locals_=gl) + + def plain(self): + """Plain Python shell.""" + from nikola import Nikola try: - import readline + import conf + SITE = Nikola(**conf.__dict__) + SITE.scan_posts() + gl = {'conf': conf, 'SITE': SITE, 'Nikola': Nikola} except ImportError: - pass + print("No configuration found, cannot run the console.") else: - import rlcompleter - readline.set_completer(rlcompleter.Completer(globals()).complete) - readline.parse_and_bind("tab:complete") + import code + try: + import readline + except ImportError: + pass + else: + import rlcompleter + readline.set_completer(rlcompleter.Completer(gl).complete) + readline.parse_and_bind("tab:complete") + + pythonrc = os.environ.get("PYTHONSTARTUP") + if pythonrc and os.path.isfile(pythonrc): + try: + execfile(pythonrc) # NOQA + except NameError: + pass + + code.interact(local=gl, banner='Nikola Console (conf = ' + 'configuration, SITE = site engine)') - pythonrc = os.environ.get("PYTHONSTARTUP") - if pythonrc and os.path.isfile(pythonrc): + def _execute(self, options, args): + """Start the console.""" + for shell in self.shells: try: - execfile(pythonrc) # NOQA - except NameError: + return getattr(self, shell)() + except ImportError: pass - code.interact(local=globals()) + raise ImportError diff --git a/nikola/plugins/command_deploy.py b/nikola/plugins/command_deploy.py index ffa86ab..3277567 100644 --- a/nikola/plugins/command_deploy.py +++ b/nikola/plugins/command_deploy.py @@ -23,7 +23,12 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. from __future__ import print_function +from ast import literal_eval +import codecs +from datetime import datetime import os +import subprocess + from nikola.plugin_categories import Command @@ -37,5 +42,24 @@ class Deploy(Command): def _execute(self, command, args): for command in self.site.config['DEPLOY_COMMANDS']: + + # Get last succesful deploy date + timestamp_path = os.path.join(self.site.config['CACHE_FOLDER'], 'lastdeploy') + try: + with open(timestamp_path, 'rb') as inf: + last_deploy = literal_eval(inf.read().strip()) + except Exception: + last_deploy = datetime(1970, 1, 1) # NOQA + print("==>", command) - os.system(command) + ret = subprocess.check_call(command, shell=True) + if ret != 0: # failed deployment + raise Exception("Failed deployment") + print("Successful deployment") + new_deploy = datetime.now() + # Store timestamp of successful deployment + with codecs.open(timestamp_path, 'wb+', 'utf8') as outf: + outf.write(repr(new_deploy)) + # Here is where we would do things with whatever is + # on self.site.timeline and is newer than + # last_deploy diff --git a/nikola/plugins/command_import_blogger.py b/nikola/plugins/command_import_blogger.py index 35a702e..ecc4676 100644 --- a/nikola/plugins/command_import_blogger.py +++ b/nikola/plugins/command_import_blogger.py @@ -73,7 +73,7 @@ class CommandImportBlogger(Command): ] def _execute(self, options, args): - """Import a Wordpress blog from an export file into a Nikola site.""" + """Import a Blogger blog from an export file into a Nikola site.""" # Parse the data if feedparser is None: @@ -126,7 +126,7 @@ class CommandImportBlogger(Command): def generate_base_site(self): if not os.path.exists(self.output_folder): - os.system('nikola init --empty ' + self.output_folder) + os.system('nikola init ' + self.output_folder) else: self.import_into_existing_site = True print('The folder {0} already exists - assuming that this is a ' @@ -176,9 +176,16 @@ class CommandImportBlogger(Command): @staticmethod def write_metadata(filename, title, slug, post_date, description, tags): + if not description: + description = "" + with codecs.open(filename, "w+", "utf8") as fd: - fd.write('\n'.join((title, slug, post_date, ','.join(tags), '', - description))) + fd.write('{0}\n'.format(title)) + fd.write('{0}\n'.format(slug)) + fd.write('{0}\n'.format(post_date)) + fd.write('{0}\n'.format(','.join(tags))) + fd.write('\n') + fd.write('{0}\n'.format(description)) def import_item(self, item, out_folder=None): """Takes an item from the feed and creates a post file.""" @@ -284,7 +291,7 @@ class CommandImportBlogger(Command): if not self.import_into_existing_site: filename = 'conf.py' else: - filename = 'conf.py.wordpress_import-{0}'.format( + filename = 'conf.py.blogger_import-{0}'.format( datetime.datetime.now().strftime('%Y%m%d_%H%M%s')) config_output_path = os.path.join(self.output_folder, filename) print('Configuration will be written to: ' + config_output_path) diff --git a/nikola/plugins/command_import_wordpress.py b/nikola/plugins/command_import_wordpress.py index e7ecca0..b45fe78 100644 --- a/nikola/plugins/command_import_wordpress.py +++ b/nikola/plugins/command_import_wordpress.py @@ -90,7 +90,6 @@ class CommandImportWordpress(Command): def _execute(self, options={}, args=[]): """Import a Wordpress blog from an export file into a Nikola site.""" # Parse the data - print(options, args) if requests is None: print('To use the import_wordpress command,' ' you have to install the "requests" package.') @@ -100,10 +99,16 @@ class CommandImportWordpress(Command): print(self.help()) return - options['filename'] = args[0] + options['filename'] = args.pop(0) - if len(args) > 1: - options['output_folder'] = args[1] + if args and ('output_folder' not in args or + options['output_folder'] == 'new_site'): + options['output_folder'] = args.pop(0) + + if args: + print('You specified additional arguments ({0}). Please consider ' + 'putting these arguments before the filename if you ' + 'are running into problems.'.format(args)) self.wordpress_export_file = options['filename'] self.squash_newlines = options.get('squash_newlines', False) @@ -204,8 +209,12 @@ class CommandImportWordpress(Command): 'PUT TITLE HERE') context['BLOG_DESCRIPTION'] = get_text_tag( channel, 'description', 'PUT DESCRIPTION HERE') - context['SITE_URL'] = get_text_tag(channel, 'link', '#') context['BASE_URL'] = get_text_tag(channel, 'link', '#') + if not context['BASE_URL']: + base_site_url = channel.find('{{{0}}}author'.format(wordpress_namespace)) + context['BASE_URL'] = get_text_tag(base_site_url, None, "http://foo.com") + context['SITE_URL'] = context['BASE_URL'] + author = channel.find('{{{0}}}author'.format(wordpress_namespace)) context['BLOG_EMAIL'] = get_text_tag( author, @@ -314,7 +323,13 @@ class CommandImportWordpress(Command): # link is something like http://foo.com/2012/09/01/hello-world/ # So, take the path, utils.slugify it, and that's our slug link = get_text_tag(item, 'link', None) - slug = utils.slugify(urlparse(link).path) + path = urlparse(link).path + + # In python 2, path is a str. slug requires a unicode + # object. Luckily, paths are also ASCII + if isinstance(path, utils.bytes_str): + path = path.decode('ASCII') + slug = utils.slugify(path) if not slug: # it happens if the post has no "nice" URL slug = get_text_tag( item, '{{{0}}}post_name'.format(wordpress_namespace), None) @@ -334,7 +349,10 @@ class CommandImportWordpress(Command): item, '{http://purl.org/rss/1.0/modules/content/}encoded', '') tags = [] - if status != 'publish': + if status == 'trash': + print('Trashed post "{0}" will not be imported.'.format(title)) + return + elif status != 'publish': tags.append('draft') is_draft = True else: diff --git a/nikola/plugins/command_install_theme.py b/nikola/plugins/command_install_theme.py index 04a2cce..2a0a0cc 100644 --- a/nikola/plugins/command_install_theme.py +++ b/nikola/plugins/command_install_theme.py @@ -64,6 +64,10 @@ class CommandInstallTheme(Command): def _execute(self, options, args): """Install theme into current site.""" + if requests is None: + print('This command requires the requests package be installed.') + return False + listing = options['list'] url = options['url'] if args: diff --git a/nikola/plugins/command_new_post.py b/nikola/plugins/command_new_post.py index a823da3..933a51a 100644 --- a/nikola/plugins/command_new_post.py +++ b/nikola/plugins/command_new_post.py @@ -49,13 +49,31 @@ def filter_post_pages(compiler, is_post, post_compilers, post_pages): if not filtered: type_name = "post" if is_post else "page" - raise Exception("Can't find a way, using your configuration, to create" + raise Exception("Can't find a way, using your configuration, to create " "a {0} in format {1}. You may want to tweak " "post_compilers or post_pages in conf.py".format( type_name, compiler)) return filtered[0] +def get_default_compiler(is_post, post_compilers, post_pages): + """Given post_compilers and post_pages, return a reasonable + default compiler for this kind of post/page. + """ + + # First throw away all the post_pages with the wrong is_post + filtered = [entry for entry in post_pages if entry[3] == is_post] + + # Get extensions in filtered post_pages until one matches a compiler + for entry in filtered: + extension = os.path.splitext(entry[0])[-1] + for compiler, extensions in post_compilers.items(): + if extension in extensions: + return compiler + # No idea, back to default behaviour + return 'rest' + + class CommandNewPost(Command): """Create a new post.""" @@ -105,7 +123,7 @@ class CommandNewPost(Command): 'short': 'f', 'long': 'format', 'type': str, - 'default': 'rest', + 'default': '', 'help': 'Markup format for post, one of rest, markdown, wiki, ' 'bbcode, html, textile, txt2tags', } @@ -140,6 +158,12 @@ class CommandNewPost(Command): post_format = options['post_format'] + if not post_format: # Issue #400 + post_format = get_default_compiler( + is_post, + self.site.config['post_compilers'], + self.site.config['post_pages']) + if post_format not in compiler_names: print("ERROR: Unknown post format " + post_format) return @@ -160,12 +184,14 @@ class CommandNewPost(Command): title = sys.stdin.readline() else: print("Title:", title) - if isinstance(title, bytes): + if isinstance(title, utils.bytes_str): title = title.decode(sys.stdin.encoding) title = title.strip() if not path: slug = utils.slugify(title) else: + if isinstance(path, utils.bytes_str): + path = path.decode(sys.stdin.encoding) slug = utils.slugify(os.path.splitext(os.path.basename(path))[0]) date = datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S') data = [title, slug, date, tags] @@ -186,7 +212,9 @@ class CommandNewPost(Command): d_name = os.path.dirname(txt_path) if not os.path.exists(d_name): os.makedirs(d_name) - compiler_plugin.create_post(txt_path, onefile, title, slug, date, tags) + compiler_plugin.create_post( + txt_path, onefile, title=title, + slug=slug, date=date, tags=tags) if not onefile: # write metadata file with codecs.open(meta_path, "wb+", "utf8") as fd: diff --git a/nikola/plugins/command_planetoid.plugin b/nikola/plugins/command_planetoid.plugin new file mode 100644 index 0000000..8636d49 --- /dev/null +++ b/nikola/plugins/command_planetoid.plugin @@ -0,0 +1,9 @@ +[Core] +Name = planetoid +Module = command_planetoid + +[Documentation] +Author = Roberto Alsina +Version = 0.1 +Website = http://nikola.ralsina.com.ar +Description = Maintain a planet-like site diff --git a/nikola/plugins/command_planetoid/__init__.py b/nikola/plugins/command_planetoid/__init__.py new file mode 100644 index 0000000..183dd51 --- /dev/null +++ b/nikola/plugins/command_planetoid/__init__.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2012 Roberto Alsina y otros. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import print_function, unicode_literals +import codecs +import datetime +import hashlib +from optparse import OptionParser +import os +import sys + +from doit.tools import timeout +from nikola.plugin_categories import Command, Task +from nikola.utils import config_changed + +try: + import feedparser +except ImportError: + feedparser = None # NOQA + +try: + import peewee +except ImportError: + peewee = None + + +if peewee is not None: + class Feed(peewee.Model): + name = peewee.CharField() + url = peewee.CharField(max_length=200) + last_status = peewee.CharField(null=True) + etag = peewee.CharField(max_length=200) + last_modified = peewee.DateTimeField() + + class Entry(peewee.Model): + date = peewee.DateTimeField() + feed = peewee.ForeignKeyField(Feed) + content = peewee.TextField(max_length=20000) + link = peewee.CharField(max_length=200) + title = peewee.CharField(max_length=200) + guid = peewee.CharField(max_length=200) + + +class Planetoid(Command, Task): + """Maintain a planet-like thing.""" + name = "planetoid" + + def init_db(self): + # setup database + Feed.create_table(fail_silently=True) + Entry.create_table(fail_silently=True) + + def gen_tasks(self): + if peewee is None or sys.version_info[0] == 3: + if sys.version_info[0] == 3: + message = 'Peewee is currently incompatible with Python 3.' + else: + message = 'You need to install the \"peewee\" module.' + + yield { + 'basename': self.name, + 'name': '', + 'verbosity': 2, + 'actions': ['echo "%s"' % message] + } + else: + self.init_db() + self.load_feeds() + for task in self.task_update_feeds(): + yield task + for task in self.task_generate_posts(): + yield task + yield { + 'basename': self.name, + 'name': '', + 'actions': [], + 'file_dep': ['feeds'], + 'task_dep': [ + self.name + "_fetch_feed", + self.name + "_generate_posts", + ] + } + + def run(self, *args): + parser = OptionParser(usage="nikola %s [options]" % self.name) + (options, args) = parser.parse_args(list(args)) + + def load_feeds(self): + "Read the feeds file, add it to the database." + feeds = [] + feed = name = None + for line in codecs.open('feeds', 'r', 'utf-8'): + line = line.strip() + if line.startswith("#"): + continue + elif line.startswith('http'): + feed = line + elif line: + name = line + if feed and name: + feeds.append([feed, name]) + feed = name = None + + def add_feed(name, url): + f = Feed.create( + name=name, + url=url, + etag='foo', + last_modified=datetime.datetime(1970, 1, 1), + ) + f.save() + + def update_feed_url(feed, url): + feed.url = url + feed.save() + + for feed, name in feeds: + f = Feed.select().where(Feed.name == name) + if not list(f): + add_feed(name, feed) + elif list(f)[0].url != feed: + update_feed_url(list(f)[0], feed) + + def task_update_feeds(self): + """Download feed contents, add entries to the database.""" + def update_feed(feed): + modified = feed.last_modified.timetuple() + etag = feed.etag + try: + parsed = feedparser.parse( + feed.url, + etag=etag, + modified=modified + ) + feed.last_status = str(parsed.status) + except: # Probably a timeout + # TODO: log failure + return + if parsed.feed.get('title'): + print(parsed.feed.title) + else: + print(feed.url) + feed.etag = parsed.get('etag', 'foo') + modified = tuple(parsed.get('date_parsed', (1970, 1, 1)))[:6] + print("==========>", modified) + modified = datetime.datetime(*modified) + feed.last_modified = modified + feed.save() + # No point in adding items from missinfg feeds + if parsed.status > 400: + # TODO log failure + return + for entry_data in parsed.entries: + print("=========================================") + date = entry_data.get('published_parsed', None) + if date is None: + date = entry_data.get('updated_parsed', None) + if date is None: + print("Can't parse date from:") + print(entry_data) + return False + print("DATE:===>", date) + date = datetime.datetime(*(date[:6])) + title = "%s: %s" % (feed.name, entry_data.get('title', 'Sin título')) + content = entry_data.get('content', None) + if content: + content = content[0].value + if not content: + content = entry_data.get('description', None) + if not content: + content = entry_data.get('summary', 'Sin contenido') + guid = str(entry_data.get('guid', entry_data.link)) + link = entry_data.link + print(repr([date, title])) + e = list(Entry.select().where(Entry.guid == guid)) + print( + repr(dict( + date=date, + title=title, + content=content, + guid=guid, + feed=feed, + link=link, + )) + ) + if not e: + entry = Entry.create( + date=date, + title=title, + content=content, + guid=guid, + feed=feed, + link=link, + ) + else: + entry = e[0] + entry.date = date + entry.title = title + entry.content = content + entry.link = link + entry.save() + flag = False + for feed in Feed.select(): + flag = True + task = { + 'basename': self.name + "_fetch_feed", + 'name': str(feed.url), + 'actions': [(update_feed, (feed, ))], + 'uptodate': [timeout(datetime.timedelta(minutes= + self.site.config.get('PLANETOID_REFRESH', 60)))], + } + yield task + if not flag: + yield { + 'basename': self.name + "_fetch_feed", + 'name': '', + 'actions': [], + } + + def task_generate_posts(self): + """Generate post files for the blog entries.""" + def gen_id(entry): + h = hashlib.md5() + h.update(entry.feed.name.encode('utf8')) + h.update(entry.guid) + return h.hexdigest() + + def generate_post(entry): + unique_id = gen_id(entry) + meta_path = os.path.join('posts', unique_id + '.meta') + post_path = os.path.join('posts', unique_id + '.txt') + with codecs.open(meta_path, 'wb+', 'utf8') as fd: + fd.write('%s\n' % entry.title.replace('\n', ' ')) + fd.write('%s\n' % unique_id) + fd.write('%s\n' % entry.date.strftime('%Y/%m/%d %H:%M')) + fd.write('\n') + fd.write('%s\n' % entry.link) + with codecs.open(post_path, 'wb+', 'utf8') as fd: + fd.write('.. raw:: html\n\n') + content = entry.content + if not content: + content = 'Sin contenido' + for line in content.splitlines(): + fd.write(' %s\n' % line) + + if not os.path.isdir('posts'): + os.mkdir('posts') + flag = False + for entry in Entry.select().order_by(Entry.date.desc()): + flag = True + entry_id = gen_id(entry) + yield { + 'basename': self.name + "_generate_posts", + 'targets': [os.path.join('posts', entry_id + '.meta'), os.path.join('posts', entry_id + '.txt')], + 'name': entry_id, + 'actions': [(generate_post, (entry,))], + 'uptodate': [config_changed({1: entry})], + 'task_dep': [self.name + "_fetch_feed"], + } + if not flag: + yield { + 'basename': self.name + "_generate_posts", + 'name': '', + 'actions': [], + } diff --git a/nikola/plugins/compile_bbcode.py b/nikola/plugins/compile_bbcode.py index 26de727..f8022f3 100644 --- a/nikola/plugins/compile_bbcode.py +++ b/nikola/plugins/compile_bbcode.py @@ -60,19 +60,17 @@ class CompileTextile(PageCompiler): output = self.parser.format(data) out_file.write(output) - def create_post(self, path, onefile=False, title="", slug="", date="", - tags=""): + def create_post(self, path, onefile=False, **kw): + metadata = {} + metadata.update(self.default_metadata) + metadata.update(kw) d_name = os.path.dirname(path) if not os.path.isdir(d_name): os.makedirs(os.path.dirname(path)) with codecs.open(path, "wb+", "utf8") as fd: if onefile: fd.write('[note][/note]\n\n') - fd.write("\nWrite your post here.") + fd.write("Write your post here.") diff --git a/nikola/plugins/compile_html.py b/nikola/plugins/compile_html.py index 6c1c381..7551b33 100644 --- a/nikola/plugins/compile_html.py +++ b/nikola/plugins/compile_html.py @@ -43,19 +43,17 @@ class CompileHtml(PageCompiler): pass shutil.copyfile(source, dest) - def create_post(self, path, onefile=False, title="", slug="", - date="", tags=""): + def create_post(self, path, onefile=False, **kw): + metadata = {} + metadata.update(self.default_metadata) + metadata.update(kw) d_name = os.path.dirname(path) if not os.path.isdir(d_name): os.makedirs(os.path.dirname(path)) with codecs.open(path, "wb+", "utf8") as fd: if onefile: fd.write('\n\n') fd.write("\n

Write your post here.

") diff --git a/nikola/plugins/compile_ipynb.plugin b/nikola/plugins/compile_ipynb.plugin new file mode 100644 index 0000000..51051e0 --- /dev/null +++ b/nikola/plugins/compile_ipynb.plugin @@ -0,0 +1,10 @@ +[Core] +Name = ipynb +Module = compile_ipynb + +[Documentation] +Author = Damián Avila +Version = 0.1 +Website = http://www.oquanta.info +Description = Compile IPython notebooks into HTML + diff --git a/nikola/plugins/compile_ipynb/README.txt b/nikola/plugins/compile_ipynb/README.txt new file mode 100644 index 0000000..2cfd45e --- /dev/null +++ b/nikola/plugins/compile_ipynb/README.txt @@ -0,0 +1,35 @@ +To make this work... + +1- First, you have to put this plugin in your_site/plugins/ folder. + +2- Then, you have to download the custom nbconvert from here: https://github.com/damianavila/compile_ipynb-for-Nikola.git +and put it inside your_site/plugins/compile_ipynb/ folder + +3- Also, you have to use the site-ipython theme (or make a new one containing the ipython css, mathjax.js and the proper template). +You can get it here: https://github.com/damianavila/site-ipython-theme-for-Nikola + +4- Finally, you have to put: + +post_pages = ( + ("posts/*.ipynb", "posts", "post.tmpl", True), + ("stories/*.ipynb", "stories", "story.tmpl", False), +) + +in your conf.py + +Then... to use it: + +$nikola new_page -f ipynb + +**NOTE**: Just IGNORE the "-1" and "-2" options in nikola new_page command, by default this compiler +create one metadata file and the corresponding naive IPython notebook. + +$nikola build + +And deploy the output folder... to see it locally: $nikola serve + +If you have any doubts, just ask: @damianavila + +Cheers. + +Damián diff --git a/nikola/plugins/compile_ipynb/__init__.py b/nikola/plugins/compile_ipynb/__init__.py new file mode 100644 index 0000000..d38f6f2 --- /dev/null +++ b/nikola/plugins/compile_ipynb/__init__.py @@ -0,0 +1,100 @@ +# Copyright (c) 2013 Damian Avila. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Implementation of compile_html based on nbconvert.""" + +from __future__ import unicode_literals, print_function +import codecs +import os + +try: + from .nbformat import current as nbformat + from .nbconvert.converters import bloggerhtml as nbconverter + bloggerhtml = True +except ImportError: + bloggerhtml = None + +from nikola.plugin_categories import PageCompiler + + +class CompileIPynb(PageCompiler): + """Compile IPynb into HTML.""" + + name = "ipynb" + + def compile_html(self, source, dest): + if bloggerhtml is None: + raise Exception('To build this site, you also need ' + 'https://github.com/damianavila/com' + 'pile_ipynb-for-Nikola.git.') + try: + os.makedirs(os.path.dirname(dest)) + except: + pass + converter = nbconverter.ConverterBloggerHTML() + with codecs.open(dest, "w+", "utf8") as out_file: + with codecs.open(source, "r", "utf8") as in_file: + data = in_file.read() + converter.nb = nbformat.reads_json(data) + output = converter.convert() + out_file.write(output) + + def create_post(self, path, onefile=False, **kw): + metadata = {} + metadata.update(self.default_metadata) + metadata.update(kw) + d_name = os.path.dirname(path) + if not os.path.isdir(d_name): + os.makedirs(os.path.dirname(path)) + meta_path = os.path.join(d_name, kw['slug'] + ".meta") + with codecs.open(meta_path, "wb+", "utf8") as fd: + if onefile: + fd.write('%s\n' % kw['title']) + fd.write('%s\n' % kw['slug']) + fd.write('%s\n' % kw['date']) + fd.write('%s\n' % kw['tags']) + print("Your post's metadata is at: ", meta_path) + with codecs.open(path, "wb+", "utf8") as fd: + fd.write("""{ + "metadata": { + "name": "%s" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +}""" % kw['slug']) diff --git a/nikola/plugins/compile_markdown/__init__.py b/nikola/plugins/compile_markdown/__init__.py index 7aa03a9..ae700e6 100644 --- a/nikola/plugins/compile_markdown/__init__.py +++ b/nikola/plugins/compile_markdown/__init__.py @@ -24,14 +24,28 @@ """Implementation of compile_html based on markdown.""" +from __future__ import unicode_literals + import codecs import os -import re try: from markdown import markdown + + from nikola.plugins.compile_markdown.mdx_nikola import NikolaExtension + nikola_extension = NikolaExtension() + + from nikola.plugins.compile_markdown.mdx_gist import GistExtension + gist_extension = GistExtension() + + from nikola.plugins.compile_markdown.mdx_podcast import PodcastExtension + podcast_extension = PodcastExtension() + except ImportError: markdown = None # NOQA + nikola_extension = None + gist_extension = None + podcast_extension = None from nikola.plugin_categories import PageCompiler @@ -41,6 +55,9 @@ class CompileMarkdown(PageCompiler): name = "markdown" + extensions = ['fenced_code', 'codehilite', gist_extension, + nikola_extension, podcast_extension] + def compile_html(self, source, dest): if markdown is None: raise Exception('To build this site, you need to install the ' @@ -52,30 +69,20 @@ class CompileMarkdown(PageCompiler): with codecs.open(dest, "w+", "utf8") as out_file: with codecs.open(source, "r", "utf8") as in_file: data = in_file.read() - output = markdown(data, ['fenced_code', 'codehilite']) - # h1 is reserved for the title so increment all header levels - for n in reversed(range(1, 9)): - output = re.sub(''.format(n), ''.format(n + 1), - output) - output = re.sub(''.format(n), ''.format(n + 1), - output) - # python-markdown's highlighter uses the class 'codehilite' to wrap - # code, # instead of the standard 'code'. None of the standard - # pygments stylesheets use this class, so swap it to be 'code' - output = re.sub(r'(]+class="[^"]*)codehilite([^>]+)', - r'\1code\2', output) + output = markdown(data, self.extensions) out_file.write(output) - def create_post(self, path, onefile=False, title="", slug="", date="", - tags=""): + def create_post(self, path, onefile=False, **kw): + metadata = {} + metadata.update(self.default_metadata) + metadata.update(kw) + d_name = os.path.dirname(path) + if not os.path.isdir(d_name): + os.makedirs(os.path.dirname(path)) with codecs.open(path, "wb+", "utf8") as fd: if onefile: fd.write('\n\n') - fd.write("\nWrite your post here.") + fd.write("Write your post here.") diff --git a/nikola/plugins/compile_markdown/mdx_gist.py b/nikola/plugins/compile_markdown/mdx_gist.py new file mode 100644 index 0000000..808e383 --- /dev/null +++ b/nikola/plugins/compile_markdown/mdx_gist.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2013 Michael Rabbitt. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# Inspired by "[Python] reStructuredText GitHub Gist directive" +# (https://gist.github.com/brianhsu/1407759), public domain by Brian Hsu + +from __future__ import print_function + + +''' +Extension to Python Markdown for Embedded Gists (gist.github.com) + +Basic Example: + + >>> import markdown + >>> text = """ + ... Text of the gist: + ... [:gist: 4747847] + ... """ + >>> html = markdown.markdown(text, [GistExtension()]) + >>> print(html) +

Text of the gist: +

+ + +
+

+ +Example with filename: + + >>> import markdown + >>> text = """ + ... Text of the gist: + ... [:gist: 4747847 zen.py] + ... """ + >>> html = markdown.markdown(text, [GistExtension()]) + >>> print(html) +

Text of the gist: +

+ + +
+

+ +Example using reStructuredText syntax: + + >>> import markdown + >>> text = """ + ... Text of the gist: + ... .. gist:: 4747847 zen.py + ... """ + >>> html = markdown.markdown(text, [GistExtension()]) + >>> print(html) +

Text of the gist: +

+ + +
+

+''' +from __future__ import unicode_literals +import warnings +from markdown.extensions import Extension +from markdown.inlinepatterns import Pattern +from markdown.util import AtomicString +from markdown.util import etree + +try: + import requests +except ImportError: + requests = None # NOQA + +GIST_JS_URL = "https://gist.github.com/{0}.js" +GIST_FILE_JS_URL = "https://gist.github.com/{0}.js?file={1}" +GIST_RAW_URL = "https://raw.github.com/gist/{0}" +GIST_FILE_RAW_URL = "https://raw.github.com/gist/{0}/{1}" + +GIST_MD_RE = r'\[:gist:\s*(?P\d+)(?:\s*(?P.+?))?\]' +GIST_RST_RE = r'(?m)^\.\.\s*gist::\s*(?P\d+)(?:\s*(?P.+))\s*$' + + +class GistPattern(Pattern): + """ InlinePattern for footnote markers in a document's body text. """ + + def __init__(self, pattern, configs): + Pattern.__init__(self, pattern) + + def get_raw_gist_with_filename(self, gist_id, filename): + url = GIST_FILE_RAW_URL.format(gist_id, filename) + return requests.get(url).text + + def get_raw_gist(self, gist_id): + url = GIST_RAW_URL.format(gist_id) + return requests.get(url).text + + def handleMatch(self, m): + gist_id = m.group('gist_id') + gist_file = m.group('filename') + + gist_elem = etree.Element('div') + gist_elem.set('class', 'gist') + script_elem = etree.SubElement(gist_elem, 'script') + + if gist_file: + script_elem.set('src', GIST_FILE_JS_URL.format( + gist_id, gist_file)) + + else: + script_elem.set('src', GIST_JS_URL.format( + gist_id)) + + if requests: + if gist_file: + raw_gist = (self.get_raw_gist_with_filename( + gist_id, gist_file)) + script_elem.set('src', GIST_FILE_JS_URL.format( + gist_id, gist_file)) + + else: + raw_gist = (self.get_raw_gist(gist_id)) + script_elem.set('src', GIST_JS_URL.format( + gist_id)) + + # Insert source as 
 within