summaryrefslogtreecommitdiffstats
path: root/nikola/plugins/command/planetoid/__init__.py
diff options
context:
space:
mode:
Diffstat (limited to 'nikola/plugins/command/planetoid/__init__.py')
-rw-r--r--nikola/plugins/command/planetoid/__init__.py289
1 files changed, 0 insertions, 289 deletions
diff --git a/nikola/plugins/command/planetoid/__init__.py b/nikola/plugins/command/planetoid/__init__.py
deleted file mode 100644
index fe1a59b..0000000
--- a/nikola/plugins/command/planetoid/__init__.py
+++ /dev/null
@@ -1,289 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright © 2012-2014 Roberto Alsina and others.
-
-# Permission is hereby granted, free of charge, to any
-# person obtaining a copy of this software and associated
-# documentation files (the "Software"), to deal in the
-# Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the
-# Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice
-# shall be included in all copies or substantial portions of
-# the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
-# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
-# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
-# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-from __future__ import print_function, unicode_literals
-import codecs
-import datetime
-import hashlib
-from optparse import OptionParser
-import os
-import sys
-
-from doit.tools import timeout
-from nikola.plugin_categories import Command, Task
-from nikola.utils import config_changed, req_missing, get_logger, STDERR_HANDLER
-
-LOGGER = get_logger('planetoid', STDERR_HANDLER)
-
-try:
- import feedparser
-except ImportError:
- feedparser = None # NOQA
-
-try:
- import peewee
-except ImportError:
- peewee = None
-
-
-if peewee is not None:
- class Feed(peewee.Model):
- name = peewee.CharField()
- url = peewee.CharField(max_length=200)
- last_status = peewee.CharField(null=True)
- etag = peewee.CharField(max_length=200)
- last_modified = peewee.DateTimeField()
-
- class Entry(peewee.Model):
- date = peewee.DateTimeField()
- feed = peewee.ForeignKeyField(Feed)
- content = peewee.TextField(max_length=20000)
- link = peewee.CharField(max_length=200)
- title = peewee.CharField(max_length=200)
- guid = peewee.CharField(max_length=200)
-
-
-class Planetoid(Command, Task):
- """Maintain a planet-like thing."""
- name = "planetoid"
-
- def init_db(self):
- # setup database
- Feed.create_table(fail_silently=True)
- Entry.create_table(fail_silently=True)
-
- def gen_tasks(self):
- if peewee is None or sys.version_info[0] == 3:
- if sys.version_info[0] == 3:
- message = 'Peewee, a requirement of the "planetoid" command, is currently incompatible with Python 3.'
- else:
- req_missing('peewee', 'use the "planetoid" command')
- message = ''
- yield {
- 'basename': self.name,
- 'name': '',
- 'verbosity': 2,
- 'actions': ['echo "%s"' % message]
- }
- else:
- self.init_db()
- self.load_feeds()
- for task in self.task_update_feeds():
- yield task
- for task in self.task_generate_posts():
- yield task
- yield {
- 'basename': self.name,
- 'name': '',
- 'actions': [],
- 'file_dep': ['feeds'],
- 'task_dep': [
- self.name + "_fetch_feed",
- self.name + "_generate_posts",
- ]
- }
-
- def run(self, *args):
- parser = OptionParser(usage="nikola %s [options]" % self.name)
- (options, args) = parser.parse_args(list(args))
-
- def load_feeds(self):
- "Read the feeds file, add it to the database."
- feeds = []
- feed = name = None
- for line in codecs.open('feeds', 'r', 'utf-8'):
- line = line.strip()
- if line.startswith("#"):
- continue
- elif line.startswith('http'):
- feed = line
- elif line:
- name = line
- if feed and name:
- feeds.append([feed, name])
- feed = name = None
-
- def add_feed(name, url):
- f = Feed.create(
- name=name,
- url=url,
- etag='foo',
- last_modified=datetime.datetime(1970, 1, 1),
- )
- f.save()
-
- def update_feed_url(feed, url):
- feed.url = url
- feed.save()
-
- for feed, name in feeds:
- f = Feed.select().where(Feed.name == name)
- if not list(f):
- add_feed(name, feed)
- elif list(f)[0].url != feed:
- update_feed_url(list(f)[0], feed)
-
- def task_update_feeds(self):
- """Download feed contents, add entries to the database."""
- def update_feed(feed):
- modified = feed.last_modified.timetuple()
- etag = feed.etag
- try:
- parsed = feedparser.parse(
- feed.url,
- etag=etag,
- modified=modified
- )
- feed.last_status = str(parsed.status)
- except: # Probably a timeout
- # TODO: log failure
- return
- if parsed.feed.get('title'):
- LOGGER.info(parsed.feed.title)
- else:
- LOGGER.info(feed.url)
- feed.etag = parsed.get('etag', 'foo')
- modified = tuple(parsed.get('date_parsed', (1970, 1, 1)))[:6]
- LOGGER.info("==========>", modified)
- modified = datetime.datetime(*modified)
- feed.last_modified = modified
- feed.save()
- # No point in adding items from missinfg feeds
- if parsed.status > 400:
- # TODO log failure
- return
- for entry_data in parsed.entries:
- LOGGER.info("=========================================")
- date = entry_data.get('published_parsed', None)
- if date is None:
- date = entry_data.get('updated_parsed', None)
- if date is None:
- LOGGER.error("Can't parse date from:\n", entry_data)
- return False
- LOGGER.info("DATE:===>", date)
- date = datetime.datetime(*(date[:6]))
- title = "%s: %s" % (feed.name, entry_data.get('title', 'Sin título'))
- content = entry_data.get('content', None)
- if content:
- content = content[0].value
- if not content:
- content = entry_data.get('description', None)
- if not content:
- content = entry_data.get('summary', 'Sin contenido')
- guid = str(entry_data.get('guid', entry_data.link))
- link = entry_data.link
- LOGGER.info(repr([date, title]))
- e = list(Entry.select().where(Entry.guid == guid))
- LOGGER.info(
- repr(dict(
- date=date,
- title=title,
- content=content,
- guid=guid,
- feed=feed,
- link=link,
- ))
- )
- if not e:
- entry = Entry.create(
- date=date,
- title=title,
- content=content,
- guid=guid,
- feed=feed,
- link=link,
- )
- else:
- entry = e[0]
- entry.date = date
- entry.title = title
- entry.content = content
- entry.link = link
- entry.save()
- flag = False
- for feed in Feed.select():
- flag = True
- task = {
- 'basename': self.name + "_fetch_feed",
- 'name': str(feed.url),
- 'actions': [(update_feed, (feed, ))],
- 'uptodate': [timeout(datetime.timedelta(minutes=
- self.site.config.get('PLANETOID_REFRESH', 60)))],
- }
- yield task
- if not flag:
- yield {
- 'basename': self.name + "_fetch_feed",
- 'name': '',
- 'actions': [],
- }
-
- def task_generate_posts(self):
- """Generate post files for the blog entries."""
- def gen_id(entry):
- h = hashlib.md5()
- h.update(entry.feed.name.encode('utf8'))
- h.update(entry.guid)
- return h.hexdigest()
-
- def generate_post(entry):
- unique_id = gen_id(entry)
- meta_path = os.path.join('posts', unique_id + '.meta')
- post_path = os.path.join('posts', unique_id + '.txt')
- with codecs.open(meta_path, 'wb+', 'utf8') as fd:
- fd.write('%s\n' % entry.title.replace('\n', ' '))
- fd.write('%s\n' % unique_id)
- fd.write('%s\n' % entry.date.strftime('%Y/%m/%d %H:%M'))
- fd.write('\n')
- fd.write('%s\n' % entry.link)
- with codecs.open(post_path, 'wb+', 'utf8') as fd:
- fd.write('.. raw:: html\n\n')
- content = entry.content
- if not content:
- content = 'Sin contenido'
- for line in content.splitlines():
- fd.write(' %s\n' % line)
-
- if not os.path.isdir('posts'):
- os.mkdir('posts')
- flag = False
- for entry in Entry.select().order_by(Entry.date.desc()):
- flag = True
- entry_id = gen_id(entry)
- yield {
- 'basename': self.name + "_generate_posts",
- 'targets': [os.path.join('posts', entry_id + '.meta'), os.path.join('posts', entry_id + '.txt')],
- 'name': entry_id,
- 'actions': [(generate_post, (entry,))],
- 'uptodate': [config_changed({1: entry})],
- 'task_dep': [self.name + "_fetch_feed"],
- }
- if not flag:
- yield {
- 'basename': self.name + "_generate_posts",
- 'name': '',
- 'actions': [],
- }
generated by cgit v1.2.3 (git 2.46.0) at 2026-01-26 21:14:32 +0000