#!/usr/bin/python import datetime import re from . import register, Site, SiteException @register class FanFictionNet(Site): """FFN: it has a lot of stuff""" @staticmethod def matches(url): # e.g. https://www.fanfiction.net/s/4109686/3/Taking-Sights return re.match(r'^https?://www\.fanfiction\.net/s/\d+/?.*', url) def extract(self, url): soup = self._soup(url) content = soup.find(id="content_wrapper_inner") if not content: raise SiteException("No content") story = {} chapters = [] metadata = content.find(id='profile_top') story['title'] = str(metadata.find('b', class_="xcontrast_txt").string) story['author'] = str(metadata.find('a', class_="xcontrast_txt").string) dates = content.find_all('span', attrs={'data-xutime': True}) published = False updated = False if len(dates) == 1: published = datetime.datetime.fromtimestamp(int(dates[0]['data-xutime'])) elif len(dates) == 2: updated = datetime.datetime.fromtimestamp(int(dates[0]['data-xutime'])) published = datetime.datetime.fromtimestamp(int(dates[1]['data-xutime'])) chapter_select = content.find(id="chap_select") if chapter_select: base_url = re.search(r'(https?://[^/]+/s/\d+/?)', url) if not base_url: raise SiteException("Can't find base URL for chapters") base_url = base_url.group(0) # beautiful soup doesn't handle ffn's unclosed option tags at all well here options = re.findall(r']*>([^<]+)', str(chapter_select)) for option in options: chapters.append((option[1], self._chapter(base_url + option[0]), False)) chapters[-1] = (chapters[-1][0], chapters[-1][1], updated) chapters[0] = (chapters[0][0], chapters[0][1], published) else: chapters.append((story['title'], self._extract_chapter(url), published)) story['chapters'] = chapters return story def _chapter(self, url): print("Extracting chapter from", url) soup = self._soup(url) content = soup.find(id="content_wrapper_inner") if not content: raise SiteException("No chapter content") text = content.find(id="storytext") # clean up some invalid xhtml attributes # TODO: be more selective about this somehow try: for tag in text.find_all(True): tag.attrs = None except Exception as e: print("Trouble cleaning attributes", e) return text.prettify()