# -*- coding: utf-8 -*- import os import re import sys import cgi import uuid import shutil import base64 import os.path import logging import unittest import urllib as u import pprint as pp import urllib2 as u2 import login_password import urlparse as up import BeautifulSoup as bs import htmlentitydefs as hdefs from constants import * class FFA: def __init__(self, url): self.url = url parsedUrl = up.urlparse(url) self.host = parsedUrl.netloc self.path = parsedUrl.path self.opener = u2.build_opener(u2.HTTPCookieProcessor()) logging.debug("Created FFA: url=%s" % (self.url)) def _getLoginScript(self): return self.path def requiresLogin(self, url = None): resp = self.opener.open(self.url) data = resp.read() if data.find('') != -1: return True else: return False def performLogin(self, url = None): if url == None: url = self.url data = {} data['username'] = self.login data['password'] = self.password data['submit'] = 'Submit' urlvals = u.urlencode(data) loginUrl = 'http://' + self.host + self._getLoginScript() logging.debug("Will now login to URL %s" % loginUrl) req = self.opener.open(loginUrl, urlvals) if self.requiresLogin(): return False else: return True def extractIndividualUrls(self): data = self.opener.open(self.url).read() soup = bs.BeautifulStoneSoup(data) self.author = soup.find('a', {'href' : '/contact/'}).string self.storyName = str(soup.find('h1', {'class' : 'textCenter'}).contents[0]).strip() logging.debug("Story `%s` by `%s`" % (self.storyName, self.author)) selector = soup.find('select', {'class' : 'tinput'}) options = selector.findAll('option') urls = [] for o in options: title = o.string url = o['value'] urls.append((url,title)) return urls def getText(self, url): if url.find('http://') == -1: url = 'http://' + self.host + '/' + url data = self.opener.open(url).read() lines = data.split('\n') emit = False post = '' for l in lines: if l.find('') != -1: logging.debug('emit = True') emit = True continue elif l.find('