#!/usr/bin/env  python

__license__   = 'GPL v3'

'''
online.wsj.com
'''
import re
from calibre.web.feeds.recipes import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import Tag, NavigableString
from datetime import timedelta, date

class WSJ(BasicNewsRecipe):
    # formatting adapted from original recipe by Kovid Goyal and Sujata Raman
    title          = u'Wall Street Journal (free)'
    __author__     = 'Nick Redding'
    language = 'en'
    description = ('All the free content from the Wall Street Journal (business, financial and political news)')

    no_stylesheets = True
    timefmt = ' [%b %d]'

    # customization notes: delete sections you are not interested in
    # set omit_paid_content to False if you want the paid content article snippets
    # set oldest_article to the maximum number of days back from today to include articles
    sectionlist = [
                        ['/home-page','Front Page'],
                        ['/public/page/news-opinion-commentary.html','Commentary'],
                        ['/public/page/news-global-world.html','World News'],
                        ['/public/page/news-world-business.html','US News'],
                        ['/public/page/news-business-us.html','Business'],
                        ['/public/page/news-financial-markets-stock.html','Markets'],
                        ['/public/page/news-tech-technology.html','Technology'],
                        ['/public/page/news-personal-finance.html','Personal Finnce'],
                        ['/public/page/news-lifestyle-arts-entertainment.html','Life & Style'],
                        ['/public/page/news-real-estate-homes.html','Real Estate'],
                        ['/public/page/news-career-jobs.html','Careers'],
                        ['/public/page/news-small-business-marketing.html','Small Business']
                    ]
    oldest_article = 2
    omit_paid_content = True

    extra_css   = '''h1{font-size:large; font-family:Times,serif;}
                    h2{font-family:Times,serif; font-size:small; font-style:italic;}
                    .subhead{font-family:Times,serif; font-size:small; font-style:italic;}
                    .insettipUnit {font-family:Times,serif;font-size:xx-small;}
                    .targetCaption{font-size:x-small; font-family:Times,serif; font-style:italic; margin-top: 0.25em;}
                    .article{font-family:Times,serif; font-size:x-small;}
                    .tagline { font-size:xx-small;}
                    .dateStamp {font-family:Times,serif;}
                    h3{font-family:Times,serif; font-size:xx-small;}
                    .byline {font-family:Times,serif; font-size:xx-small; list-style-type: none;}
                    .metadataType-articleCredits {list-style-type: none;}
                    h6{font-family:Times,serif; font-size:small; font-style:italic;}
                    .paperLocation{font-size:xx-small;}'''


    remove_tags_before = dict({'class':re.compile('^articleHeadlineBox')})
    remove_tags =   [   dict({'id':re.compile('^articleTabs_tab_')}),
                        #dict(id=["articleTabs_tab_article", "articleTabs_tab_comments",
                        #         "articleTabs_tab_interactive","articleTabs_tab_video",
                        #         "articleTabs_tab_map","articleTabs_tab_slideshow"]),
			{'class':  ['footer_columns','network','insetCol3wide','interactive','video','slideshow','map',
                                    'insettip','insetClose','more_in', "insetContent",
                        #            'articleTools_bottom','articleTools_bottom mjArticleTools',
                                    'aTools', 'tooltip',
                                    'adSummary', 'nav-inline','insetFullBracket']},
                        dict({'class':re.compile('^articleTools_bottom')}),
                        dict(rel='shortcut icon')
                    ]
    remove_tags_after = [dict(id="article_story_body"), {'class':"article story"}]

    def get_browser(self):
        br = BasicNewsRecipe.get_browser()
        return br


    def preprocess_html(self,soup):

        def decode_us_date(datestr):
            udate = datestr.strip().lower().split()
            m = ['january','february','march','april','may','june','july','august','september','october','november','december'].index(udate[0])+1
            d = int(udate[1])
            y = int(udate[2])
            return date(y,m,d)

        # check if article is paid content
        if self.omit_paid_content:
            divtags = soup.findAll('div','tooltip')
            if divtags:
                for divtag in divtags:
                    if divtag.find(text="Subscriber Content"):
                        return None

        # check if article is too old
        datetag = soup.find('li',attrs={'class' : re.compile("^dateStamp")})
        if datetag:
            dateline_string = self.tag_to_string(datetag,False)
            date_items = dateline_string.split(',')
            datestring = date_items[0]+date_items[1]
            article_date = decode_us_date(datestring)
            earliest_date = date.today() - timedelta(days=self.oldest_article)
            if article_date < earliest_date:
                self.log("Skipping article dated %s" % datestring)
                return None
            datetag.parent.extract()

            # place dateline in article heading

            bylinetag = soup.find('h3','byline')
            if bylinetag:
                h3bylinetag = bylinetag
            else:
                bylinetag = soup.find('li','byline')
                if bylinetag:
                    h3bylinetag = bylinetag.h3
                    if not h3bylinetag:
                        h3bylinetag = bylinetag
                    bylinetag = bylinetag.parent
            if bylinetag:
                if h3bylinetag.a:
                    bylinetext = 'By '+self.tag_to_string(h3bylinetag.a,False)
                else:
                    bylinetext = self.tag_to_string(h3bylinetag,False)
                h3byline = Tag(soup,'h3',[('class','byline')])
                if bylinetext.isspace() or (bylinetext == ''):
                    h3byline.insert(0,NavigableString(date_items[0]+','+date_items[1]))
                else:
                    h3byline.insert(0,NavigableString(bylinetext+u'\u2014'+date_items[0]+','+date_items[1]))
                bylinetag.replaceWith(h3byline)
            else:
                headlinetag = soup.find('div',attrs={'class' : re.compile("^articleHeadlineBox")})
                if headlinetag:
                    dateline = Tag(soup,'h3', [('class','byline')])
                    dateline.insert(0,NavigableString(date_items[0]+','+date_items[1]))
                    headlinetag.insert(len(headlinetag),dateline)
        else: # if no date tag, don't process this page--it's not a news item
            return None
        # This gets rid of the annoying superfluous bullet symbol preceding columnist bylines
        ultag = soup.find('ul',attrs={'class' : 'cMetadata metadataType-articleCredits'})
        if ultag:
            a = ultag.h3
            if a:
                ultag.replaceWith(a)
        return soup

    def parse_index(self):

        articles = {}
        key = None
        ans = []

        def parse_index_page(page_name,page_title):

            def article_title(tag):
                atag = tag.find('h2') # title is usually in an h2 tag
                if not atag: # if not, get text from the a tag
                    atag = tag.find('a',href=True)
                    if not atag:
                        return ''
                    t = self.tag_to_string(atag,False)
                    if t == '':
                        # sometimes the title is in the second a tag
                        atag.extract()
                        atag = tag.find('a',href=True)
                        if not atag:
                            return ''
                        return self.tag_to_string(atag,False)
                    return t
                return self.tag_to_string(atag,False)

            def article_author(tag):
                atag = tag.find('strong') # author is usually in a strong tag
                if not atag:
                     atag = tag.find('h4') # if not, look for an h4 tag
                     if not atag:
                         return ''
                return self.tag_to_string(atag,False)

            def article_summary(tag):
                atag = tag.find('p')
                if not atag:
                    return ''
                subtag = atag.strong
                if subtag:
                    subtag.extract()
                return self.tag_to_string(atag,False)

            def article_url(tag):
                atag = tag.find('a',href=True)
                if not atag:
                    return ''
                url = re.sub(r'\?.*', '', atag['href'])
                return url

            def handle_section_name(tag):
                # turns a tag into a section name with special processing
                # for Wat's News, U.S., World & U.S. and World
                s = self.tag_to_string(tag,False)
                if ("What" in s) and ("News" in s):
                    s = "What's News"
                elif (s == "U.S.") or (s == "World & U.S.") or (s == "World"):
                    s = s + " News"
                return s



            mainurl = 'http://online.wsj.com'
            pageurl = mainurl+page_name
            #self.log("Page url %s" % pageurl)
            soup = self.index_to_soup(pageurl)
            # Find each instance of div with class including "headlineSummary"
            for divtag in soup.findAll('div',attrs={'class' : re.compile("^headlineSummary")}):
                # divtag contains all article data as ul's and li's
                # first, check if there is an h3 tag which provides a section name
                stag = divtag.find('h3')
                if stag:
                    if stag.parent.get('class', '') == 'dynamic':
                        # a carousel of articles is too complex to extract a section name
                        # for each article, so we'll just call the section "Carousel"
                        section_name = 'Carousel'
                    else:
                        section_name = handle_section_name(stag)
                else:
                    section_name = "What's News"
                #self.log("div Section %s" % section_name)
                # find each top-level ul in the div
                # we don't restrict to class = newsItem because the section_name
                # sometimes changes via a ul tag inside the div
                for ultag in divtag.findAll('ul',recursive=False):
                    stag = ultag.find('h3')
                    if stag:
                        if stag.parent.name == 'ul':
                            # section name has changed
                            section_name = handle_section_name(stag)
                            #self.log("ul Section %s" % section_name)
                            # delete the h3 tag so it doesn't get in the way
                            stag.extract()
                    # find each top level li in the ul
                    for litag in ultag.findAll('li',recursive=False):
                        stag = litag.find('h3')
                        if stag:
                            # section name has changed
                            section_name = handle_section_name(stag)
                            #self.log("li Section %s" % section_name)
                            # delete the h3 tag so it doesn't get in the way
                            stag.extract()
                        # if there is a ul tag inside the li it is superfluous;
                        # it is probably a list of related articles
                        utag = litag.find('ul')
                        if utag:
                            utag.extract()
                        # now skip paid subscriber articles if desired
                        subscriber_tag = litag.find(text="Subscriber Content")
                        if subscriber_tag:
                                if self.omit_paid_content:
                                    continue
                                # delete the tip div so it doesn't get in the way
                                tiptag = litag.find("div", { "class" : "tipTargetBox" })
                                if tiptag:
                                    tiptag.extract()
                        h1tag = litag.h1
                        # if there's an h1 tag, it's parent is a div which should replace
                        # the li tag for the analysis
                        if h1tag:
                            litag = h1tag.parent
                        h5tag = litag.h5
                        if h5tag:
                            # section mame has changed
                            section_name = self.tag_to_string(h5tag,False)
                            #self.log("h5 Section %s" % section_name)
                            # delete the h5 tag so it doesn't get in the way
                            h5tag.extract()
                        url = article_url(litag)
                        if url == '':
                            continue
                        if url.startswith("/article"):
                            url = mainurl+url
                        if not url.startswith("http://online.wsj.com"):
                            continue
                        if not url.endswith(".html"):
                            continue
                        if 'video' in url:
                            continue
                        title = article_title(litag)
                        if title == '':
                            continue
                        #self.log("URL %s" % url)
                        #self.log("Title %s" % title)
                        pubdate = ''
                        #self.log("Date %s" % pubdate)
                        author = article_author(litag)
                        if author == '':
                            author = section_name
                        elif author == section_name:
                            author = ''
                        else:
                            author = section_name+': '+author
                        #if not author == '':
                        #    self.log("Author %s" % author)
                        description = article_summary(litag)
                        #if not description == '':
                        #    self.log("Description %s" % description)
                        if not articles.has_key(page_title):
                            articles[page_title] = []
                        articles[page_title].append(dict(title=title,url=url,date=pubdate,description=description,author=author,content=''))


        for page_name,page_title in self.sectionlist:
            parse_index_page(page_name,page_title)
            ans.append(page_title)

        ans = [(key, articles[key]) for key in ans if articles.has_key(key)]
        return ans
