From c818eb30b5d001b9ea519a7dba3f2d56f72c284e Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 1 Aug 2018 12:55:45 -0500 Subject: [PATCH] Fixes for encoding/make unicode issues. --- fanficfare/adapters/adapter_adultfanfictionorg.py | 3 +-- fanficfare/adapters/adapter_fireflyfansnet.py | 3 +-- fanficfare/adapters/adapter_lotrgficcom.py | 3 +-- fanficfare/adapters/adapter_wwwarea52hkhnet.py | 3 +-- fanficfare/adapters/adapter_wwwutopiastoriescom.py | 3 +-- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/fanficfare/adapters/adapter_adultfanfictionorg.py b/fanficfare/adapters/adapter_adultfanfictionorg.py index 8baf8273..4acb861b 100644 --- a/fanficfare/adapters/adapter_adultfanfictionorg.py +++ b/fanficfare/adapters/adapter_adultfanfictionorg.py @@ -30,7 +30,6 @@ from .. import exceptions as exceptions # py2 vs py3 transition from ..six import text_type as unicode -from ..six import ensure_text from ..six.moves.urllib.error import HTTPError from .base_adapter import BaseSiteAdapter, makeDate @@ -338,7 +337,7 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter): ##There is also a double
, so we have to fix that, then remove the leading and trailing '-:-'. ##They are always in the same order. ## EDIT 09/26/2016: Had some trouble with unicode errors... so I had to put in the decode/encode parts to fix it - liMetadata = ensure_text(lc2).replace('\n','').replace('\r','').replace('\t',' ').replace(' ',' ').replace(' ',' ').replace(' ',' ') + liMetadata = unicode(lc2).replace('\n','').replace('\r','').replace('\t',' ').replace(' ',' ').replace(' ',' ').replace(' ',' ') liMetadata = stripHTML(liMetadata.replace(r'
','-:-').replace('','-:-')) liMetadata = liMetadata.strip('-:-').strip('-:-').encode('utf-8') for i, value in enumerate(liMetadata.decode('utf-8').split('-:-')): diff --git a/fanficfare/adapters/adapter_fireflyfansnet.py b/fanficfare/adapters/adapter_fireflyfansnet.py index b5a6676a..a6abba78 100644 --- a/fanficfare/adapters/adapter_fireflyfansnet.py +++ b/fanficfare/adapters/adapter_fireflyfansnet.py @@ -24,7 +24,6 @@ import re import sys # py2 vs py3 transition from ..six import text_type as unicode -from ..six import ensure_text from ..six.moves.urllib.error import HTTPError from .base_adapter import BaseSiteAdapter, makeDate @@ -134,7 +133,7 @@ class FireFlyFansNetSiteAdapter(BaseSiteAdapter): # which is usualy FireFly on this site, but I'm going to get them # anyway.a category = soup.find('span', {'id': 'MainContent_txtItemDetails'}) - category = stripHTML(ensure_text(category).replace(b"\xc2\xa0", ' ')) + category = stripHTML(unicode(category).replace(u"\xc2\xa0", ' ')) metad = category.split(' ') for meta in metad: if ":" in meta: diff --git a/fanficfare/adapters/adapter_lotrgficcom.py b/fanficfare/adapters/adapter_lotrgficcom.py index 3ac0cc06..d1e32d81 100644 --- a/fanficfare/adapters/adapter_lotrgficcom.py +++ b/fanficfare/adapters/adapter_lotrgficcom.py @@ -28,7 +28,6 @@ from .. import exceptions as exceptions # py2 vs py3 transition from ..six import text_type as unicode -from ..six import ensure_text from ..six.moves.urllib.error import HTTPError from .base_adapter import BaseSiteAdapter, makeDate @@ -287,7 +286,7 @@ class LOTRgficComAdapter(BaseSiteAdapter): #
#

## we'll have to remove the non-breaking spaces to get this to work. - metad = ensure_text(metad).replace(b"\xc2\xa0",'').replace('\n','') + metad = unicode(metad).replace(u"\xc2\xa0",'').replace('\n','') for txt in metad.split('
'): if 'Challenges:' in txt: txt = txt.replace('Challenges:','').strip() diff --git a/fanficfare/adapters/adapter_wwwarea52hkhnet.py b/fanficfare/adapters/adapter_wwwarea52hkhnet.py index 8dce5df5..b49503f1 100644 --- a/fanficfare/adapters/adapter_wwwarea52hkhnet.py +++ b/fanficfare/adapters/adapter_wwwarea52hkhnet.py @@ -33,7 +33,6 @@ from .. import exceptions as exceptions # py2 vs py3 transition from ..six import text_type as unicode -from ..six import ensure_text from ..six.moves.urllib.error import HTTPError from .base_adapter import BaseSiteAdapter, makeDate @@ -192,7 +191,7 @@ class WWWArea52HKHNetAdapter(BaseSiteAdapter): ## I've seen a non-breaking space in some of the storyblocks ## so we are going to remove them. - series = stripHTML(ensure_text(series.renderContents()).replace(b"\xc2\xa0",'')).strip() + series = stripHTML(unicode(series.renderContents()).replace(u"\xc2\xa0",'')).strip() if len(series) > 0: self.story.setMetadata('series',series) diff --git a/fanficfare/adapters/adapter_wwwutopiastoriescom.py b/fanficfare/adapters/adapter_wwwutopiastoriescom.py index a62c8d8a..1f7882ef 100644 --- a/fanficfare/adapters/adapter_wwwutopiastoriescom.py +++ b/fanficfare/adapters/adapter_wwwutopiastoriescom.py @@ -41,7 +41,6 @@ from .. import exceptions as exceptions # py2 vs py3 transition from ..six import text_type as unicode -from ..six import ensure_text from ..six.moves.urllib.error import HTTPError from ..six.moves.urllib.parse import quote @@ -148,7 +147,7 @@ class WWWUtopiastoriesComAdapter(BaseSiteAdapter): for detail in soup.findAll('li'): - det = ensure_text(detail).replace(b"\xc2\xa0",'') + det = unicode(detail).replace(u"\xc2\xa0",'') heading = stripHTML(det).split(' - ')[0] text = stripHTML(det).replace(heading+' - ','') if 'Author' in heading: