Backup_Repos/leech
1
0
Fork 0
mirror of https://github.com/kemayo/leech synced 2026-05-08 12:34:37 +02:00
Code Issues Projects Releases Wiki Activity

Handle some threadmarks better

Browse source
This commit is contained in:
David Lynch 2015-08-14 01:03:04 -05:00
parent 43f0ec9fef
commit d8250fa7bf
1 changed files with 10 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
sites/spacebattles.py
View file

@ -36,13 +36,19 @@ def extract(url, fetch):
print("Extracting chapter from", href) print("Extracting chapter from", href)
match = re.match(r'posts/(\d+)/?', href) match = re.match(r'posts/(\d+)/?', href)
if not match: if not match:
print("Unparseable threadmark href", href) match = re.match(r'.+#post-(\d+)$', href)
return if not match:
postid = match.group(1) print("Unparseable threadmark href", href)
chapter_postid = match and match.group(1)
chapter_page = fetch(base + href) chapter_page = fetch(base + href)
chapter_soup = BeautifulSoup(chapter_page, 'html5lib') chapter_soup = BeautifulSoup(chapter_page, 'html5lib')
post = chapter_soup.find('li', id='post-'+postid).find('blockquote', class_='messageText') if chapter_postid:
post = chapter_soup.find('li', id='post-'+chapter_postid)
else:
# just the first one in the thread, then
post = chapter_soup.find('li', class_='message')
post = post.find('blockquote', class_='messageText')
post.name = 'div' post.name = 'div'
chapters.append((str(mark.a.string), post.prettify())) chapters.append((str(mark.a.string), post.prettify()))