Backup_Repos/leech
1
0
Fork 0
mirror of https://github.com/kemayo/leech synced 2025-12-06 08:22:56 +01:00
Code Issues Projects Releases Wiki Activity

process all next_link items

Browse source
This commit is contained in:
Kevin Pedro 2025-03-04 22:13:20 -06:00
parent 53bc2045f0
commit 0066a148bb
1 changed files with 10 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
sites/arbitrary.py
View file

@ -77,7 +77,9 @@ class Arbitrary(Site):
# set of already processed urls. Stored to detect loops. # set of already processed urls. Stored to detect loops.
found_content_urls = set() found_content_urls = set()
content_url = definition.url content_url = definition.url
while content_url and content_url not in found_content_urls: def process_content_url(content_url):
if content_url in found_content_urls:
return
found_content_urls.add(content_url) found_content_urls.add(content_url)
for chapter in self._chapter(content_url, definition): for chapter in self._chapter(content_url, definition):
story.add(chapter) story.add(chapter)
@ -85,14 +87,13 @@ class Arbitrary(Site):
soup, base = self._soup(content_url) soup, base = self._soup(content_url)
next_link = soup.select(definition.next_selector) next_link = soup.select(definition.next_selector)
if next_link: if next_link:
next_link_url = str(next_link[0].get('href')) for next_link_item in next_link:
if base: next_link_url = str(next_link_item.get('href'))
next_link_url = self._join_url(base, next_link_url) if base:
content_url = self._join_url(content_url, next_link_url) next_link_url = self._join_url(base, next_link_url)
else: content_url = self._join_url(content_url, next_link_url)
content_url = False process_content_url(content_url)
else: process_content_url(content_url)
content_url = False
if not story: if not story:
raise SiteException("No story content found; check the content selectors") raise SiteException("No story content found; check the content selectors")