Backup_Repos/leech
1
0
Fork 0
mirror of https://github.com/kemayo/leech synced 2025-12-06 08:22:56 +01:00
Code Issues Projects Releases Wiki Activity

process all next_link items

Browse source
This commit is contained in:
Kevin Pedro 2025-03-04 22:13:20 -06:00
parent 53bc2045f0
commit 0066a148bb
1 changed files with 10 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
sites/arbitrary.py
View file

@ -77,7 +77,9 @@ class Arbitrary(Site):
# set of already processed urls. Stored to detect loops.
found_content_urls = set()
content_url = definition.url
while content_url and content_url not in found_content_urls:
def process_content_url(content_url):
if content_url in found_content_urls:
return
found_content_urls.add(content_url)
for chapter in self._chapter(content_url, definition):
story.add(chapter)
@ -85,14 +87,13 @@ class Arbitrary(Site):
soup, base = self._soup(content_url)
next_link = soup.select(definition.next_selector)
if next_link:
next_link_url = str(next_link[0].get('href'))
for next_link_item in next_link:
next_link_url = str(next_link_item.get('href'))
if base:
next_link_url = self._join_url(base, next_link_url)
content_url = self._join_url(content_url, next_link_url)
else:
content_url = False
else:
content_url = False
process_content_url(content_url)
process_content_url(content_url)
if not story:
raise SiteException("No story content found; check the content selectors")