Backup_Repos/leech
1
0
Fork 0
mirror of https://github.com/kemayo/leech synced 2025-12-06 08:22:56 +01:00
Code Issues Projects Releases Wiki Activity

Merge b2f15eb76c into cfd073fb5c

Browse source
This commit is contained in:
Kevin Pedro 2025-03-07 08:54:12 -06:00 committed by GitHub
commit 8df8a593ff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 16 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
sites/arbitrary.py
View file

@ -77,7 +77,10 @@ class Arbitrary(Site):
# set of already processed urls. Stored to detect loops. # set of already processed urls. Stored to detect loops.
found_content_urls = set() found_content_urls = set()
content_url = definition.url content_url = definition.url
while content_url and content_url not in found_content_urls:
def process_content_url(content_url):
if content_url in found_content_urls:
return False
found_content_urls.add(content_url) found_content_urls.add(content_url)
for chapter in self._chapter(content_url, definition): for chapter in self._chapter(content_url, definition):
story.add(chapter) story.add(chapter)
@ -85,14 +88,18 @@ class Arbitrary(Site):
soup, base = self._soup(content_url) soup, base = self._soup(content_url)
next_link = soup.select(definition.next_selector) next_link = soup.select(definition.next_selector)
if next_link: if next_link:
next_link_url = str(next_link[0].get('href')) for next_link_item in next_link:
next_link_url = str(next_link_item.get('href'))
if base: if base:
next_link_url = self._join_url(base, next_link_url) next_link_url = self._join_url(base, next_link_url)
content_url = self._join_url(content_url, next_link_url) content_url = self._join_url(content_url, next_link_url)
else: # stop loop once a new link is found
content_url = False status = process_content_url(content_url)
else: if status:
content_url = False break
return True
process_content_url(content_url)
if not story: if not story:
raise SiteException("No story content found; check the content selectors") raise SiteException("No story content found; check the content selectors")