Backup_Repos/leech
1
0
Fork 0
mirror of https://github.com/kemayo/leech synced 2025-12-06 08:22:56 +01:00
Code Issues Projects Releases Wiki Activity

simplify algorithm

Browse source
This commit is contained in:
Kevin Pedro 2025-03-08 09:48:32 -06:00
parent d4e1214be3
commit de6913a9af
1 changed files with 5 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
sites/arbitrary.py
View file

@ -80,25 +80,20 @@ class Arbitrary(Site):
def process_content_url(content_url): def process_content_url(content_url):
if content_url in found_content_urls: if content_url in found_content_urls:
return False return None
found_content_urls.add(content_url) found_content_urls.add(content_url)
for chapter in self._chapter(content_url, definition): for chapter in self._chapter(content_url, definition):
story.add(chapter) story.add(chapter)
return True return content_url
while content_urls: while content_urls:
status = False for temp_url in content_urls:
for content_url in content_urls:
# stop inner loop once a new link is found # stop inner loop once a new link is found
status = process_content_url(content_url) if content_url := process_content_url(temp_url):
if status:
break break
# stop outer loop if no new links found
if not status:
break
# reset url list # reset url list
content_urls = [] content_urls = []
if definition.next_selector: if content_url and definition.next_selector:
soup, base = self._soup(content_url) soup, base = self._soup(content_url)
next_link = soup.select(definition.next_selector) next_link = soup.select(definition.next_selector)
if next_link: if next_link:
@ -108,8 +103,6 @@ class Arbitrary(Site):
next_link_url = self._join_url(base, next_link_url) next_link_url = self._join_url(base, next_link_url)
content_urls.append(self._join_url(content_url, next_link_url)) content_urls.append(self._join_url(content_url, next_link_url))
process_content_url(content_url)
if not story: if not story:
raise SiteException("No story content found; check the content selectors") raise SiteException("No story content found; check the content selectors")