eneller/epub2go.py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: crawl unified from list

Browse Source
This commit is contained in:
eneller
2025-03-03 22:56:58 +01:00
parent 8f77a97733
commit 7be0fbc126
1 changed files with 5 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/epub2go/crawl.py
View File

@@ -6,23 +6,14 @@ from bs4 import ResultSet
import os import os
from urllib.parse import urljoin from urllib.parse import urljoin
from convert import GBConvert, allbooks_url from convert import GBConvert, allbooks_url, get_all_books
def parse_book_tags ()-> ResultSet:
response = requests.get(allbooks_url)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'html.parser', from_encoding='utf-8')
books = soup.find('dl').find_all('a')
books = [(book.get_text(), book.get('href')) for book in books]
return books
def main(): def main():
books = parse_book_tags() books = get_all_books()
# NOTE consider making this a map() # NOTE consider making this a map()
for book in tqdm(parse_book_tags()): for book in tqdm(books):
(book_title, book_url_relative) = book book_url = book['url']
if book_url_relative is not None: if book_url is not None:
book_url = urljoin(allbooks_url, book_url_relative)
GBConvert(book_url).run() GBConvert(book_url).run()