ap7sc/task4
2
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Use regex for filter condition

Browse source
This commit is contained in:
Peter Vacho 2024-11-19 20:14:52 +01:00
parent 47c9a9f555
commit 7f9798ed28
Signed by: school
GPG key ID: 8CFC3837052871B4
1 changed files with 8 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/__main__.py
View file

@ -1,4 +1,5 @@
import asyncio import asyncio
import re
from pprint import pprint from pprint import pprint
import httpx import httpx
@ -6,12 +7,18 @@ import httpx
from src.link_scraper import get_urlmap from src.link_scraper import get_urlmap
URL = httpx.URL("https://ailab.fai.utb.cz") URL = httpx.URL("https://ailab.fai.utb.cz")
ALLOWED_HOSTS_RE = re.compile(r"(?:.*\.)?utb\.cz")
async def main() -> None: async def main() -> None:
"""Program entrypoint.""" """Program entrypoint."""
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
url_map = await get_urlmap(client, URL, max_depth=2, filter_condition=lambda url: url.host == URL.host) url_map = await get_urlmap(
client,
URL,
max_depth=5,
filter_condition=lambda url: ALLOWED_HOSTS_RE.fullmatch(url.host) is not None,
)
pprint(url_map) pprint(url_map)