Force initial link into URI for parsing
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed

This commit is contained in:
Blizzard Finnegan 2024-09-21 20:38:05 -04:00
parent 2424063212
commit 664c65d916
Signed by: blizzardfinnegan
GPG key ID: 33BC649AD444BD96

View file

@ -33,7 +33,6 @@ def check_for_links(address, agent, indexed_pages = [])
link.fragment = nil
# Clear query (TODO: Stash away later)
link.query = nil
next if link.is_a?(String)
# Check if we've seen the link before
# Check if we're leaving the host
@ -127,7 +126,7 @@ if opts.discover?
crawl_starting_address = homepage.uri
end
addresses = check_for_links(crawl_starting_address, agent)
addresses = check_for_links(Uri.parse(crawl_starting_address), agent)
old_addresses = []
until addresses - old_addresses == [] && old_addresses - addresses == []
old_addresses = Array.new(addresses)