Force initial link into URI for parsing
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
This commit is contained in:
parent
2424063212
commit
664c65d916
1 changed files with 1 additions and 2 deletions
|
@ -33,7 +33,6 @@ def check_for_links(address, agent, indexed_pages = [])
|
|||
link.fragment = nil
|
||||
# Clear query (TODO: Stash away later)
|
||||
link.query = nil
|
||||
next if link.is_a?(String)
|
||||
|
||||
# Check if we've seen the link before
|
||||
# Check if we're leaving the host
|
||||
|
@ -127,7 +126,7 @@ if opts.discover?
|
|||
crawl_starting_address = homepage.uri
|
||||
end
|
||||
|
||||
addresses = check_for_links(crawl_starting_address, agent)
|
||||
addresses = check_for_links(Uri.parse(crawl_starting_address), agent)
|
||||
old_addresses = []
|
||||
until addresses - old_addresses == [] && old_addresses - addresses == []
|
||||
old_addresses = Array.new(addresses)
|
||||
|
|
Loading…
Add table
Reference in a new issue