From db5feb1642802d5885a50c1daefed3000285a798 Mon Sep 17 00:00:00 2001 From: Samuel Huang Date: Mon, 7 Jan 2019 09:55:32 -0800 Subject: [PATCH] Add support for bad url locations --- check_urls.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/check_urls.py b/check_urls.py index 918f1f937..bbfff812a 100644 --- a/check_urls.py +++ b/check_urls.py @@ -1,9 +1,10 @@ #!/usr/bin/env python -from concurrent import futures import multiprocessing as mp import os import json import uuid +from concurrent import futures +from collections import defaultdict from bs4 import BeautifulSoup from markdown import markdown @@ -18,20 +19,23 @@ URL_BOT_ID = f'Bot {str(uuid.uuid4())}' -def extract_urls_from_html(content, all_urls): +def extract_urls_from_html(content): soup = BeautifulSoup(content, 'html.parser') + html_urls = set() for a in soup.find_all('a', href=True): url = a['href'] if url.startswith('http'): - all_urls.add(url) + html_urls.add(url) + return html_urls def extract_urls(discover_path): exclude = ['.git', '.vscode'] - all_urls = set() + all_urls = defaultdict(list) max_strlen = -1 for root, dirs, files in os.walk(discover_path, topdown=True): dirs[:] = [d for d in dirs if d not in exclude] + short_root = root.lstrip(discover_path) for file in files: output = f'Currently checking: file={file}' file_path = os.path.join(root, file) @@ -40,10 +44,14 @@ def extract_urls(discover_path): print(output.ljust(max_strlen), end='\r') if file_path.endswith('.html'): content = open(file_path) - extract_urls_from_html(content, all_urls) + extract_urls_from_html(content) elif file_path.endswith('.markdown'): content = markdown(open(file_path).read()) - extract_urls_from_html(content, all_urls) + else: + continue + html_urls = extract_urls_from_html(content) + for url in html_urls: + all_urls[url].append(os.path.join(short_root, file)) return all_urls @@ -87,20 +95,27 @@ def bad_url(url_status): def main(): print('Extract urls...') - all_urls = extract_urls(os.getcwd()) + all_urls = extract_urls(os.getcwd() + os.path.sep + 'content') print('\nCheck urls...') - bad_urls = {} + bad_url_status = {} url_id = 1 max_strlen = -1 - for url_path, url_status in run_workers(get_url_status, all_urls): + for url_path, url_status in run_workers(get_url_status, all_urls.keys()): output = f'Currently checking: id={url_id} host={urllib3.util.parse_url(url_path).host}' if max_strlen < len(output): max_strlen = len(output) print(output.ljust(max_strlen), end='\r') if bad_url(url_status) is True: - bad_urls[url_path] = url_status + bad_url_status[url_path] = url_status url_id += 1 - print(f'\nBad urls: {json.dumps(bad_urls, indent=4)}') + bad_url_location = { + bad_url: all_urls[bad_url] + for bad_url in bad_url_status + } + status_content = json.dumps(bad_url_status, indent=4) + location_content = json.dumps(bad_url_location, indent=4) + print(f'\nBad url status: {status_content}') + print(f'\nBad url locations: {location_content}') if __name__ == '__main__':