From 2161f4ba01fcb981632551484ef8232d396f7a10 Mon Sep 17 00:00:00 2001 From: emamaker Date: Sun, 6 Feb 2022 09:18:28 +0100 Subject: [PATCH] replace ended threads in real time instead of waiting for them all to finish --- page_viewer.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/page_viewer.py b/page_viewer.py index 2b0fb69..0b6c4e9 100644 --- a/page_viewer.py +++ b/page_viewer.py @@ -117,14 +117,36 @@ def launch_and_visit(use_tor, page_url, headless=False): browser_manager.close_browser(driver, tor_process) if __name__ == "__main__": - while True: - threads = [] + NO_PROCESSES = 10 + threads = [] - for i in range(0, 5): - t1 = threading.Thread(target=launch_and_visit, args=(True, 'https://giangillorossi.altervista.org', True)) + while True: + if len(threads) < NO_PROCESSES: + print(f"[BOT] Starting thread n.{len(threads)}/{NO_PROCESSES}") + t1 = threading.Thread(target=launch_and_visit, args=(False, 'https://giangillorossi.altervista.org', False)) t1.start() threads.append(t1) - for t in threads: - t.join() + i = 0 + while i < len(threads): + # print(i) + if not (threads[i].is_alive()): + print(f"[BOT] Thread n.{i} has stopped, removing from list") + threads.pop(i) + + else: + i += 1 + + + + + + # for i in range( 0, 5): + # t1 = threading.Thread(target=launch_and_visit, args=(True, 'https://giangillorossi.altervista.org', True)) + # t1.start() + + # threads.append(t1) + + # for t in threads: + # t.join()