multithreading python avec Beautifulsoup

Question

C'est la fonction qui lit le lien Url et le convertit en Beautifulsoup.

multithreadding=[]
    def scraper_worker(url):
        r=requests.get(url)
        soup = BeautifulSoup(r.text,"html.parser")
        data=soup.find("div",{"class":"main-container"})
        multithreadding.append(data) 

threadding=[]
 for u in split_link:
     t=Thread(target=scraper_worker,args=(u, ))
     t.start()
     threadding.append(t)

split_link est la liste où 50 liens impairs sont stockés. Je suis confronté à un problème d'exécution de la partie multithreading.

Answer 1

Il s'agit d'un exemple d'utilisation queue pour envoyer les résultats du fil au fil principal.

import requests
from bs4 import BeautifulSoup
from threading import Thread
import queue

# --- functions ---

def worker(url, queue): # get queue as argument
    r = requests.get(url)

    soup = BeautifulSoup(r.text, "html.parser")
    data = soup.find("span", {"class": "text"}).get_text()

    # send result to main thread using queue
    queue.put(data)

# --- main ---

all_links = [
    'http://quotes.toscrape.com/page/' + str(i) for i in range(1, 11)
]

all_threads = []
all_results = []
my_queue = queue.Queue()

# run threads
for url in all_links:
    t = Thread(target=worker, args=(url, my_queue))
    t.start()
    all_threads.append(t)

# get results from queue    
while len(all_results) < len(all_links):
    # get result from queue
    data = my_queue.get()
    all_results.append(data)

    # or with queue.empty if loop has to do something more
    # because queue.get() wait for data if queue is empty and blocks loop

    #if not my_queue.empty():
    #    data = my_queue.get()
    #    all_results.append(data)

# display results        
for item in all_results:        
    print(item[:50], '...')