python多執行緒學習

【莫煩Python】Threading 學會多執行緒 Python
【2021最新版】Python 并發編程實戰，用多執行緒、多行程、多協程加速程式運行
【莫煩Python】Multiprocessing 讓你的多核計算機發揮真正潛力 Python

threading

知識點1.添加執行緒和join的作用

import threading
import time
def thread_job():
    print("T1 start\n")
    for i in range(10):
        time.sleep(0.1)
    print("T1 finish\n")

def T2_job():
    print("T2 start\n")
    print("T2 finish\n")

def main():
    added_thread = threading.Thread(target=thread_job, name='T1')
    thread2 = threading.Thread(target=T2_job, name='T2')
    added_thread.start()
    thread2.start()
    added_thread.join()
    #thread2.join()
    print("all done\n")
    # print(threading.active_count())
    # print(threading.enumerate())
    # print(threading.current_thread())

if __name__=='__main__':
    main()

多執行緒爬蟲的例子
blog_spider.py

import requests

urls = [
    f"http://www.cnblogs.com/#p{page}"
    for page in range(1, 50 + 1)
]

def craw(url):
    r = requests.get(url)
    print(url,len(r.text))

craw(urls[0])

01.multi_thread_craw.py

import blog_spider
import threading
import time

def multi_thread():
    print("multi_thread begin")
    threads = []
    for url in blog_spider.urls:
        threads.append(
            threading.Thread(target=blog_spider.craw, args=(url,)) #加逗號，這是元組，不加逗號就是字串了
        )

    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join()
    print("multi_thread end")

if __name__ == "__main__":
    start = time.time()
    multi_thread()
    end = time.time()
    print("multi_thread cost:",end - start, "seconds")

知識點2.多執行緒呼叫的函式不能用return回傳值，所以用佇列保存——用于多執行緒資料通信的queue.Queue
生產者消費者爬蟲例子來說明這幾個概念：

好處添加和獲取有阻塞，必須添加了元素才進行下面的代碼，必須有空才獲取元素

import threading
import time
from queue import Queue

def job(l,q):
    for i in range(len(l)):
        l[i] = l[i]**2
    q.put(l) #多執行緒呼叫的函式不能用return回傳值
    return l

def multithreading():
    q = Queue()
    threads = []
    data = [[1,2,3],[3,4,5],[4,4,4],[5,5,5]]
    for i in range(4):
        t = threading.Thread(target=job, args=(data[i],q))
        t.start()
        threads.append(t)
    for thread in threads:
        thread.join()
    results = []
    for _ in range(4):
        results.append(q.get())
    print(results)

if __name__=='__main__':
    multithreading()

知識點3.鎖，鎖住第一個執行緒，等它處理完后再進行下一個——對共享記憶體的處理

import threading
def job1():
    global A
    lock.acquire()
    for i in range(10):
        A += 1
        print("job1",A)
    lock.release()
def job2():
    global A
    lock.acquire()
    for i in range(10):
        A += 10
        print("job2",A)
    lock.release()
if __name__=='__main__':
    lock = threading.Lock()
    A = 0
    t1 = threading.Thread(target=job1)
    t2 = threading.Thread(target=job2)
    t1.start()
    t2.start()
    t1.join()
    t2.join()

multiprocessing

區別1.Queue 是 multiprocessing 里的一個功能 q=mp.Queue()

區別2.行程池 job可以有回傳值了，map&async

import multiprocessing as mp
def job(x):
    return x*x

def multicore():
    # map--自動分配給定義個數的每一個行程/cpu核
    pool = mp.Pool(processes=2)
    res = pool.map(job,range(10))
    print(res)
    # async--一次只能在一個行程
    res = pool.apply_async(job,(2,))
    print(res.get())
    # 迭代器——達到map的效果
    multi_res = [pool.apply_async(job,(i,)) for i in range(10)]
    print([res.get() for res in multi_res])

if __name__== '__main__':
    multicore()

區別3.共享記憶體