如何用Python 快速发送1000条网络请求

如何使用 Python 最快发送 1000 个http 请求，这里做个测试记录。

方式

threading+requests

• 多线程

import time
import threading
import requests


def req(url):
    with requests.get(url) as resp:
        return resp.status_code

def main():
    url = 'http://httpbin.org/status/200'
    threads = []
    results = []
    start = time.time()

    # Function to handle each request and store its result
    def handle_request(url):
        status = req(url)
        results.append(status)

    for _ in range(1000):
        thread = threading.Thread(target=handle_request, args=(url,))
        threads.append(thread)
        thread.start()

    # Wait for all threads to complete
    for thread in threads:
        thread.join()

    print("Len: ", len(results), " Status: ", results[-1])
    print(f"Costs: {time.time() - start:.3f}")

if __name__ == "__main__":
    main()

async aiohttp

• async 异步 aiohttp

import asyncio
import time
import aiohttp


async def req(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as resp:
            return resp.status


async def main():
    start = time.time()

    tasks = [req('http://httpbin.org/status/200') for _ in range(1000)]
    result = await asyncio.gather(*tasks)

    print("Len: ", len(result), " Status: ", result[-1])
    print(f"Costs: {time.time() - start:.3f}")

asyncio.run(main())

golang net.http

package main

import (
 "fmt"
 "net/http"
 "sync"
 "time"
)

func main() {
 var wg sync.WaitGroup
 startTime := time.Now()
 var statuses []string
 var mu sync.Mutex

 client := http.Client{
  Transport: &http.Transport{
   Proxy: nil,
  },
 }
 for i := 0; i < 1000; i++ {
  wg.Add(1)
  go func(u string) {
   defer wg.Done()
   resp, err := client.Get(u)
   if err != nil {
    fmt.Println("Error fetching URL:", u, err)
    return
   }
   defer resp.Body.Close()
   mu.Lock()
   statuses = append(statuses, resp.Status)
   mu.Unlock()
  }("http://httpbin.org/status/200")
 }
 wg.Wait()
 elapsedTime := time.Since(startTime)
 fmt.Println("Len： ", len(statuses), " Status:", statuses[len(statuses)-1])
 fmt.Println("Costs:", elapsedTime)
}

测试结果

• get 请求 1000 次http://httpbin.org/status/200 3 次取平均结果如下：

环境	方式	结果(秒)
py3.10	threading + requests	2.895
py3.10	asyncio + aiohttp	1.904
go1.21	goroutine + net.http	1.816

• 结论： python 异步网络I/O 性能和Golang 差不多。
• 之前还看到过Skywind（林伟）写的python Redis 的性能测试

  python 3.11 下，asyncio 比 gevent 快 50%，加上 uvloop 可以快一倍。纯用 asyncio 性能可以做到 redis 的 68%，而加上 uvloop 后可以做到 redid的 88%。

• 同样基于asyncio 的FastAPI Web 开发框架号称比肩 Golang
• Python Asyncio 还是非常高效的。