Python异步编程最佳实践:从理论到实战的完整指南

在现代Python开发中,异步编程已成为提升应用性能的关键技术。本文将带你系统学习异步编程的最佳实践。

1. 异步编程核心概念

1.1 事件循环(Event Loop)

1
2
3
4
5
6
7
8
9
import asyncio

async def main():
print("开始执行")
await asyncio.sleep(1) # 模拟I/O操作
print("完成")

# 启动事件循环
asyncio.run(main())

1.2 协程(Coroutine)与任务(Task)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
import asyncio

async def fetch_data(url):
print(f"正在获取 {url}")
await asyncio.sleep(1)
return f"数据来自 {url}"

async def main():
# 创建多个协程任务
tasks = [
fetch_data("https://api1.com"),
fetch_data("https://api2.com"),
fetch_data("https://api3.com")
]

# 并发执行所有任务
results = await asyncio.gather(*tasks)
print(results)

asyncio.run(main())

2. 最佳实践指南

2.1 线程与进程选择

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
import asyncio
import concurrent.futures
from multiprocessing import Pool

# CPU密集型任务:使用multiprocessing
def cpu_intensive_task(n):
return sum(i * i for i in range(n))

async def cpu_bound_async(n):
with Pool() as pool:
result = pool.apply(cpu_intensive_task, (n,))
return result

# I/O密集型任务:使用asyncio
async def io_bound_async(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
return await response.text()

2.2 错误处理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
import asyncio
import logging

logging.basicConfig(level=logging.INFO)

async def safe_fetch(url):
try:
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
if response.status == 200:
return await response.text()
else:
raise Exception(f"HTTP {response.status}")
except asyncio.TimeoutError:
logging.error(f"请求 {url} 超时")
return None
except Exception as e:
logging.error(f"请求 {url} 失败: {e}")
return None

async def main():
urls = ["https://api1.com", "https://api2.com", "https://api3.com"]
tasks = [safe_fetch(url) for url in urls]
results = await asyncio.gather(*tasks, return_exceptions=True)

# 处理异常结果
for i, result in enumerate(results):
if isinstance(result, Exception):
logging.warning(f"第{i+1}个请求失败: {result}")
else:
logging.info(f"第{i+1}个请求成功: {len(result)} 字符")

2.3 资源管理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
import asyncio
import asyncpg

class DatabaseManager:
def __init__(self, dsn):
self.dsn = dsn
self.pool = None

async def __aenter__(self):
self.pool = await asyncpg.create_pool(self.dsn)
return self

async def __aexit__(self, exc_type, exc_val, exc_tb):
if self.pool:
await self.pool.close()

async def execute_query(self, query, *args):
async with self.pool.acquire() as conn:
return await conn.fetchval(query, *args)

# 使用示例
async def main():
async with DatabaseManager("postgresql://user:pass@localhost/db") as db:
result = await db.execute_query("SELECT 1")
print(f"数据库查询结果: {result}")

3. 性能优化技巧

3.1 连接池管理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
import asyncio
import aiomysql

# 高并发场景下的连接池配置
async def create_connection_pool():
return await aiomysql.create_pool(
host='localhost',
port=3306,
user='user',
password='pass',
db='test',
minsize=10, # 最小连接数
maxsize=100, # 最大连接数
echo=False, # 是否打印SQL
autocommit=True # 自动提交
)

3.2 并发控制

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
import asyncio
from asyncio import Semaphore

# 限制并发请求数量
async def limited_fetch(semaphore, url):
async with semaphore:
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
return await response.text()

async def main():
semaphore = Semaphore(10) # 限制同时进行10个请求
urls = ["https://api1.com"] * 100

tasks = [limited_fetch(semaphore, url) for url in urls]
results = await asyncio.gather(*tasks)

4. 常见陷阱与解决方案

问题 解决方案
回调地狱 使用 async/await 语法
阻塞式代码 使用 asyncio.to_thread() 或 concurrent.futures
资源泄漏 使用 async with 和上下文管理器
调试困难 使用 asyncio.run_coroutine_threadsafe() 调试

5. 实战项目:异步爬虫

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
import asyncio
import aiohttp
from bs4 import BeautifulSoup

async def fetch_page(session, url):
async with session.get(url) as response:
return await response.text()

async def parse_page(html):
soup = BeautifulSoup(html, 'html.parser')
titles = [tag.text for tag in soup.find_all('h1')]
return titles

async def main():
urls = ['https://example.com/page1', 'https://example.com/page2']

# 创建会话池
connector = aiohttp.TCPConnector(limit=100)
async with aiohttp.ClientSession(connector=connector) as session:
# 并发获取页面
pages = await asyncio.gather(*[fetch_page(session, url) for url in urls])

# 并发解析
titles = await asyncio.gather(*[parse_page(page) for page in pages])

for i, title_list in enumerate(titles):
print(f"页面 {i+1} 标题: {title_list[:3]}")

asyncio.run(main())

经验总结:异步编程不是万能药,要根据具体场景选择合适的技术方案。对于I/O密集型任务,异步能带来显著性能提升;对于CPU密集型任务,多线程或多进程可能更合适。

阅读更多