Python爬虫中如何在pipelines.py将数据写入MySQL数据库？

在Scrapy的pipelines.py里把数据存到MySQL，核心就这几步：

1. 先在settings.py里配好数据库连接：

MYSQL_HOST = 'localhost'
MYSQL_DATABASE = 'your_db'
MYSQL_USER = 'root'
MYSQL_PASSWORD = 'your_password'

1. 在pipelines.py里写个处理类：

import pymysql
from itemadapter import ItemAdapter

class MySQLPipeline:
    def __init__(self, mysql_host, mysql_db, mysql_user, mysql_password):
        self.host = mysql_host
        self.db = mysql_db
        self.user = mysql_user
        self.password = mysql_password
    
    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mysql_host=crawler.settings.get('MYSQL_HOST'),
            mysql_db=crawler.settings.get('MYSQL_DATABASE'),
            mysql_user=crawler.settings.get('MYSQL_USER'),
            mysql_password=crawler.settings.get('MYSQL_PASSWORD')
        )
    
    def open_spider(self, spider):
        self.connection = pymysql.connect(
            host=self.host,
            user=self.user,
            password=self.password,
            database=self.db,
            charset='utf8mb4',
            cursorclass=pymysql.cursors.DictCursor
        )
        self.cursor = self.connection.cursor()
    
    def close_spider(self, spider):
        self.connection.close()
    
    def process_item(self, item, spider):
        # 根据你的item结构写SQL
        sql = """
        INSERT INTO your_table (title, url, content, created_at)
        VALUES (%s, %s, %s, NOW())
        """
        values = (
            item.get('title'),
            item.get('url'),
            item.get('content')
        )
        
        try:
            self.cursor.execute(sql, values)
            self.connection.commit()
        except Exception as e:
            self.connection.rollback()
            spider.logger.error(f"MySQL error: {e}")
        
        return item

1. 别忘了在settings.py里启用这个pipeline：

ITEM_PIPELINES = {
    'your_project.pipelines.MySQLPipeline': 300,
}

简单说就是配连接、写插入逻辑、记得提交事务。