def __init__(self, host, database, user, password, port):
def from_crawler(cls, crawler):
return cls(host=crawler.settings.get('MYSQL_HOST'),
database=crawler.settings.get('MYSQL_DATABASE'),
user=crawler.settings.get('MYSQL_USER'),
password=crawler.settings.get('MYSQL_PASSWORD'),
port=crawler.settings.get('MYSQL_PORT'),
def open_spider(self, spider):
self.db = pymysql.connect(self.host, self.user, self.password, self.database, charset='utf8', port=self.port)
self.cursor = self.db.cursor()
def close_spider(self, spider):
def process_item(self, item, spider):
keys = ', '.join(data.keys())
values = ', '.join(['% s'] * len(data))
sql = 'insert into % s (% s) values (% s)' % (item.table, keys, values)
self.cursor.execute(sql, tuple(data.values()))