采纳答案成功!
向帮助你的同学说点啥吧!感谢那些助人为乐的人
Failure instance: Traceback: <class 'AttributeError'>: 'MysqlTwistedPipline' object has no attribute 'cursor'
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | # -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html import codecs import json from scrapy.pipelines.images import ImagesPipeline from scrapy.exporters import JsonItemExporter from twisted.enterprise import adbapi import MySQLdb import MySQLdb.cursors class ArticlespiderPipeline( object ): def process_item( self , item, spider): return item class JsonWithEncodingPipeline( object ): #自定义json文件的导出 def __init__( self ): self . file = codecs. open ( 'article.json' , 'w' , encoding = "utf-8" ) def process_item( self , item, spider): lines = json.dumps( dict (item), ensure_ascii = False ) + "\n" self . file .write(lines) return item def spider_closed( self , spider): self . file .close() class MysqlPipeline( object ): #采用同步的机制写入mysql def __init__( self ): self .conn = MySQLdb.connect( '192.168.0.106' , 'root' , 'root' , 'article_spider' , charset = "utf8" , use_unicode = True ) self .cursor = self .conn.cursor() def process_item( self , item, spider): insert_sql = """ insert into jobbole_article(title, url, create_date, fav_nums) VALUES (%s, %s, %s, %s) """ self .cursor.execute(insert_sql, (item[ "title" ], item[ "url" ], item[ "create_date" ], item[ "fav_nums" ])) self .conn.commit() class MysqlTwistedPipline( object ): def __init__( self , dbpool): self .dbpool = dbpool @ classmethod def from_settings( cls , settings): dbparms = dict ( host = settings[ "MYSQL_HOST" ], db = settings[ "MYSQL_DBNAME" ], user = settings[ "MYSQL_USER" ], passwd = settings[ "MYSQL_PASSWORD" ], charset = 'utf8' , cursorclass = MySQLdb.cursors.DictCursor, use_unicode = True , ) dbpool = adbapi.ConnectionPool( "MySQLdb" , * * dbparms) return cls (dbpool) def process_item( self , item, spider): #使用twisted将mysql插入变成异步执行 query = self .dbpool.runInteraction( self .do_insert, item) query.addErrback( self .handle_error, item, spider) #处理异常 def handle_error( self , failure, item, spider): #处理异步插入的异常 print (failure) def do_insert( self , cursor, item): #执行具体的插入 #根据不同的item 构建不同的sql语句并插入到mysql中 insert_sql, params = item.get_insert_sql() cursor.execute(insert_sql, params) class JsonExporterPipleline( object ): #调用scrapy提供的json export导出json文件 def __init__( self ): self . file = open ( 'articleexport.json' , 'wb' ) self .exporter = JsonItemExporter( self . file , encoding = "utf-8" , ensure_ascii = False ) self .exporter.start_exporting() def close_spider( self , spider): self .exporter.finish_exporting() self . file .close() def process_item( self , item, spider): self .exporter.export_item(item) return item class ArticleImagePipeline(ImagesPipeline): def item_completed( self , results, item, info): if "front_image_url" in item: for ok, value in results: image_file_path = value[ "path" ] item[ "front_image_path" ] = image_file_path return item |
建议大家可以将我这里的源码拷贝过去, 然后运行一下 然后对照一下看看哪个地方代码不一样, 一般这种问题都是代码某个地方写错了 或者遗漏了
登录后可查看更多问答,登录/注册
带你彻底掌握Scrapy,用Django+Elasticsearch搭建搜索引擎
了解课程