link_1999
/
crawler-MiniProgram
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

上传文件至 'dagongPro' 

新增：将图片url保存到mysql数据库
This commit is contained in:
link_1999 2022-04-06 21:45:02 +08:00
parent ef4d4170dc
commit 5dd1363076
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dagongPro/pipelines.py
View File

@ -8,6 +8,7 @@
from itemadapter import ItemAdapter from itemadapter import ItemAdapter
import pymysql import pymysql
class DagongwangproPipeline: class DagongwangproPipeline:
def process_item(self, item, spider): def process_item(self, item, spider):
@ -17,7 +18,7 @@ class DagongwangproPipeline:
content = item['content'] content = item['content']
date = item['date'] date = item['date']
news_path = './新闻/军事新闻/' + title + '.txt' news_path = './新闻/内地新闻/' + title + '.txt'
with open(news_path, 'w', encoding='utf-8') as fp: with open(news_path, 'w', encoding='utf-8') as fp:
fp.write(date+'\n'+content) fp.write(date+'\n'+content)
@ -33,7 +34,7 @@ class mysqlPileLine(object):
self.cursor = self.conn.cursor() self.cursor = self.conn.cursor()
try: try:
self.cursor.execute('insert into junshi_news values("%s", "%s", "%s")'%(item['title'], item['date'], item['content'])) self.cursor.execute('insert into junshi_news values("%s", "%s", "%s", "%s")'%(item['title'], item['date'], item['content'], item['src']))
self.conn.commit() self.conn.commit()
except Exception as e: except Exception as e:
print(e) print(e)
@ -44,3 +45,4 @@ class mysqlPileLine(object):
def close_spider(self, spider): def close_spider(self, spider):
self.cursor.close() self.cursor.close()
self.conn.close() self.conn.close()