大蟒蛇python教程共享python爬取豆瓣评论制作词云代码

目录

• 一、爬取豆瓣热评
• 二、制作词云
• 总结

import requests  from lxml import etree  import json  import time  class spider(object):      def __init__(self):          #seif.ure='https://movie.douban.com/subject/23885074/reviews?start=0'          self.headers={              'user-agent':'mozilla/5.0(windows nt6.1;win64;x64)applewebkit/537.36(khtml,like gecko)chrome/75.0.3700.100safari/537.36'          }      def get_data(self,url):          response = requests.get(url,headers=self.headers).content.decode('utf-8')          page=etree.html(response)#xpath 对象          #获取所有数据节点          node_list = page.xpath('//div[@class="review-list  "]/div')          for node in node_list:              #作者              author = node.xpath('.//header[@class="main-hd"]//a[2]/text()')[0]              #评论              text = node.xpath('string(.//div[@class="main-bd"]//div[@class="short-content"])')                items={                  'author':author,                  'text':text.strip()              }              #持久化存储              with open('yewen.json','a',encoding='utf-8') as f:                  f.write(json.dumps(items,ensure_ascii=false)+'n')      def run(self):          for i in range(1,47):              url='https://movie.douban.com/subject/26885074/reviews?start{}'.format(i*20)              print('正在爬取第{}页'.format(i))              self.get_data(url)              time.sleep(3)  if __name__=='__main__':      s=spider()      s.run()  

import jieba  from wordcloud import wordcloud  import json  f= open("yewen.json", "r", encoding="utf-8")  data_list= f.readlines()  str =''  for data in data_list:      text= json.loads(data)['text']      str +=text  #替换无关紧要的词语  result_str = str.replace('展开', '').replace('这篇','').replace('影评','').replace('电影','').replace('这部', '').replace('可能', '').replace('剧情','')  cut_text = jieba.lcut(result_str)  result = " ".join(cut_text)  wc = wordcloud(font_path='simhei.ttf',                 background_color="white",                 max_words=600,                 width=1000,                 height=1000,                 min_font_size=20,                 max_font_size=100,)      #mast=plt.imreda('snake.jpg')#背景图片  wc.generate(result)#转化为词云的操作  wc.to_file("text.jpg")#保存  f.close()