init

README.md

@@ -0,0 +1,6 @@
+## 微博爬虫
+
+* 热搜爬虫
+* 用户爬虫
+
+

main.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+'''
+@Author  :   liuyuqi
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2019/08/27 15:30:32
+@Version :   1.0
+@License :   (C)Copyright 2019
+@Desc    :   
+'''

requirements.txt

@@ -0,0 +1,8 @@
+pandas
+json
+requests
+jieba
+scipy
+wordcloud
+matplotlib
+collections

user/get_user.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+'''
+@Author  :   liuyuqi
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2019/08/28 03:11:28
+@Version :   1.0
+@License :   (C)Copyright 2019
+@Desc    :   用户爬虫
+'''

utils/top_topic.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+'''
+@Author  :   liuyuqi
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2019/08/27 15:30:32
+@Version :   1.0
+@License :   (C)Copyright 2019
+@Desc    :   微博热搜爬虫，分析
+'''
+## 调用要使用的包
+from pyecharts.charts import Page, WordCloud, TreeMap
+from pyecharts.charts import Calendar, Pie
+from pyecharts.globals import SymbolType
+from pyecharts import GraphicShapeOpts
+from pyecharts import options as opts
+import json
+import random
+import requests
+import time
+import pandas as pd
+import os
+import jieba
+from scipy.misc import imread  # 这是一个处理图像的函数
+from wordcloud import WordCloud, ImageColorGenerator
+import matplotlib.pyplot as plt
+import datetime
+from collections import Counter
+
+os.chdir('D:/爬虫/微博热搜')
+
+## 获得日期
+def getBetweenDay(begin_date, end_date):
+    date_list = []
+    begin_date = datetime.datetime.strptime(begin_date, "%Y/%m/%d")
+    end_date = datetime.datetime.strptime(end_date, "%Y/%m/%d")
+    while begin_date <= end_date:
+        date_str = begin_date.strftime("%Y/%m/%d")
+        date_list.append(date_str)
+        begin_date += datetime.timedelta(days=1)
+    return date_list
+
+## 分词
+def get_words_list(df):
+    df['words_list'] = []
+    word_generator = jieba.cut_for_search(df['title'])
+    for word in word_generator:
+        df['words_list'].append(word)
+    return df
+
+## explode
+def dataframe_explode(dataframe, fieldname):
+    temp_fieldname = fieldname + '_made_tuple_'
+    dataframe[temp_fieldname] = dataframe[fieldname].apply(tuple)
+    list_of_dataframes = []
+    for values in dataframe[temp_fieldname].unique().tolist():
+        list_of_dataframes.append(pd.DataFrame({
+            temp_fieldname: [values] * len(values),
+            fieldname: list(values),
+        }))
+    dataframe = dataframe[list(set(dataframe.columns) - set([fieldname]))].merge(
+        pd.concat(list_of_dataframes), how='left', on=temp_fieldname)
+    del dataframe[temp_fieldname]
+    return dataframe
+
+## 绘制词云
+def draw_word_cloud(word):
+    word_title = resou_word[resou_word['title'].str.contains(word)]
+    word_title = word_title.groupby(
+        ['title'], as_index=False).agg({'searchCount': ['max']})
+    word_title.columns = ['title', 'count']
+    data = [(word_title['title'][i], word_title['count'][i]/1000000)
+            for i in range(word_title.shape[0])]
+    wc = (WordCloud(init_opts=opts.InitOpts(theme=ThemeType.ROMA))
+          .add("", data, word_size_range=[20, 50], shape='pentagon')
+          .set_global_opts(title_opts=opts.TitleOpts(title=''))
+          .render('{}词云.html'.format(word))
+          )
+
+
+## 设置headers和cookie，数据爬取
+header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win32; x32; rv:54.0) Gecko/20100101 Firefox/54.0',
+          'Connection': 'keep-alive'}
+cookies = 'v=3; iuuid=1A6E888B4A4B29B16FBA1299108DBE9CDCB327A9713C232B36E4DB4FF222CF03; webp=true; ci=1%2C%E5%8C%97%E4%BA%AC; __guid=26581345.3954606544145667000.1530879049181.8303; _lxsdk_cuid=1646f808301c8-0a4e19f5421593-5d4e211f-100200-1646f808302c8; _lxsdk=1A6E888B4A4B29B16FBA1299108DBE9CDCB327A9713C232B36E4DB4FF222CF03; monitor_count=1; _lxsdk_s=16472ee89ec-de2-f91-ed0%7C%7C5; __mta=189118996.1530879050545.1530936763555.1530937843742.18'
+cookie = {}
+for line in cookies.split(';'):
+    name, value = cookies.strip().split('=', 1)
+    cookie[name] = value
+
+resou = pd.DataFrame(columns=['date', 'title', 'searchCount', 'rank'])
+resou_date = getBetweenDay('2019/01/01', '2019/07/12')
+for i in resou_date:
+    print(i)
+    url = 'https://www.enlightent.cn/research/top/getWeiboHotSearchDayAggs.do?date={}'.format(
+        str(i))
+    html = requests.get(url=url, cookies=cookie, headers=header).content
+    data = json.loads(html.decode('utf-8'))
+    for j in range(100):
+        resou = resou.append({'date': i, 'title': data[j]['keyword'],
+                              'searchCount': data[j]['searchCount'], 'rank': j+1}, ignore_index=True)
+
+
+## 按天统计
+resou = resou.apply(get_words_list, axis=1)
+resou.to_excel('热搜数据.xlsx')
+resou_dt = resou.groupby('date', as_index=False).agg({'searchCount': ['mean']})
+resou_dt.columns = ['date', 'avg_count']
+
+
+## 绘制日历图
+data = [
+    [resou_dt['date'][i], resou_dt['avg_count'][i]]
+    for i in range(resou_dt.shape[0])
+]
+
+
+calendar = (
+    Calendar(init_opts=opts.InitOpts(width='1800px', height='1500px'))
+    .add("", data, calendar_opts=opts.CalendarOpts(range_=['2019-01-01', '2019-07-12']))
+    .set_global_opts(
+        title_opts=opts.TitleOpts(title="2019每日热搜平均指数", pos_left='15%'),
+        visualmap_opts=opts.VisualMapOpts(
+            max_=3600000,
+            min_=0,
+            orient="horizontal",
+            is_piecewise=False,
+            pos_top="230px",
+            pos_left="100px",
+            pos_right="10px"
+        )
+    )
+    .render('日期热力图.html')
+)
+
+
+## 词频统计
+resou_word = dataframe_explode(resou, 'words_list')
+resou_word_stat = resou_word.groupby(
+    ['words_list'], as_index=False).agg({'title': ['count']})
+resou_word_stat.columns = ['word', 'num']
+resou_word_stat.to_excel('词频统计.xlsx')
+
+func = pd.read_excel('名词.xlsx')
+love = pd.read_excel('婚恋.xlsx')
+person = pd.read_excel('人物.xlsx')
+resou_word_func = pd.merge(resou_word_stat, func, how='inner', on='word')
+resou_word_love = pd.merge(resou_word_stat, love, how='inner', on='word')
+resou_word_person = pd.merge(resou_word_stat, person, how='inner', on='word')
+
+
+resou_word_func = resou_word_func.sort_values(
+    'count', ascending=False).reset_index()
+words = [(resou_word_func['word'][i], resou_word_func['num'][i])
+         for i in range(resou_word_func.shape[0])]
+pie = (
+    Pie(init_opts=opts.InitOpts(theme=ThemeType.CHALK))
+    .add("", words)
+    .set_global_opts(title_opts=opts.TitleOpts(title="微博常用词出现次数", pos_left='center'),
+                     legend_opts=opts.LegendOpts(is_show=False))
+    .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}", font_size=16),)
+    .render('热搜词饼图.html')
+)
+
+
+resou_word_love = resou_word_love.sort_values(
+    'count', ascending=False).reset_index()
+words = [(resou_word_love['word'][i], resou_word_love['num'][i])
+         for i in range(resou_word_love.shape[0])]
+pie = (
+    Pie(init_opts=opts.InitOpts(theme=ThemeType.CHALK))
+    .add("", words)
+    .set_global_opts(title_opts=opts.TitleOpts(title="微博婚恋类词语出现次数", pos_left='center'),
+                     legend_opts=opts.LegendOpts(is_show=False))
+    .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}", font_size=16),)
+    .render('婚恋类热搜词饼图.html')
+)
+
+
+resou_word_person = resou_word_person.sort_values(
+    'num', ascending=False).reset_index()
+words = [{'"value": {}, "name": {}"'.format(str(resou_word_person['num'][i]),
+                                            resou_word_person['name'][i])} for i in range(resou_word_person.shape[0])]
+
+tree = (
+    TreeMap(init_opts=opts.InitOpts(theme=ThemeType.ESSOS))
+    .add("", words, pos_left=0, pos_right=0, pos_top=50, pos_bottom=50)
+    .set_global_opts(title_opts=opts.TitleOpts(title="热搜明星出现次数排名", pos_left='center'),
+                     legend_opts=opts.LegendOpts(is_show=False))
+    .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}\n\n {c}", font_size=17,
+                                               color='black', position='inside', font_weight='bolder'))
+    .render('排序.html')
+)