crawl_house/beike/beike.py

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@Contact :   liuyuqi.gov@msn.cn
@Time    :   2023/05/29 22:03:35
@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
@Desc    :   
'''
import requests
from lxml import etree
import csv,os,sys,re
import xlwt
import pandas as pd
import argparse
import logging

class Beike(object):
    ''' 贝壳网数据 '''

    def __init__(self,city:str,save_type:str='csv'):
        ''' 初始化 '''
        self.sess=requests.session()
        self.logger=logging.getLogger(__name__)
        self.logger.setLevel(logging.INFO)
        self.formatter=logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        self.ch=logging.StreamHandler()
        self.ch.setLevel(logging.INFO)
        self.ch.setFormatter(self.formatter)
        self.logger.addHandler(self.ch)
        
        self.args=self.get_args()

    def get_proxy(self):
        ''' 获取代理 '''
        pass
    
    def set_proxy(self):
        pass

    def get_args(self):
        ''' 获取参数 '''
        parser=argparse.ArgumentParser(description='贝壳网数据')
        parser.add_argument('-c','--city',type=str,help='城市')
        parser.add_argument('-p','--page',type=int,default=1,help='页码')
    
    def run(self,city:dict, type='ershoufang'):
        for city in citys:
            for i in range(1,101):
                pass

    def get_zufang(self,city:str,page:int=1):
        ''' 获取租房数据 '''
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
                'Cookie': 'select_city=510100; lianjia_uuid=db980944-7c31-4bcc-8aae-78cdd9ba138d; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22181a93da768e3e-047b67455463da-26021a51-1327104-181a93da769efd%22%2C%22%24device_id%22%3A%22181a93da768e3e-047b67455463da-26021a51-1327104-181a93da769efd%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E4%BB%98%E8%B4%B9%E5%B9%BF%E5%91%8A%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22https%3A%2F%2Fwww.baidu.com%2Fother.php%22%2C%22%24latest_referrer_host%22%3A%22www.baidu.com%22%2C%22%24latest_search_keyword%22%3A%22%E8%B4%9D%E5%A3%B3%E6%89%BE%E6%88%BF%22%2C%22%24latest_utm_source%22%3A%22baidu%22%2C%22%24latest_utm_medium%22%3A%22pinzhuan%22%2C%22%24latest_utm_campaign%22%3A%22wychengdu%22%2C%22%24latest_utm_content%22%3A%22biaotimiaoshu%22%2C%22%24latest_utm_term%22%3A%22biaoti%22%7D%7D; GUARANTEE_BANNER_SHOW=true; login_ucid=2000000245569411; lianjia_ssid=712b2a49-fc10-4962-8fea-379234735b80; lianjia_token=2.0013a6ea83788090ec020bc3b2663bffe3; lianjia_token_secure=2.0013a6ea83788090ec020bc3b2663bffe3; security_ticket=R2sd09nxk8Sm81pfJkvABUssf/StkYIdArkOZN0QqdFQvHLLFF7LBqExRYZbcQYf29gNoQKxH9O7MpAHGu2v73TQnJFksR9KIw/NJ+itit98LyB2Ncs/DgCZmgm3w0ypGjqFK49ik+KFpzJtvr4ukZOE94RSX5+eLymj/Vb+D18=; srcid=eyJ0Ijoie1wiZGF0YVwiOlwiYTRkYjMxYmRhYTM3YTU1ZWI3NDg5OGFiNTVhNTk3ZGQxY2ZhMDkyNTg2YWYwM2NiZDlhMWMwNzM3YmQ2ZmI1ZmZiODA4YmQ4ZTVkMWU4NDNjMjU2M2FhMGZhMzlkM2U4MjkzYTE4Y2U4NjVkNGU3Nzc5NTM5ZmFkNzlmMTE3ZGVkZmFlZDdkNmQxNjc4ZTA0ZDhiOTNiNzhhMDEyM2Q1Y2M0N2QzMDRlYjhkYjliOGM4MGQ2ZTdiMjc0N2UxMzViNjRkYjcwNjA5ZmVkMmIyMjhjMzc2NDQyN2E5YTkzNTY0MjkwYjU2NGE1OGZmYjcxNjMzZDAwZjMxNDBkY2YwNlwiLFwia2V5X2lkXCI6XCIxXCIsXCJzaWduXCI6XCJjODFiYmNhYVwifSIsInIiOiJodHRwczovL2NkLnp1LmtlLmNvbS96dWZhbmciLCJvcyI6IndlYiIsInYiOiIwLjEifQ=='}
        resp = requests.get(url=url,headers=headers)
        #xpath解析
        parser = etree.HTMLParser(encoding='utf-8')
        #转换树对象
        tree = etree.XML(resp.text,parser=parser)

        price = tree.xpath('//*[@id="content"]/div[1]/div[1]/div[1]/div/span/em/text()')
        #print(price)
        addres = tree.xpath('//*[@id="content"]/div[1]/div[1]/div[1]/div/p[2]/a[1]/text()')
        #print(addres)
        access = tree.xpath('//*[@id="content"]/div[1]/div[1]/div[1]/div/p[2]/a[2]/text()')
        #print(access)
        area = tree.xpath('//*[@id="content"]/div[1]/div[1]/div[1]/div/p[2]/a[3]/text()')
        #print(area)
        #拿到一整列的房源标题
        titles = tree.xpath('//*[@id="content"]/div[1]/div[1]/div/div/p[1]/a/text()')
        #print(titles)
        prices = tree.xpath('//*[@id="content"]/div[1]/div[1]/div/div/span/em/text()\n')
        #print(prices)
        #获取你需要的信息的所有板块
        all_list = tree.xpath('//*[@id="content"]/div[1]/div[1]/div')
        #print(all)
        #遍历子板块信

        for all in all_list:
            title = all.xpath('./div/p[1]/a/text()')
            price = all.xpath('./div/span/em/text()')
            path = all.xpath('./div/p[2]/a[1]/text()')
            sub =  all.xpath('./div/p[2]/a[2]/text()')
            home = all.xpath('./div/p[2]/a[3]/text()')
            jx = all.xpath('./div/p[2]/text()[1]')[0]
            if '精选' in jx:
                area = all.xpath('./div/p[2]/text()[6]')
                toward = all.xpath('./div/p[2]/text()[7]')
                house_type = all.xpath('./div/p[2]/text()[8]')
            else:
                area = all.xpath('./div/p[2]/text()[5]')
                toward = all.xpath('./div/p[2]/text()[6]')
                house_type = all.xpath('./div/p[2]/text()[7]')
            list_box.append([title, price, path, sub, home,area,toward,house_type])


        for i in list_box:
            i[0][0] = i[0][0].strip()
            i[1][0] = i[1][0].strip()
            i[2][0] = i[2][0].strip()
            i[4][0] = i[4][0].strip()
            i[5][0] = i[5][0].strip()
            i[6][0] = i[6][0].strip()
            i[7][0] = i[7][0].strip()

        for i in list_box:
            print(i)
    #设置页 加到表头
    '''pages = range(1,3)
    for page in pages:
        url = f'https://cd.zu.ke.com/zufang/pg{page}/#contentList'''
    #存数字方式之一
    #存到csv文件，逗号分隔符文件
    # writer = csv.writer(open('data1.csv','w',encoding='utf-8'))#打开，未创建文件
    # writer.writerow(['title', 'price', 'path', 'sub', 'home','area','toward','house_type'])
    # writer.writerows(list_box)
    # 存储数据方式二
    # 写入excel文件
    wb = xlwt.Workbook()#建立工作布
    sheet = wb.add_sheet(('data'))#建立表
    titles = ('title', 'price', 'path', 'sub', 'home','area','toward','house_type')
    for index,title in enumerate(titles):# enumerate 输出两个维度的数据 1、数据系列的索引值，2、数据本身
        sheet.write(0,index,title)# 参数-：行索引 参数二：列索引 参数三：数据本身
    for i , item in enumerate(list_box):# i:行索引 item：一行数据
        for j ,data in enumerate(item):# j：列索引 data：一个数据
            sheet.write(i+1,j,data)
    wb.save('house.xls')


    def get_ershoufang(self):
        ''' 获取二手房数据 '''
        pass
    
    def get_xinfang(self):
        ''' 获取新房数据 '''
        pass
    
    @staticmethod
    def save( df:pd.DataFrame, filename:str):
        ''' 保存数据 '''
        if os.path.exists(filename):
            df.to_excel(filename, index=False, header=False, mode='a', encoding='utf-8-sig')
        else:
            df.to_excel(filename, index=False, encoding='utf-8-sig')

if __name__=='__main__':
    citys = {
        '成都':'cd',
        '上海':'sh',
        '绵阳':'mianyang',
        "吉安":"jian"
        }
    beike=Beike()
    beike.run(citys, "ershoufang")