crawl_secondhand/crawl_secondhand/extractor/xianyu.py

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@Contact :   liuyuqi.gov@msn.cn
@Time    :   2023/09/23 00:38:01
@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
@Desc    :   闲鱼
'''
import os,sys,re,csv
import requests
from bs4 import BeautifulSoup
from lxml import etree

class Xianyu(object):
    
    def __init__(self):
        self.sess = requests.Session()

    def crawl(self, keyword):
        payload = {
            "st_edtime ": 1,  # 最新发布
            "_input_charset": "utf8",
            "search_type": "item",
            "q": self.keyword,
            "page": self.page
            # "start": minPrice,  # 价格范围
            # "end": maxPrice,
        }
        try:
            rep = self.sess.get(url="https://s.2.taobao.com/list/?", params=payload)
            rep.encoding = rep.apparent_encoding
            res = rep.text
            return res
        except Exception as e:
            print('error' * 22)
            return False
        self.save_to_csv()
        pass

    def save_to_csv(self, filename, data):
        if not os.path.exists('data'):
            os.mkdir('data')
        with open(filename, 'w') as f:
            csv_header = ['title', 'price', 'location', 'url']
            writer = csv.DictWriter(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            writer.writerow(csv_header)
            writer.writerows(data)

if __name__ == "__main__":
    if not os.path.exists('data'):
        os.mkdir('data')
    keywords = ['羽毛球拍', 'iphone']
    for keyword in keywords:
        xianyu = Xianyu()
        xianyu.crawl(keyword)