crawl_secondhand/crawl_house.py

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@Contact :   liuyuqi.gov@msn.cn
@Time    :   2023/09/22 17:42:53
@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
@Desc    :   
'''

from scrapy import cmdline
from scrapy.utils.project import get_project_settings
from scrapy.scheduler import Scheduler, CrawlerProcess, CrawlerRunner
from scrapy.utils.project import reactor, defer
from twisted.internet import CrawlerRunner
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging

import os,crawl_house

configure_loging()
runner = CrawlerRunner(get_project_settings())

@defer.inlineCallbacks
def crawl():
    # yield runner.crawl(crawl_house)
    cmdline.execute('scrapy genspider -o houses.csv houses.com'.split())
    reactor.stop()

if __name__=='__main__':
    crawl()
    reactor.run()