settings.py 3.2 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394
  1. # -*- coding: utf-8 -*-
  2. # Scrapy settings for house_spider project
  3. #
  4. # For simplicity, this file contains only settings considered important or
  5. # commonly used. You can find more settings consulting the documentation:
  6. #
  7. # https://doc.scrapy.org/en/latest/topics/settings.html
  8. # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
  9. # https://doc.scrapy.org/en/latest/topics/spider-middleware.html
  10. BOT_NAME = 'house_spider'
  11. SPIDER_MODULES = ['house_spider.spiders']
  12. NEWSPIDER_MODULE = 'house_spider.spiders'
  13. # Crawl responsibly by identifying yourself (and your website) on the user-agent
  14. #USER_AGENT = 'house_spider (+http://www.yourdomain.com)'
  15. # Obey robots.txt rules
  16. ROBOTSTXT_OBEY = True
  17. # Configure maximum concurrent requests performed by Scrapy (default: 16)
  18. #CONCURRENT_REQUESTS = 32
  19. # Configure a delay for requests for the same website (default: 0)
  20. # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
  21. # See also autothrottle settings and docs
  22. #DOWNLOAD_DELAY = 3
  23. # The download delay setting will honor only one of:
  24. #CONCURRENT_REQUESTS_PER_DOMAIN = 16
  25. #CONCURRENT_REQUESTS_PER_IP = 16
  26. # Disable cookies (enabled by default)
  27. #COOKIES_ENABLED = False
  28. # Disable Telnet Console (enabled by default)
  29. #TELNETCONSOLE_ENABLED = False
  30. # Override the default request headers:
  31. #DEFAULT_REQUEST_HEADERS = {
  32. # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  33. # 'Accept-Language': 'en',
  34. #}
  35. # Enable or disable spider middlewares
  36. # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
  37. #SPIDER_MIDDLEWARES = {
  38. # 'house_spider.middlewares.HouseSpiderSpiderMiddleware': 543,
  39. #}
  40. # Enable or disable downloader middlewares
  41. # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
  42. #DOWNLOADER_MIDDLEWARES = {
  43. # 'house_spider.middlewares.HouseSpiderDownloaderMiddleware': 543,
  44. #}
  45. # Enable or disable extensions
  46. # See https://doc.scrapy.org/en/latest/topics/extensions.html
  47. #EXTENSIONS = {
  48. # 'scrapy.extensions.telnet.TelnetConsole': None,
  49. #}
  50. # Configure item pipelines
  51. # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
  52. ITEM_PIPELINES = {
  53. 'house_spider.pipelines.HouseSpiderPipeline': 300,
  54. 'house_spider.pipelines.LianjiaVillageSavePipeline': 301,
  55. }
  56. # Enable and configure the AutoThrottle extension (disabled by default)
  57. # See https://doc.scrapy.org/en/latest/topics/autothrottle.html
  58. #AUTOTHROTTLE_ENABLED = True
  59. # The initial download delay
  60. #AUTOTHROTTLE_START_DELAY = 5
  61. # The maximum download delay to be set in case of high latencies
  62. #AUTOTHROTTLE_MAX_DELAY = 60
  63. # The average number of requests Scrapy should be sending in parallel to
  64. # each remote server
  65. #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
  66. # Enable showing throttling stats for every response received:
  67. #AUTOTHROTTLE_DEBUG = False
  68. # Enable and configure HTTP caching (disabled by default)
  69. # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
  70. #HTTPCACHE_ENABLED = True
  71. #HTTPCACHE_EXPIRATION_SECS = 0
  72. #HTTPCACHE_DIR = 'httpcache'
  73. #HTTPCACHE_IGNORE_HTTP_CODES = []
  74. #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
  75. MONGO_HOST = "127.0.0.1" # 主机IP
  76. MONGO_PORT = 27017 # 端口号