crawl_price/config/settings.py

import os
from dotenv import load_dotenv

load_dotenv()

DATABASE_TYPE = os.getenv('DB_TYPE', 'sqlite').lower()

DATABASE_CONFIG = {
    'type': DATABASE_TYPE,
    'sqlite': {
        'path': os.getenv('SQLITE_PATH', 'data/price_crawler.db'),
    },
    'mysql': {
        'host': os.getenv('DB_HOST', 'localhost'),
        'port': int(os.getenv('DB_PORT', 3306)),
        'user': os.getenv('DB_USER', 'root'),
        'password': os.getenv('DB_PASSWORD', ''),
        'database': os.getenv('DB_NAME', 'price_crawler'),
        'charset': 'utf8mb4'
    }
}

CRAWLER_CONFIG = {
    'taobao': {
        'base_url': 'https://s.taobao.com',
        'search_url': 'https://s.taobao.com/search',
        'headers': {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Referer': 'https://www.taobao.com',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        },
        'timeout': 30,
        'retry_times': 3,
        'retry_delay': 2,
        'delay_range': (1, 3),
    },
    'jd': {
        'base_url': 'https://search.jd.com',
        'search_url': 'https://search.jd.com/Search',
        'headers': {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Referer': 'https://www.jd.com',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        },
        'timeout': 30,
        'retry_times': 3,
        'retry_delay': 2,
        'delay_range': (1, 3),
    },
    'alibaba1688': {
        'base_url': 'https://s.1688.com',
        'search_url': 'https://s.1688.com/selloffer/offer_search.htm',
        'headers': {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Referer': 'https://www.1688.com',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        },
        'timeout': 30,
        'retry_times': 3,
        'retry_delay': 2,
        'delay_range': (1, 3),
    }
}

LOGGING_CONFIG = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'level': 'INFO',
            'formatter': 'standard'
        },
        'file': {
            'class': 'logging.handlers.RotatingFileHandler',
            'level': 'INFO',
            'formatter': 'standard',
            'filename': 'logs/crawler.log',
            'maxBytes': 1024*1024*10,
            'backupCount': 5,
            'encoding': 'utf-8'
        }
    },
    'loggers': {
        '': {
            'handlers': ['console', 'file'],
            'level': 'INFO',
            'propagate': True
        }
    }
}