crawl_yfsp/download.py

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@Contact :   liuyuqi.gov@msn.cn
@Time    :   2025/11/22 13:07:40
@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
@Desc    :   pip install PySocks

'''


import requests
import os

def download():
    with open('image-url-1.md', 'r', encoding='utf-8') as f:
        lines = f.readlines()
    
    if not os.path.exists('downloads'):
        os.makedirs('downloads')
    
    sess = requests.Session()
    # sess.headers.update({
    #     'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    #     'accept-language': 'en-US,en;q=0.9',
    #     'cache-control': 'max-age=0',
    #     'if-modified-since': 'Fri, 04 Oct 2024 08:39:09 GMT',
    #     'if-none-match': '"9d88c0e13816db1:0"',
    #     'dnt': '1',
    #     'priority': 'u=0, i',
    #     'sec-ch-ua': '"Chromium";v="142", "Microsoft Edge";v="142", "Not_A Brand";v="99"',
    #     'sec-ch-ua-mobile': '?0',
    #     'sec-ch-ua-platform': '"Windows"',
    #     'sec-fetch-dest': 'document',
    #     'sec-fetch-mode': 'navigate',
    #     'sec-fetch-site': 'none',
    #     'sec-fetch-user': '?1',
    #     'upgrade-insecure-requests': '1',
    #     'user-agent': 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0'
    # })


    # set proxy: tcp://localhost:12180
    sess.proxies.update({
        'http': 'socks5h://localhost:12180',
        'https': 'socks5h://localhost:12180'
    })

    for line in lines:
        url = line.strip()
        if url:
            try:
                response = sess.get(url)
                response.raise_for_status()
                
                filename = os.path.join('downloads', url.split('/')[-1])
                with open(filename, 'wb') as img_file:
                    img_file.write(response.content)
                
                print(f'Downloaded: {url} -> {filename}')
            except Exception as e:
                print(f'Failed to download {url}: {e}')


if __name__=='__main__':
    download()