crawl_xiaohua/xiaohua/main.py

#coding=utf-8
'''
Created on 2017年7月15日
@vsersion:python3.6
@author: liuyuqi
'''
import csv
import os
from time import sleep
import random
from urllib import request

project_dir="C:/Users/dell/Desktop/xiaohua-crawl"
img_dir=project_dir+"/images"
data_dir=project_dir+"/data"

def downloadImg(imgUrl,fileName):
    try:  
        headers = { 
        'User-Agent' : "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
        'Referer':'http://www.xiaohuar.com'
             }
        req=request.Request(url=imgUrl)
        for i in headers:
            req.add_header(i,headers[i])
        res=request.urlopen(req)
        with open(img_dir+"/"+fileName+imgUrl[-4:], "wb") as code:
            code.write(res.read())
#         sleep(random.randint(1,5))
    except Exception as err:  
        print(err)  
    finally:  
        print("pic："+ fileName+".jpg")

def __init__():
    if(os.path.exists(img_dir)!=True):
                os.mkdir(img_dir)
    if(os.path.exists(data_dir)!=True):
                os.mkdir(data_dir)

def main():
    file=data_dir+"/result.csv"
    with open(file, 'r') as f:
#         data=csv.reader(f, csv.excel_tab)
        data=csv.reader(f)
        for row in data:
            imgUrl=""
            fileName=""
            for i in range(len(row)):
                fileName=row[4]+"-"+row[3]
                imgUrl="http://www.xiaohuar.com"+row[2]
            
            downloadImg(imgUrl,fileName)
__init__()
main()