crawl_electricity/crawl_electricity.py

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@Contact :   liuyuqi.gov@msn.cn
@Time    :   2021/10/10 07:24:07
@License :   Copyright © 2017-2020 liuyuqi. All Rights Reserved.
@Desc    :   全球电力数据爬虫
https://www.iea.org/fuels-and-technologies/electricity
'''
import re
import sys
import os
import time
import requests
import pandas as pd
import numpy as np

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}
res = requests.get(r'https://api.iea.org/stats/countries', headers=headers)

# 获取所有国家
countries = pd.DataFrame(res.json())
countries = countries.dropna(subset=["region"])

dataList = []
for i, country in enumerate(countries["stats"]):
    params = {
        "countries": country,
        "startYear": 1990, }

    res = requests.get(
        r'https://api.iea.org/stats/indicator/TotElecCons', headers=headers, params=params)
    df = pd.DataFrame(res.json())
    try:
        data = df[["year"]].copy()
        data["country"] = df["value"]
        data.set_index("year", inplace=True)
        dataList.append(data)
        print(f"{i}")
    except:
        continue

results = pd.concat(dataList, axis=1).fillna(0).loc[:"2018"]
results.to_csv("results.csv")