go on

.gitignore

@@ -81,3 +81,4 @@ vignettes/*.pdf
 .httr-oauth
 
 /images
+/crawl_xiaohua/data

crawl_xiaohua/conf/config.json

@@ -1,3 +1,4 @@
-[
-
-]
+{
+    "cookie": "",
+    "indexPage": "123650"
+}

crawl_xiaohua/crawl_xiaohua/crawl_xiaohua.py

@@ -11,6 +11,7 @@ from contextlib import closing
 import os
 import random
 import time
+from crawl_xiaohua.libs.json_conf import JsonConf
 import requests
 from crawl_xiaohua import api
 import bs4
@@ -34,7 +35,9 @@ class CrawlXiaohua():
     def __init__(self):
         self.s = requests.Session()
         self.s.headers.update(headers)
-        self.indexPage = "123655"
+        self.jsonConf = JsonConf()
+        self.conf = self.jsonConf.load()
+        self.indexPage = self.conf.get('indexPage')
         # self.s.cookies.update(JsonConf().get_cookies())
 
     def crawl(self):
@@ -56,6 +59,7 @@ class CrawlXiaohua():
             self.downloadPic(imgs[i], titleDesc + "_" + self.indexPage + "_" +
                              str(i) + "." + imgs[0].split('.')[-1])
 
+        self.jsonConf.set({"indexPage": self.indexPage})
         # 下一页
         nextPan = str.split(resHtml.find(
             'a', {'id': 'hylPrev'})["href"], r"/")[2]

crawl_xiaohua/crawl_xiaohua/libs/json_conf.py

@@ -6,56 +6,58 @@
 @License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
 @Desc    :   yaml util
 '''
-import os,json
+import os
+import json
 
-config_path = "conf/config.json"
 
 class JsonConf:
-    '''json配置文件类'''
-    @staticmethod
-    def save(data):
-        global config_path
-        with open(config_path, 'w') as json_file:
+    def __init__(self, config_path="conf/config.json"):
+        self.config_path = config_path
+
+    def save(self, data):
+        with open(self.config_path, 'w') as json_file:
             json_file.write(json.dumps(data, indent=4))
 
-    @staticmethod
-    def load():
-        global config_path
-        if not os.path.exists(config_path):
-            with open(config_path, 'w') as json_file:
+    def load(self):
+        if not os.path.exists(self.config_path):
+            with open(self.config_path, 'w') as json_file:
                 pass
-        with open(config_path, encoding="utf-8") as json_file:
+        with open(self.config_path, encoding="utf-8") as json_file:
             try:
                 data = json.load(json_file)
             except Exception as e:
                 if(str(e).index("utf-8-sig") > 0):
-                    with open(config_path, encoding="utf-8-sig") as json_file:
+                    with open(self.config_path, encoding="utf-8-sig") as json_file:
                         data = json.load(json_file)
                         return data
                 else:
                     print(e)
             return data
 
-    @staticmethod
-    def set(data_dict):
-        json_obj = JsonConf.load()
+    def set(self, data_dict):
+        json_obj = self.load()
         for key in data_dict:
             json_obj[key] = data_dict[key]
-        JsonConf.save(json_obj)
+        self.save(json_obj)
         print(json.dumps(json_obj, indent=4))
 
-    @staticmethod
-    def get(key, default_val=""):
+    def get(self, key, default_val=""):
         '''
         配置文件获取key对象的值，如果没有设置就返回默认值
         '''
         try:
-            result = JsonConf.load()[key]
+            result = self.load()[key]
             return result
         except Exception as e:
             print(e)
             return default_val
 
+    def get(self, jsonData, key, default_val=""):
+        try:
+            return jsonData[key]
+        except Exception as e:
+            return default_val
+
     @staticmethod
     def get(jsonData, key, default_val=""):
         try:

crawl_xiaohua/test/test_json.py

@@ -0,0 +1,7 @@
+from crawl_xiaohua.libs.json_conf import JsonConf
+if __name__ == '__main__':
+
+    config = {"cookie": "小舟", "indexPage": "123655"}
+    jsonConf = JsonConf(config_path="conf/config.json")
+    jsonConf.set(config)
+    print(JsonConf.get("address", "default_val"))