Browse Source

add python实现

liuyuqi-dellpc 2 months ago
parent
commit
208355a1f5
3 changed files with 269 additions and 0 deletions
  1. 25 0
      python/conf/config.json
  2. 204 0
      python/crawl_github.py
  3. 40 0
      python/template.html

+ 25 - 0
python/conf/config.json

@@ -0,0 +1,25 @@
+{
+    "notify.starttime": "11:47:33",
+    "mail.user": "system@yoqi.me",
+    "mail.pwd": "F4nzH1Qbifj542mB1ewD",
+    "mail.toUser": "liuyuqi.gov@msn.cn",
+    "mail.smtp": "smtp.qiye.aliyun.com",
+    "crawl.url": "https://e.juejin.cn/resources/github",
+    "crawl.parm": {
+        "category": "trending",
+        "period": "day",
+        "lang": "all",
+        "offset": 0,
+        "limit": 30
+    },
+    "crawl.parm2": {
+        "category": "upcome",
+        "period": "day",
+        "java": "all",
+        "offset": 0,
+        "limit": 30
+    },
+    "db.mysql.userName": "crawler",
+    "db.mysql.password": "B97NMV14",
+    "db.mysql.url": "jdbc:mysql://h5.yoqi.me:3306/crawler?useSSL=false&useUnicode=true&characterEncoding=utf8"
+}

+ 204 - 0
python/crawl_github.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+'''
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2022/03/11 17:03:00
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   github每日通知
+'''
+
+from datetime import datetime
+import json
+import os
+import smtplib
+import requests
+from email.mime.text import MIMEText
+# import pymysql as pym
+
+config_path = r"python/conf/config.json"
+s = requests.Session()
+
+
+class JsonConf:
+    '''json配置文件类'''
+    @staticmethod
+    def save(data):
+        global config_path
+        with open(config_path, 'w') as json_file:
+            json_file.write(json.dumps(data, indent=4))
+
+    @staticmethod
+    def load():
+        global config_path
+        if not os.path.exists(config_path):
+            with open(config_path, 'w') as json_file:
+                pass
+        with open(config_path, encoding="utf-8") as json_file:
+            try:
+                data = json.load(json_file)
+            except Exception as e:
+                print(e)
+                data = {}
+            return data
+
+    @staticmethod
+    def set(data_dict):
+        json_obj = JsonConf.load()
+        for key in data_dict:
+            json_obj[key] = data_dict[key]
+        JsonConf.save(json_obj)
+        print(json.dumps(json_obj, indent=4))
+
+    @staticmethod
+    def get(key, default_val=""):
+        '''
+        配置文件获取key对象的值,如果没有设置就返回默认值
+        '''
+        try:
+            result = JsonConf.load()[key]
+            return result
+        except Exception as e:
+            print(e)
+            return default_val
+
+
+class GithubTopEntity:
+    '''github每日排行榜'''
+
+    def __init__(self,  data):
+        self.data = []
+
+    @staticmethod
+    def fromJson(self, json_str):
+        self.data = Data.fromJson(json_str)
+
+    class Data:
+        def __init__(self):
+            self.url = ""
+            self.description = ""
+            self.starCount = 0
+            self.forkCount = 0
+
+        @staticmethod
+        def fromJson(self, json_str):
+            self.data = json.loads(json_str)
+            self.url = self.data["url"]
+            self.description = self.data["description"]
+            self.starCount = self.data["stargazers_count"]
+            self.forkCount = self.data["forks_count"]
+
+
+class CrawlGithub:
+    '''github每日通知类'''
+
+    def __init__(self):
+        self.res = []
+        self.mailUser = JsonConf.get("mail.user")
+        self.mailPass = JsonConf.get("mail.pwd")
+        self.mailTo = JsonConf.get("mail.toUser")
+        self.mailServer = JsonConf.get("mail.smtp")
+        # mailPort=JsonConf.get("mailPort")
+
+        self.crawlUrl = JsonConf.get("crawl.url")
+        self.crawlParm = JsonConf.get("crawl.parm")
+        self.crawlParm2 = JsonConf.get("crawl.parm2")
+
+        self.mailSubject = "小舟日报-Github开源 " + datetime.now().strftime("%Y-%m-%d")
+        self.mailBody = """
+        <!DOCTYPE html>
+            <html lang="en">
+
+            <head>
+                <meta charset="UTF-8">
+                <meta http-equiv="X-UA-Compatible" content="IE=edge">
+                <meta name="viewport" content="width=device-width, initial-scale=1.0">
+                <title>Document</title>
+            </head>
+
+            <body>
+                <div style="background-color: rgb(9, 230, 64);">
+                    <div style="line-height:20px;margin-left: 20px;height: 100px;display: inline-block; ">
+                        <div style="font-size: 20px;color:white;display: inline-block;margin-top: 28px;margin-bottom: 6px">
+                            芝舟开源</div>
+                        <div style="color:white;">源码分享,技术深度剖析。</div>
+                    </div>
+                </div>
+         """
+
+    def crawl_github(self):
+        """
+        param :
+        return:
+        """
+        print("start------" + datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+        self.getGithubUP()
+        self.getGithubHot()
+        self.sendMail()
+        print("end------" + datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+
+    def saveToDB(self):
+        pass
+        # db = pym.connect(DB_HOST, DB_USER, DB_PASS, DB_NAME, charset='utf8')
+
+
+    def sendMail(self):
+        mailserver = self.mailServer
+        username = self.mailUser
+        password = self.mailPass
+        self.mailBody += """
+                <p style="color: rgb(170, 170, 170); ">&amp;copyright; 2022 &nbsp;liuyuqi.gov@msn.cn</p>
+
+                <table cellpadding="0" cellspacing="0" border="0" width="100%">
+                    <tbody>
+                        <tr>
+                            <td align="center" style="padding-top: 10px;">
+                                <div style="border-top:1px solid #ddd;padding:10px;"><a href="https://git.yoqi.me/todo"
+                                        target="_blank"
+                                        style="display: table-cell;display:inline-block;background:#ddd; border-radius:4px;padding: 3px 15px;color:#a6a6a6;text-decoration:none;font-size:12px;">点击这里取消订阅<br>click
+                                        to unsubscribe</a></div>
+                            </td>
+                        </tr>
+                    </tbody>
+                </table>
+            </body>
+
+            </html>
+        """
+        msg = MIMEText(self.mailBody, 'html', 'utf-8')
+        msg['Subject'] = self.mailSubject
+        msg['From'] = self.mailUser
+        msg['To'] = self.mailTo
+        s = smtplib.SMTP(mailserver)
+        s.login(username, password)
+        print(s.send_message(msg))
+        s.quit()
+
+    def getGithubUP(self):
+        res = s.post(self.crawlUrl, data=self.crawlParm2)
+        self.mailBody += """
+                <h2>今日新出项目<h2>
+        """
+        for i in res.json()['data']:
+            self.mailBody += """
+                <p><strong> {i} </strong></p>
+                <p></p>
+                <p></p>
+                <p></p>
+                """
+
+    def getGithubHot(self):
+        res = s.post(self.crawlUrl, data=self.crawlParm)
+        self.mailBody += """
+                <h2>今日热门项目<h2>
+        """
+        for i in res.json()['data']:
+            self.mailBody += """
+                <p><strong> {i} </strong></p>
+                <p></p>
+                <p></p>
+                <p></p>
+                """
+
+if __name__ == '__main__':
+    crawl = CrawlGithub()
+    crawl.crawl_github()

+ 40 - 0
python/template.html

@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Document</title>
+</head>
+
+<body>
+    <div style="background-color: rgb(9, 230, 64);">
+        <div style="line-height:20px;margin-left: 20px;height: 100px;display: inline-block; ">
+            <div style="font-size: 20px;color:white;display: inline-block;margin-top: 28px;margin-bottom: 6px">
+                芝舟开源</div>
+            <div style="color:white;">源码分享,技术深度剖析。</div>
+        </div>
+    </div>
+    <p><strong>1. 如何接单: </strong></p>
+    <p>雇主有需求时,实现网工作人员会与首先与雇主进行需求的前期沟通,确认需求靠谱后,实现网工作人员根据需求中的&amp;ldquo;技术关键词&amp;rdquo;匹配到你的简历并联系到你。</p>
+    <p>技术顾问也可根据雇主发布的需求主动申请(http://shixian.com/jobs)投递职位,申请后实现网人工帮助对接双方需求。</p>
+    <p>&nbsp;</p>
+
+    <p style="color: rgb(170, 170, 170); ">&amp;copyright; 2022 &nbsp;liuyuqi.gov@msn.cn</p>
+
+    <table cellpadding="0" cellspacing="0" border="0" width="100%">
+        <tbody>
+            <tr>
+                <td align="center" style="padding-top: 10px;">
+                    <div style="border-top:1px solid #ddd;padding:10px;"><a href="https://git.yoqi.me/todo"
+                            target="_blank"
+                            style="display: table-cell;display:inline-block;background:#ddd; border-radius:4px;padding: 3px 15px;color:#a6a6a6;text-decoration:none;font-size:12px;">点击这里取消订阅<br>click
+                            to unsubscribe</a></div>
+                </td>
+            </tr>
+        </tbody>
+    </table>
+</body>
+
+</html>