完成coding

.github/workflows/build.yml

@@ -0,0 +1,53 @@
+name: Publish Installers
+
+on:
+  workflow_dispatch: ~
+  push:
+    branches: [master]
+    tags: [v*]
+
+jobs:
+  build:
+    name: Build ${{ matrix.os }} Package
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix: 
+        os: [windows-2019, ubuntu-20.04]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Set Release Version
+        id: get_version
+        shell: bash
+        run: |
+          echo "::set-output name=hash::$(git rev-parse --short HEAD)"
+          echo "::set-output name=date::$(date +%Y%m%d)"
+
+      - name: Set Up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+          cache: pip
+          cache-dependency-path: '**/requirements*.txt'
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip wheel setuptools
+          pip install -r requirements.txt
+          python -m pip install pyinstaller
+
+      - name: Build Package
+        run: |
+          python -m PyInstaller -F -c  -i favicon.ico --name repo_sync main.py
+
+      - name: Update to ali oss
+        uses: yizhoumo/setup-ossutil@v1
+        with:
+          endpoint: oss-cn-qingdao.aliyuncs.com
+          access-key-id: ${{ secrets.OSS_KEY_ID }}
+          access-key-secret: ${{ secrets.OSS_KEY_SECRET }}
+          
+      - name: cp files to aliyun
+        run: |
+          ossutil cp -rf dist/ oss://yoqi-software/develop/repo_sync/${{ steps.get_version.outputs.date }}-${{ steps.get_version.outputs.hash }}/

.gitignore

@@ -58,3 +58,4 @@ docs/_build/
 # PyBuilder
 target/
 
+*.csv

README.md

@@ -21,9 +21,8 @@
 python repo_sync.py --debug true --repo data/repo.txt --type github
 ```
 
-把 data/repo.txt 中的项目同步到 gitlab 上：
 ```
-python repo_sync.py --type gitlab
+python main.py pull --platform coding --repo_path F:\workspace\python\crawl_github
 ```
 
 ## 计划任务

conf/config.yml

@@ -1,19 +0,0 @@
-zhizhou:
-  token: xx1
-  user: lyq
-
-github:
-  token: xx1
-  user: jianboy
-
-coding:
-  token: xx2
-  user: heavyrain/python-note
-
-gitlab:
-  token: xx3
-  user: jianboy
-
-gitee:
-  token: xx4
-  user: jianboy

data/repo.txt

@@ -1 +0,0 @@
-https://git.yoqi.me/lyq/repo_sync

repo_sync/__init__.py

@@ -1,6 +1,20 @@
-from repo_sync.base_repo import BaseRepo
-from repo_sync.sync_utils import SyncUtils
+from .repo_sync import RepoSync
+from .version import __version__
+from .options import parser_args
+import sys
 
-def main():
-    sync_utils = SyncUtils()
-    sync_utils.run()
+def main(argv=None):
+    """Main entry point of the program"""
+    try:
+        args = parser_args()
+        if args.get('version'):
+            print(__version__)
+            sys.exit(0)
+        if args.get('command', '') == '':
+            # logging.error("command is empty")
+            # argparser.print_help()
+            sys.exit(1)
+        rs = RepoSync(args)
+        rs.run()
+    except KeyboardInterrupt:
+        sys.exit('\nERROR: Interrupted by user')

repo_sync/coding_repo.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-'''
-@Contact :   liuyuqi.gov@msn.cn
-@Time    :   2023/04/27 03:16:43
-@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
-@Desc    :   coding repo
-'''
-
-from repo_sync.base_repo import BaseRepo
-import logging
-import os
-class CodingRepo(BaseRepo):
-    
-    def __init__(self, user_name, repo_name, logger : logging.Logger):
-        super(CodingRepo, self).__init__(user_name, repo_name,logger)
-        
-    def sync(self):
-        '''
-        sync repo
-        '''
-        with open("conf/config.yml", "r") as f:
-            config = yaml.load(f, Loader=yaml.FullLoader)
-        zUser = config["zhizhou"]["user"]
-        if zUser == self.user_name:
-            self.repo_url = "hhttps://e.coding.net/"+config["coding"]["user"]+"/"+self.repo_name+".git"
-            self.logger.info("sync repo: %s", self.repo_name)
-            self.sess.headers={
-                "Authorization": "token "+config["coding"]["token"],
-            }
-            if self.repos==None:
-                self.repos = self.sess.get("https://e.coding.net/api/user/projects").json()
-            # if repo is not exist, create it
-            if self.repo_name not in [repo["name"] for repo in self.repos]:
-                self.logger.info("create repo: %s", self.repo_name)
-                self.sess.post("https://e.coding.net/api/user/projects", json={"name": self.repo_name})
-            os.system("git push %s %s" % (self.repo_url, self.repo_path))
-        else:
-            self.logger.info("not sync repo: %s", self.repo_name)

repo_sync/gitee_repo.py

@@ -1,51 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-'''
-@Contact :   liuyuqi.gov@msn.cn
-@Time    :   2023/04/27 05:12:59
-@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
-@Desc    :   gitee repo
-'''
-
-from repo_sync.base_repo import BaseRepo
-import logging
-import os
-import yaml
-
-class GiteeRepo(BaseRepo):
-    
-    def __init__(self, user_name, repo_name, logger : logging.Logger):
-        super(GiteeRepo, self).__init__(user_name, repo_name,logger)
-        self.repos=None
-
-    def sync(self):
-        ''' sync repo
-        '''
-        # read conf/config.yml
-        with open("conf/config.yml", "r") as f:
-            config = yaml.load(f, Loader=yaml.FullLoader)
-        # get github token
-        zUser = config["zhizhou"]["user"]
-        if zUser == self.user_name:
-            self.repo_url = "https://gitee.com"+"/"+config["gitee"]["user"]+"/"+self.repo_name+".git"
-            self.logger.info("sync repo: %s", self.repo_name)
-            self.sess.headers={
-                "Authorization": "token "+config["gitee"]["token"],
-            }
-            if self.repos==None:
-                res = self.sess.get("https://api.gitee.com/user/repos")
-                if res.status_code != 200:
-                    self.logger.error("get gitee repo list failed, status_code: %s, please check the token.", res.status_code)
-                    return
-                self.repos = res.json()
-            # if repo is not exist, create it
-            if self.repo_name not in [repo["name"] for repo in self.repos]:
-                self.logger.info("create repo: %s", self.repo_name)
-                self.sess.post("https://api.gitee.com/user/repos", json={"name": self.repo_name})
-            # git push it
-            os.chdir(self.user_name+"/"+self.repo_name)
-            os.system("git remote add origin2 "+self.repo_url)
-            os.system("git push -u origin2 master")
-            os.chdir("../..")
-        else:
-            self.logger.info("not sync repo: %s", self.repo_name)

repo_sync/github_repo.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-'''
-@Contact :   liuyuqi.gov@msn.cn
-@Time    :   2023/04/27 02:59:42
-@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
-@Desc    :   github repo
-'''
-import os,sys,re
-import logging
-import argparse
-import yaml
-
-from repo_sync.base_repo import BaseRepo
-
-class GitHubRepo(BaseRepo):
-    '''
-    GitHubRepo class
-    '''
-    def __init__(self, user_name, repo_name, logger : logging.Logger):
-        super(GitHubRepo, self).__init__(user_name, repo_name,logger)
-        self.repos=None
-
-    def sync(self):
-        '''
-        sync repo
-        '''
-        # read conf/config.yml
-        with open("conf/config.yml", "r") as f:
-            config = yaml.load(f, Loader=yaml.FullLoader)
-        # get github token
-        zUser = config["zhizhou"]["user"]
-        if zUser == self.user_name:
-            self.repo_url = "https://github.com"+"/"+config["github"]["user"]+"/"+self.repo_name+".git"
-            self.logger.info("sync repo: %s", self.repo_name)
-            self.sess.headers={
-                "Authorization": "token "+config["github"]["token"],
-            }
-            if self.repos==None:
-                res = self.sess.get("https://api.github.com/user/repos")
-                if res.status_code != 200:
-                    self.logger.error("get github repo list failed, status_code: %s, please check the token.", res.status_code)
-                    return
-                self.repos = res.json()
-            # if repo is not exist, create it
-            if self.repo_name not in [repo["name"] for repo in self.repos]:
-                self.logger.info("create repo: %s", self.repo_name)
-                self.sess.post("https://api.github.com/user/repos", json={"name": self.repo_name})
-            # git push it
-            os.chdir(self.user_name+"/"+self.repo_name)
-            os.system("git remote add origin2 "+self.repo_url)
-            os.system("git push -u origin2 master")
-            os.chdir("../..")
-        else:
-            self.logger.info("not sync repo: %s", self.repo_name)

repo_sync/gitlab_repo.py

@@ -1,41 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-'''
-@Contact :   liuyuqi.gov@msn.cn
-@Time    :   2023/04/27 04:23:34
-@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
-@Desc    :   github repo
-'''
-from repo_sync.base_repo import BaseRepo
-import logging
-import yaml
-
-class GitlabRepo(BaseRepo):
-    '''
-    GitlabRepo class
-    '''
-    def __init__(self, user_name, repo_name, logger : logging.Logger):
-        super(GitlabRepo, self).__init__(user_name, repo_name,logger)
-    
-    def sync(self):
-        ''' sync repo '''
-        # read conf/config.yml
-        with open("conf/config.yml", "r") as f:
-            config = yaml.load(f, Loader=yaml.FullLoader)
-        # get github token
-        zUser = config["zhizhou"]["user"]
-        if zUser == self.user_name:
-            self.repo_url="https://gitlab.com/"+config["gitlab"]["user"]+"/"+self.repo_name+".git"
-            self.logger.info("sync repo: %s", self.repo_name)
-            self.sess.headers={
-                "PRIVATE-TOKEN": config["gitlab"]["token"],
-            }
-            if self.repos==None:
-                self.repos = self.sess.get("https://gitlab.com/api/v4/projects").json()
-            # if repo is not exist, create it
-            if self.repo_name not in [repo["name"] for repo in self.repos]:
-                self.logger.info("create repo: %s", self.repo_name)
-                self.sess.post("https://gitlab.com/api/v4/projects", json={"name": self.repo_name})
-            os.system("git push %s %s" % (self.repo_url, self.repo_path))
-        else:
-            self.logger.info("not sync repo: %s", self.repo_name)

repo_sync/options.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/11/01 00:01:04
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   命令行参数，或配置文件
+"""
+
+import argparse
+import os
+import shlex
+import dotenv
+from collections import OrderedDict
+from .utils.str_util import preferredencoding
+
+
+def parser_args(overrideArguments=None):
+    """解析参数"""
+
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument('-c', '--config', help='config file', default='config.ini')
+    argparser.add_argument(
+        'command',
+        help='command: ',
+        choices=['create', 'clone', 'push', 'delete', 'pull'],
+    )
+    argparser.add_argument('-d', '--debug', help='debug mode', action='store_true')
+    argparser.add_argument(
+        '-p',
+        '--platform',
+        help='set a platform',
+        choices=['github', 'gitee', 'gitlab', 'gogs', 'gitea', 'bitbucket', 'coding'],
+        default='github',
+    )
+    argparser.add_argument('-token', '--token', help='set a token')
+    argparser.add_argument(
+        '-repo_path', '--repo_path', help='set a repo'
+    )  # , default=os.getcwd())
+    args = argparser.parse_args()
+
+    # remove None
+    command_line_conf = OrderedDict(
+        {k: v for k, v in args.__dict__.items() if v is not None}
+    )
+
+    system_conf = user_conf = custom_conf = OrderedDict()
+    user_conf = _read_user_conf()
+
+    if args.config:
+        custom_conf = _read_custom_conf(args.config)
+
+    system_conf.update(user_conf)
+    system_conf.update(command_line_conf)
+    if args.command == None and args.extractor == None:
+        raise 'Error, please input cmd and extractor params11'
+    return system_conf
+
+
+def _read_custom_conf(config_path: str) -> OrderedDict:
+    """读取自定义配置文件 config.yaml"""
+
+    def compat_shlex_split(s, comments=False, posix=True):
+        if isinstance(s, str):
+            s = s.encode('utf-8')
+        return list(map(lambda s: s.decode('utf-8'), shlex.split(s, comments, posix)))
+
+    try:
+        with open(config_path, 'r', encoding=preferredencoding()) as f:
+            contents = f.read()
+            res = compat_shlex_split(contents, comments=True)
+    except Exception as e:
+        return []
+    return res
+
+
+def _read_user_conf() -> OrderedDict:
+    """读取用户配置文件: .env 文件"""
+    user_conf = OrderedDict()
+    dotenv_path = '.env'
+    if os.path.exists(dotenv_path):
+        user_conf = dotenv.dotenv_values(dotenv_path)
+    return OrderedDict(user_conf)

repo_sync/platform/__init__.py

@@ -0,0 +1,17 @@
+from .github import GithubIE
+from .bitbucket import BitbucketIE
+from .gitee import GiteeIE
+from .gitea import GiteaIE
+from .gitlab import GitlabIE
+from .gogs import GogsIE
+from .coding import CodingIE
+
+_ALL_CLASSES = [klass for name, klass in globals().items()
+                if name.endswith('IE')]
+
+
+def gen_extractor_classes():
+    """Return a list of supported extractors.
+    The order does matter; the first extractor matched is the one handling the URL.
+    """
+    return _ALL_CLASSES

repo_sync/platform/base_platform.py

@@ -0,0 +1,64 @@
+import requests,csv,os
+from repo_sync.repo import Repo
+
+class BasePlatform(object):
+    """base platform"""
+    
+    repo_list_path = 'repo_list.csv'
+
+    def __init__(self, username:str, token:str ) -> None:
+        """init"""
+        self.sess = requests.Session()
+        self.username = username
+        self.token = token
+        self.repos = []
+        self.sess.headers.update(
+            {
+                'Accept': 'application/json',
+                'Content-Type': 'application/json',
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
+                'Authorization': f'token {self.token}',
+            }
+        )
+        if os.path.exists(self.repo_list_path):
+            with open(self.repo_list_path, 'r', encoding='utf8') as f:
+                reader = csv.DictReader(f)
+                for row in reader:
+                    repo = Repo()
+                    repo.__dict__ = row
+                    self.repos.append(repo)
+
+    def create_repo(self, repo_name: str):
+        """create a repo"""
+        raise NotImplementedError('crawl not implemented')
+
+    def delete(self, repo_name: str):
+        """delete a repo, maybe request a confirm by input"""
+        raise NotImplementedError('crawl not implemented')
+    
+    def clone(self, repo_name: str):
+        """clone a repo"""
+        raise NotImplementedError('crawl not implemented')
+    
+    def pull(self, repo_path: str):
+        """pull a repo"""
+        raise NotImplementedError('crawl not implemented')
+
+    def push(self, repo_path: str):
+        """push a repo"""
+        raise NotImplementedError('crawl not implemented')
+
+    @classmethod
+    def suitable(cls, extractor: str) -> bool:
+        """check if this extractor is suitable for this platform"""
+        raise NotImplementedError('crawl not implemented')
+
+    def save_csv(self):
+        with open(self.repo_list_path, 'w', newline='') as f:
+            if len(self.repos) == 0:
+                print('repo list is empty, please delete repo_list.csv and try again')
+                return
+            writer = csv.DictWriter(f, fieldnames=self.repos[0].__dict__.keys(), lineterminator='\n')
+            writer.writeheader()
+            for repo in self.repos:
+                writer.writerow(repo.__dict__)

repo_sync/platform/bitbucket.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/11/08 14:59:46
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   bitbucket.com
+"""
+from .base_platform import BasePlatform
+import csv
+
+
+class BitbucketIE(BasePlatform):
+    """bitbucket extract"""
+
+    _host = 'https://api.bitbucket.org/2.0'
+    bitbucket_repo_list = 'bitbucket_repo_list.csv'
+
+    def __init__(self, username:str, token:str,host:str =None ,params: dict = None) -> None:
+        super().__init__(username=username,token=token)
+
+    def create_repo(self, repo_name: str):
+        """create a repo"""
+        pass
+
+    def delete(self, repo_name: str):
+        """delete a repo"""
+        pass
+
+    def get_repo_list(self, username: str):
+        """get repo list"""
+        pass
+
+    def clone(self):
+        pass
+    def pull(self, repo_path: str):
+        return super().pull(repo_path)
+    
+
+    def push(self):
+        pass
+
+    @classmethod
+    def suitable(cls, extractor: str) -> bool:
+        """check if this extractor is suitable for this platform"""
+        if extractor == 'bitbucket':
+            return True
+        else:
+            return False

repo_sync/platform/coding/__init__.py

@@ -0,0 +1 @@
+from .coding import CodingIE

repo_sync/platform/coding/coding.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/09/27 10:35:25
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   coding.net repo sync
+
+两种授权： token 授权，OAuth2.0 授权
+
+"""
+import os
+from repo_sync.platform.base_platform import BasePlatform
+from .project import Project
+from .repo import Repo
+
+class CodingIE(BasePlatform):
+    """coding util"""
+    client_id = ''
+    client_serect = ''
+    _host = 'https://e.coding.net'  # 新版接口统一地址
+
+    def __init__(self, username:str, token:str,host:str =None ,params: dict = None) -> None:
+        """init"""
+        super().__init__(username , token)
+        self.project_name = params.get('coding_project', '')
+
+    def create_project(self):
+        ''' createt a project '''
+        url = f'{self._host}/open-api'
+        data = {
+            'Action': 'CreateCodingProject',
+            'Name': '',
+            'DisplayName': '',
+            'Description': '',
+            'GitReadmeEnabled': False,
+            'VscType': 'git',
+            'CreateSvnLayout': False,
+            'Shared': 1,
+            'ProjectTemplateId': 'DEV_OPS',
+        }
+        r = self.sess.post(url, json=data)
+        if r.status_code == 200:
+            res_data = r.json()
+            return True
+        else:
+            return False
+
+    def delete_project(self):
+        url = f'{self._host}/open-api'
+        data = {
+            'Action': 'DeleteOneProject',
+            'ProjectId': 0,
+        }
+        r = self.sess.post(url, json=data)
+        if r.status_code == 200:
+            res_data = r.json()
+            return True
+        else:
+            return False
+
+    def get_project_list(self):
+        pass
+
+    def get_repo_list(self, username: str):
+        """get repo list"""
+        url = f'{self._host}/open-api'
+        data = {
+            'Action': 'DescribeTeamDepotInfoList',
+            'ProjectName': '',
+            'DepotName': '',
+            'PageNumber': 1,
+            'PageSize': 50
+        }
+        r = self.sess.post(url, json=data)
+    
+    def get_repo_info(self, repo_name: str):
+        """get repo list"""
+        url = f'{self._host}/open-api'
+        data = {
+            'Action': 'DescribeTeamDepotInfoList',
+            'ProjectName': self.project_name,
+            'DepotName': repo_name,
+            'PageNumber': 1,
+            'PageSize': 50
+        }
+        r = self.sess.post(url, json=data)
+        if r.status_code == 200:
+            res_data = r.json()
+            if res_data['Response']["DepotData"]["Page"]["TotalRow"] > 0:
+                DepotList = res_data['Response']["DepotData"]["Depots"]
+                depot = Repo(
+                    Id=DepotList[0]['Id'],
+                    Name=DepotList[0]['Name'],
+                    HttpsUrl=DepotList[0]['HttpsUrl'],
+                    ProjectId=DepotList[0]['ProjectId'],
+                    SshUrl=DepotList[0]['SshUrl'],
+                    WebUrl=DepotList[0]['WebUrl'],
+                    ProjectName=DepotList[0]['ProjectName'],
+                    Description=DepotList[0]['Description'],
+                    CreatedAt=DepotList[0]['CreatedAt'],
+                    GroupId=DepotList[0]['GroupId'],
+                    GroupName=DepotList[0]['GroupName']
+                )
+                return depot
+
+    def get_project_info(self)->Project:
+        url = f'{self._host}/open-api'
+        data = {
+            "Action": "DescribeCodingProjects",
+            "ProjectName": self.project_name,
+            "DepotName": "",
+            "PageNumber": 1,
+            "PageSize": 50
+            }
+        r = self.sess.post(url, json=data)
+        if r.status_code == 200:
+            res_data = r.json()
+            if res_data['Response']["Data"]["TotalCount"] > 0:
+                ProjectList = res_data['Response']["Data"]["ProjectList"]
+                projet = Project(
+                    Id=ProjectList[0]['Id'],
+                    Name=ProjectList[0]['Name'],
+                    DisplayName=ProjectList[0]['DisplayName'],
+                    Description=ProjectList[0]['Description'],
+                    TeamOwnerId=ProjectList[0]['TeamOwnerId'],
+                    TeamId=ProjectList[0]['TeamId']
+                )
+                return projet
+
+    def create_repo(self, repo_name: str):
+        """create a repo"""
+        # get project id
+        project = self.get_project_info()
+
+        url = f'{self._host}/open-api/repos'
+        data = {
+            "Action": "CreateGitDepot",
+            "ProjectId": project.Id,
+            "DepotName": repo_name,
+            "Shared": False,
+            "Description": ""
+            }
+        r = self.sess.post(url, json=data)
+        if r.status_code == 200:
+            print(f'create repo {repo_name} success', data,r.json())
+            return True
+        else:
+            return False
+
+    def delete(self, repo_name: str):
+        """delete a repo"""
+        repo = self.get_repo_info(repo_name=repo_name)
+        url = f'{self._host}/open-api/repos'
+        data = {
+            "Action": "DeleteGitDepot",
+            "DepotId": repo.Id
+            }
+        r = self.sess.post(url, json=data)
+        if r.status_code == 200:
+            print(f'delete repo {repo_name} success', data,r.json())
+            return True
+        else:
+            return False
+
+    def pull(self, local_repo_path: str):
+        ''' pull a repo from remote
+            Args: local_repo_path: local repo path
+         '''
+        if local_repo_path[-1] == os.path.sep:
+            local_repo_path = local_repo_path[:-1]
+        repo_name = local_repo_path.split(os.path.sep)[-1]
+        print(f'pull repo:{self.username}/{repo_name} from coding')
+        os.chdir(local_repo_path)
+        os.system('git remote remove origin_coding')
+        os.system(
+            f'git remote add origin_coding https://{self.username}:{self.token}@e.coding.net/{self.username}/{self.project_name}/{repo_name}.git'
+        )
+        os.system('git pull origin_coding')
+        os.system('git remote remove origin_coding')
+        os.chdir('..')
+        print('pull success')
+
+    def push(self, local_repo_path: str):
+        ''' push a local repo to remote
+            Args: local_repo_path: local repo path
+         '''
+        if local_repo_path[-1] == os.path.sep:
+            local_repo_path = local_repo_path[:-1]
+        repo_name = local_repo_path.split(os.path.sep)[-1]
+        print(f'push repo:{self.username}/{repo_name} to coding')
+        os.chdir(local_repo_path)
+
+        os.system('git remote remove origin_coding')
+        os.system(
+            f'git remote add origin_coding https://{self.username}:{self.token}@e.coding.net/{self.username}/{self.project_name}/{repo_name}.git'
+        )
+        os.system('git push -u origin_coding')
+        os.system('git remote remove origin_coding')
+        os.chdir('..')
+        print('push success')
+
+    def clone(self, repo_name: str):
+        pass
+
+    @classmethod
+    def suitable(cls, extractor: str) -> bool:
+        """check if this extractor is suitable for this platform"""
+        if extractor == 'coding':
+            return True
+        else:
+            return False

repo_sync/platform/coding/project.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+'''
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/09/27 11:15:55
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   project manger
+'''
+import os,sys,re,requests,json
+
+class Project(object):
+    
+    host="https://e.coding.net/open-api"   
+    def __init__(self, Id, Name, DisplayName, Description, TeamOwnerId, TeamId) -> None:
+        self.Id = Id
+        self.Name = Name
+        self.DisplayName = DisplayName
+        self.Description = Description
+        self.TeamOwnerId = TeamOwnerId
+        self.TeamId= TeamId
+
+    def create_project(self):
+        ''' 创建项目 '''
+        # https://{your-team}.coding.net/api/user/{user}/project
+        pass
+    
+    def delete_project(self):
+        ''' 删除项目 '''
+        # https://{your-team}.coding.net/api/user/{user}/project/{project}
+        pass
+    
+    def get_project(self):
+        ''' 查询项目信息 '''
+        # https://{your-team}.coding.net/api/user/{user}/project/{project}
+        url=self.host + "/open-api?Action=DescribeOneProject"
+        payload = {
+            
+        }
+            
+    def get_project_uesr(self):
+        ''' 查询项目成员 '''
+        # https://{your-team}.coding.net/api/user/{user}/project/{project}/members
+        pass
+ 

repo_sync/platform/coding/repo.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/09/27 10:48:38
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   
+"""
+import requests, json
+
+
+class Repo(object):
+    ''' Depot info, repo info '''
+    
+    host = 'https://e.coding.net/open-api'
+
+    def __init__(self,Id, Name, HttpsUrl, ProjectId, SshUrl, WebUrl, ProjectName, Description, CreatedAt, GroupId, GroupName):
+        self.Id = Id
+        self.Name = Name
+        self.HttpsUrl = HttpsUrl
+        self.ProjectId = ProjectId
+        self.SshUrl = SshUrl
+        self.WebUrl = WebUrl
+        self.ProjectName = ProjectName
+        self.Description = Description
+        self.CreatedAt = CreatedAt
+        self.GroupId = GroupId
+        self.GroupName = GroupName
+
+    def create_repo(self):
+        """创建项目"""
+        url = self.host + '/open-api?Action=CreateGitDepot'
+        payload = {
+            'ProjectId': 0,
+            'DepotName': '',
+            'Shared': True,
+            'Description': '',
+        }
+        r = self.sess.post(url, data=json.dumps(payload))
+
+    def delete_repo(self):
+        """删除项目"""
+        pass
+
+    def get_repo(self):
+        """查询项目信息"""
+        pass
+
+    def get_repo_user(self):
+        """查询项目成员"""
+        # https://{your-team}.coding.net/api/user/{user}/project/{project}/members
+

repo_sync/platform/gitea.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/11/09 19:17:57
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   
+"""
+
+
+class GiteaIE(object):
+    def __init__(self, username:str, token:str,host:str =None ,params: dict = None) -> None:
+        super().__init__(username=username,token=token)
+        self._host = 'https://git.yoqi.com'
+
+    def create_repo(self, repo_name: str):
+        pass
+
+    def delete(self, repo_name: str):
+        pass
+
+    def get_repo_list(self) -> list:
+        pass
+
+    def clone(self):
+        pass
+    def push(self):
+        pass
+    def pull(self, repo_path: str):
+        return super().pull(repo_path)
+    
+    @classmethod
+    def suitable(cls, extractor: str) -> bool:
+        """check if this extractor is suitable for this platform"""
+        if extractor == 'gitea':
+            return True
+        else:
+            return False

repo_sync/platform/gitee.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/11/09 17:40:42
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   gitee async
+
+"""
+from .base_platform import BasePlatform
+import csv
+import os
+from repo_sync.repo import Repo
+
+class GiteeIE(BasePlatform):
+    """gitee async"""
+
+    _host = 'https://gitee.com'
+    _api = _host + '/api/v5'
+
+    def __init__(self, username:str, token:str,host:str =None ,params: dict = None) -> None:
+        super().__init__(username=username, token=token)
+        self.sess.headers.update({'Content-Type': 'multipart/form-data'})
+
+    def create_repo(self, repo_name: str):
+        """create a repo"""
+        url = f'{self._api}/user/repos'
+        form_data = {'name': repo_name, 'private': True}
+        r = self.sess.post(url, params=form_data)
+        if r.status_code != 201:
+            print(
+                'create repo {} failed, status code {}'.format(repo_name, r.status_code)
+            )
+            return
+        print('create repo {} success'.format(repo_name))
+
+    def delete(self, repo_name: str):
+        """delete a repo"""
+        # print("delete repo:"+repo_name)
+        url = f'{self._api}/repos/{self.username}/{repo_name}'
+
+        response = self.sess.delete(url)
+        if response.status_code == 204:
+            print(f'Repository: {repo_name} deleted from gitee successfully!')
+        else:
+            print(
+                f'Failed to delete repository: {repo_name} from github. Error {response.status_code}: {response.text}'
+            )
+
+    def get_repo_list(self) -> list:
+        """get repo list"""
+
+        if os.path.exists(self.repo_list_path):
+            with open(self.repo_list_path, 'r', encoding='utf8') as f:
+                reader = csv.reader(f)
+                for row in reader:
+                    repo = Repo()
+                    repo.__dict__ = row
+                    self.repos.append(repo)
+            return self.repos
+
+        url = f'{self._api}/user/repos'
+        r = self.sess.get(url)
+        if r.status_code != 200:
+            print('get repo list failed, status code {}'.format(r.status_code))
+            return
+        repo_list = r.json()
+        self.save_csv()
+        return repo_list
+
+    def clone(self):
+        pass
+    
+    def pull(self, local_repo_path: str):
+        repo_name = local_repo_path.split(os.path.sep)[-1]
+        print(f'pull repo:{self.username}/{repo_name} from gitee')
+        os.chdir(local_repo_path)
+        os.system('git remote remove origin_gitee')
+        os.system(
+            f'git remote add origin_gitee https://gitee.com/{self.username}/{repo_name}.git'
+        )
+        os.system('git pull origin_gitee')
+        os.system('git remote remove origin_gitee')
+        os.chdir('..')
+                 
+    def push(self, local_repo_path: str):
+        repo_name = local_repo_path.split(os.path.sep)[-1]
+        print(f'push repo:{self.username}/{repo_name} to gitee')
+        # if not self.repo_exists(repo_name):
+        self.create_repo(repo_name)
+        os.chdir(local_repo_path)
+        os.system('git remote remove origin_gitee')
+        os.system(
+            f'git remote add origin_gitee https://{self.username}:{self.token}@gitee.com/{self.username}/{repo_name}.git'
+        )
+        os.system('git push -u origin_gitee')
+        os.system('git remote remove origin_gitee')
+        os.chdir('..')
+
+    @classmethod
+    def suitable(cls, extractor: str) -> bool:
+        """check if this extractor is suitable for this platform"""
+        if extractor == 'gitee':
+            return True
+        else:
+            return False

repo_sync/platform/github.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/06/04 13:43:46
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   github get a user all open source repo
+"""
+import os
+import json
+import csv
+from repo_sync.repo import Repo
+from .base_platform import BasePlatform
+
+class GithubIE(BasePlatform):
+    """github util"""
+
+    _host = 'https://api.github.com'
+
+    def __init__(self, username:str, token:str,host:str =None ,params: dict = None) -> None:
+        """init"""
+        super().__init__(username=username, token=token)
+        # 60 unauthenticated requests per hour
+        if self.token:
+            self.sess.headers.update({'Accept': 'application/vnd.github.v3+json'})
+
+    def create_repo(self, repo_name: str):
+        """create a repo"""
+        url = f'{self._host}/user/repos'
+        payload = {
+            'name': repo_name,
+            'private': True,
+            'has_issues': True,
+            'has_projects': False,
+            'has_wiki': False,
+        }
+        r = self.sess.post(url, data=json.dumps(payload))
+        if r.status_code != 201:
+            print(
+                'create repo {} failed, status code {}'.format(repo_name, r.status_code)
+            )
+            return
+        print('create repo {} success'.format(repo_name))
+
+    def delete(self, repo_name: str):
+        """delete a repo, maybe request a confirm by input"""
+        # print("delete repo:"+repo_name)
+        url = f'{self._host}/repos/{self.username}/{repo_name}'
+
+        response = self.sess.delete(url)
+        if response.status_code == 204:
+            print(f'Repository:{repo_name} deleted from github successfully!')
+        else:
+            print(
+                f'Failed to delete repository: {repo_name} from github. Error {response.status_code}: {response.text}'
+            )
+
+    def repo_exists(self, repo_name: str):
+        """check if a repo exists"""
+        url = f'{self._host}/repos/{self.username}/{repo_name}'
+        print('check repo:' + repo_name)
+        try:
+            response = self.sess.get(url)
+            if response.status_code == 200:
+                return True
+        except Exception as e:
+            return False
+
+    def pull(self, repo_path: str):
+        """pull a repo"""
+        repo_name = repo_path.split(os.path.sep)[-1]
+        print(f'pull repo:{self.username}/{repo_name} from github')
+        if not self.repo_exists(repo_name):
+            self.create_repo(repo_name)
+        os.chdir(repo_path)
+        os.system('git remote remove origin_github')
+        os.system(
+            f'git remote add origin_github')
+
+    def clone(self, repo_name: str):
+        pass
+    
+    def push(self, local_repo_path: str):
+        """push a local repo to remote"""
+        repo_name = local_repo_path.split(os.path.sep)[-1]
+        print(f'push repo:{self.username}/{repo_name} to github')
+        if not self.repo_exists(repo_name):
+            self.create_repo(repo_name)
+        os.chdir(local_repo_path)
+
+        os.system('git remote remove origin_github')
+        os.system(
+            f'git remote add origin_github https://{self.username}:{self.token}@github.com/{self.username}/{repo_name}.git'
+        )
+        os.system('git push -u origin_github')
+        os.system('git remote remove origin_github')
+        os.chdir('..')
+
+    def get_repo_list(self) -> list:
+        """get all repo list of a user"""
+        if os.path.exists(self.repo_list_path):
+            print(
+                'repo is exist, please read from {} file'.format(self.repo_list_path)
+            )
+            with open(self.repo_list_path, 'r', newline='') as csvfile:
+                reader = csv.reader(csvfile)
+                for row in reader:
+                    repo = Repo()
+                    repo.__dict__ = row
+                    self.repos.append(repo)
+        else:
+            page_num = 1
+            url = '{}/users/{}/repos'.format(self._host, self.username)
+            while True:
+                r = self.sess.get(url, params={'type': 'all', 'page': page_num})
+                if r.status_code != 200:
+                    print(
+                        'request url {} failed, status code {}'.format(
+                            url, r.status_code
+                        )
+                    )
+                    return
+                # rate limit
+                repo_list = r.json()
+                for repo in repo_list:
+                    repo_obj = Repo()
+                    repo_obj.name = repo.get('name')
+                    repo_obj.url = repo.get('html_url')
+                    repo_obj.description = repo.get('description')
+                    repo_obj.language = repo.get('language')
+                    repo_obj.star = repo.get('stargazers_count')
+                    repo_obj.fork = repo.get('forks_count')
+                    repo_obj.watch = repo.get('watchers_count')
+                    self.repos.append(repo_obj)
+                self.repos.sort(key=lambda x: x.star, reverse=True)
+                # Link: <https://api.github.com/user/123456/repos?page=2>; rel="next", <https://api.github.com/user/123456/repos?page=3>; rel="last"
+                links = r.headers.get('Link')
+
+                if not links or 'rel="next"' not in links:
+                    break
+
+                next_url = None
+                for link in links.split(','):
+                    if 'rel="next"' in link:
+                        next_url = link.split(';')[0].strip('<>')
+                        break
+
+                # Break loop if next URL is not valid
+                if not next_url:
+                    break
+
+                # Increment page number for next iteration
+                page_num += 1
+            self.save_csv()
+        return self.repos
+
+    def _clone_all_repo(self):
+        """cloen all repo"""
+        for repo in self.repos:
+            os.system('git clone {}'.format(repo.url))
+
+    def clone_user_repos(self):
+        """clone all repo of a user"""
+        # if github.csv exist, read it
+        if os.path.exists(self.repo_list_path):
+            with open(self.repo_list_path, 'r', newline='') as csvfile:
+                reader = csv.reader(csvfile)
+                for row in reader:
+                    if row[0] == 'name':
+                        continue
+                    repo = Repo()
+                    repo.__dict__ = row
+                    self.repos.append(repo)
+        else:
+            self.get_repo_list(self.username)
+        self._clone_all_repo()
+
+    @classmethod
+    def suitable(cls, extractor: str) -> bool:
+        """check if this extractor is suitable for this platform"""
+        if extractor == 'github':
+            return True
+        else:
+            return False

repo_sync/platform/gitlab.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/09/27 12:16:56
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   gitlab async
+"""
+import os
+import json,re 
+import csv
+from .base_platform import BasePlatform
+from repo_sync.repo import Repo
+
+class GitlabIE(BasePlatform):
+    """gitlab async"""
+
+    def __init__(self, username:str, token:str,host:str =None ,params: dict = None) -> None:
+        super().__init__(username=username, token=token)
+        self.host = self.host or 'https://gitlab.com'
+        self.sess.headers.update({"Authorization": f"Bearer {self.token}"})
+
+    def create_repo(self, repo_name: str):
+        """create a repo
+            and save project id to csv: gitlab_repo_list.csv
+        """
+        url = f"{self.host}/api/v4/projects"
+        payload = {
+            "name": repo_name,
+            "visibility": "private",
+        }
+        r = self.sess.post(url, data=json.dumps(payload))
+        if r.status_code != 201:
+            print(
+                "create repo {} failed, status code {}".format(repo_name, r.status_code)
+            )
+            return
+        print("create repo {} success".format(repo_name))
+        for repo in self.repos:
+            if repo.name == repo_name:
+                repo.url = r.json()["web_url"]
+                repo.id = r.json()["id"]
+                break
+        self.save_csv()
+        
+    def delete(self, repo_name: str):
+        """delete a repo,
+            find project id from csv: gitlab_repo_list.csv
+        """
+        project_id=""
+        r = self.sess.get(f"{self.host}/api/v4/users/{self.username}/projects?search={repo_name}")
+        if r.status_code == 200:
+            project_id = r.json()[0]["id"]
+            url = f"{self.host}/api/v4/projects/{project_id}"
+            response = self.sess.delete(url)
+            if response.status_code == 202:
+                print(f"Repository: {repo_name} deleted from gitlab successfully!")
+            else:
+                print(
+                    f"Failed to delete repository: {repo_name} from gitlab. Error {response.status_code}: {response.text}"
+                )
+        else:
+            print(f"Failed to delete repository: {repo_name} from gitlab. Error {r.status_code}: {r.text}")
+       
+    def get_repo_list(self, username: str)->list:
+        """get repo list"""
+        url = f"{self.host}/api/v4/users/{username}/projects"
+        r = self.sess.get(url)
+        if r.status_code != 200:
+            print("get repo list failed, status code {}".format(r.status_code))
+            return
+        repo_list = []
+        for res in r.json():
+            repo = Repo()
+            repo.name = res["name"]
+            repo.url = res["web_url"]
+            repo.local_path = None
+            repo.id = res["id"]
+            repo_list.append(repo)
+        self.save_csv()
+        return repo_list
+
+    def pull(self):
+        pass
+
+    def push(self, local_repo_path: str):
+        """push a local repo to remote
+        Args:
+            local_repo_path (str): local repo path
+        """
+        repo_name = local_repo_path.split(os.path.sep)[-1]
+        print(f"push repo:{self.username}/{repo_name} to gitlab")
+        self.create_repo(repo_name)
+        pur_host = re.search(r'(?<=//)[^/]+', self.host).group()
+
+        os.chdir(local_repo_path)
+
+        os.system("git remote remove origin_gitlab")
+        os.system(
+            f"git remote add origin_gitlab https://{self.username}:{self.token}@{pur_host}/{self.username}/{repo_name}.git"
+        )
+        os.system("git push -u origin_gitlab")
+        os.system("git remote remove origin_gitlab")
+        os.chdir("..")
+
+    def clone(self):
+        pass
+
+    @classmethod
+    def suitable(cls, extractor: str) -> bool:
+        """check if this extractor is suitable for this platform"""
+        if extractor == 'gitlab':
+            return True
+        else:
+            return False

repo_sync/platform/gogs.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/11/08 14:29:44
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   
+"""
+from .base_platform import BasePlatform
+import csv, re
+import json, os
+
+
+class GogsIE(BasePlatform):
+    """ """
+
+    gityoqi_repo_list = 'gityoqi_repo_list.csv'
+
+    def __init__(self, username:str, token:str,host:str =None ,params: dict = None) -> None:
+        super().__init__(username=username,token=token)
+        self._host = 'https://git.yoqi.me' if self.host is None else self.host
+
+    def create_org_repo(self, org_name: str, repo_name: str):
+        """create org repo"""
+        url = f'{self._host}/api/v1/orgs/{org_name}/repos'
+        payload = {
+            'name': repo_name,
+            'description': 'This is your first repository',
+            'private': True,
+            'auto_init': True,
+            'gitignores': 'Go',
+            'license': 'Apache v2 License',
+            'readme': 'Default',
+        }
+        r = self.sess.post(url, data=json.dumps(payload))
+        if r.status_code != 201:
+            print(
+                'create org repo {} failed, status code {}'.format(
+                    repo_name, r.status_code
+                )
+            )
+            return
+        print('create org repo {} success'.format(repo_name))
+
+    def create_repo(self, repo_name: str):
+        """create a repo"""
+        url = f'{self._host}/api/v1/user/repos'
+        payload = {
+            'name': repo_name,
+            'description': 'This is your first repository',
+            'private': True,
+            # "auto_init": True,
+            # "gitignores": "Go",
+            # "license": "Apache v2 License",
+            # "readme": "Default",
+        }
+        r = self.sess.post(url, data=json.dumps(payload))
+        if r.status_code != 201:
+            print(
+                'create repo {} failed, status code {}'.format(repo_name, r.status_code)
+            )
+            return
+        print('create repo {} success'.format(repo_name))
+
+    def delete(self, repo_name: str):
+        """delete a repo, maybe request a confirm by input"""
+        url = f'{self._host}/api/v1/repos/{self.username}/{repo_name}'
+        r = self.sess.delete(url)
+        if r.status_code != 204:
+            print(
+                'delete repo {} failed, status code {}'.format(repo_name, r.status_code)
+            )
+            return
+        print('delete repo {} success'.format(repo_name))
+
+    def get_repo_list(self, repo_name: str):
+        """get repo list"""
+        url = f'{self._host}/api/v1/users/{self.username}/repos'
+        r = self.sess.get(url)
+        if r.status_code != 200:
+            print('get repo list failed, status code {}'.format(r.status_code))
+            return
+        self.repos = r.json()
+        print('get repo list success')
+
+    def clone(self):
+        pass
+    
+    def pull(self, local_repo_path: str):
+        pass
+    
+    def push(self, local_repo_path: str):
+        repo_name = local_repo_path.split(os.path.sep)[-1]
+        print(f'push repo:{self.username}/{repo_name} to {self._host}')
+        self.create_repo(repo_name)
+
+        pur_host = re.search(r'(?<=//)[^/]+', self._host).group()
+
+        os.chdir(local_repo_path)
+        os.system('git remote remove origin_gogs')
+        os.system(
+            f'git remote add origin_gogs https://{self.username}:{self.token}@{pur_host}/{self.username}/{repo_name}.git'
+        )
+        os.system('git push -u origin_gogs')
+        os.system('git remote remove origin_gogs')
+        os.chdir('..')
+
+    @classmethod
+    def suitable(cls, extractor: str) -> bool:
+        """check if this extractor is suitable for this platform"""
+        if extractor == 'gogs':
+            return True
+        else:
+            return False

repo_sync/repo.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/09/26 16:30:04
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   
+"""
+
+
+class Repo(object):
+    """repo model"""
+
+    def __init__(self):
+        self.id = None
+        self.platform = "github"
+        self.name = None
+        self.url = None
+        self.remote_url = None
+        self.description = None
+        self.language = None
+        self.star = None
+        self.fork = None
+        self.watch = None
+        self.issues = None
+        self.local_path = None

repo_sync/repo_sync.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+'''
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/04/27 03:09:58
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   sync utils
+'''
+import os,csv
+import logging
+from .platform import gen_extractor_classes
+from .repo import Repo
+
+class RepoSync(object):
+    '''
+    SyncUtils class
+    '''
+    repo_list_path = 'repo_list.csv'
+
+    def __init__(self, params: dict, debug=False):
+        self.args = None
+        self.logger = None
+        self.init_logger(debug)
+
+        self.params = params
+        self.params['logger'] = self.logger
+        self.platforms = []
+        self.repos = []
+        for p in gen_extractor_classes():
+            self.platforms.append(p)
+        if params.get('repo_path', None) is not None:
+            self.get_local_repo_list(params.get('repo_path', None))
+    
+    def get_local_repo_list(self, repo_path):
+        """get git repo list from a folder"""
+        if os.path.isdir(repo_path) and os.path.exists(os.path.join(repo_path, '.git')):
+            self._find_git_repo(
+                path=repo_path, repo_name=repo_path.split(os.path.sep)[-1])
+        else:
+            for dir in os.listdir(repo_path):
+                current_path = os.path.join(repo_path, dir)
+                if os.path.isdir(current_path) and os.path.exists(os.path.join(current_path, '.git')):
+                    self._find_git_repo(path=current_path, repo_name=dir)
+        with open(self.repo_list_path, 'w') as f:
+            if len(self.repos) == 0:
+                print('repo list is empty, please delete repo_list.csv and try again')
+                return
+            writer = csv.DictWriter(
+                f, fieldnames=self.repos[0].__dict__.keys(), lineterminator='\n'
+            )
+            writer.writeheader()
+            for repo in self.repos:
+                writer.writerow(repo.__dict__)
+                
+    def _find_git_repo(self, path: str, repo_name:str =None):
+        try:
+            with open('{}/.git/config'.format(path), 'r') as f:
+                # get the remote url
+                repo =Repo()
+                try:
+                    url = re.findall(r'url\s+=\ (.*)', f.read())[0]
+                    # print(url)
+                    repo.name = url.split('/')[-1].replace('.git', '')
+                    repo.url = url
+                except Exception as e:
+                    repo.name = repo_name
+                repo.local_path = path
+                self.repos.append(repo)
+        except Exception as e:
+            print("skip {} because of {}".format(path, e))
+
+    def run(self):
+        '''
+        run repo
+        '''
+        command = self.params.get('command')
+        platform = self.params.get('platform', 'github')
+        current_platform = None
+        for p in self.platforms:
+            if p.suitable(platform):
+                current_platform = p
+        if current_platform is not None:
+            username = self.params.get(f'{platform}_username', None)
+            token = self.params.get(f'{platform}_token', None)
+            host = self.params.get(f'{platform}_host', None)
+
+            if command == 'create':
+                return
+            if command == 'clone':
+                # current_platform(username, token, host = host).clone(name)
+                return
+            if os.path.exists(self.repo_list_path):
+                with open(self.repo_list_path, 'r', encoding='utf8') as f:
+                    reader = csv.DictReader(f)
+                    for row in reader:
+                        repo = Repo()
+                        repo.__dict__ = row
+                        if command == 'push':
+                            current_platform(username,token,host, self.params).push(repo.local_path)
+                        elif command == 'delete':
+                            current_platform(username,token, host, self.params).delete(repo.name)
+                        elif command =='pull':
+                            current_platform(username,token, host, self.params).pull(repo.local_path)
+            else:
+                logging.info(
+                    'repo list is not exist, please run list_local command first'
+                )
+
+    def init_logger(self, debug:bool):
+        '''
+        init logger
+        '''
+        self.logger = logging.getLogger(__name__)
+        if debug:
+            self.logger.setLevel(logging.DEBUG)
+        else:
+            self.logger.setLevel(logging.INFO)
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(logging.DEBUG)
+        console_handler.setFormatter(formatter)
+        self.logger.addHandler(console_handler)

repo_sync/sync_utils.py

@@ -1,82 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-'''
-@Contact :   liuyuqi.gov@msn.cn
-@Time    :   2023/04/27 03:09:58
-@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
-@Desc    :   sync utils
-'''
-import os
-import sys
-import time
-import json
-import logging
-import argparse
-from repo_sync.base_repo import BaseRepo
-from repo_sync.gitlab_repo import GitlabRepo
-from repo_sync.coding_repo import CodingRepo
-from repo_sync.github_repo import GitHubRepo
-from repo_sync.gitee_repo import GiteeRepo
-
-class SyncUtils:
-    '''
-    SyncUtils class
-    '''
-    def __init__(self, debug=False):
-        self.args = None
-        self.logger = None
-        self.init_logger(debug)
-        self.init_args()
-
-    def run(self):
-        '''
-        run repo
-        '''
-        repos= []
-        with open("data/repo.txt", "r") as f:
-            repos = f.readlines()
-        for repo in repos:
-            repo_name = repo.split("/")[-1].replace(".git","")
-            user_name=repo.split("/")[-2]
-            if not os.path.exists(user_name):
-                os.mkdir(user_name)
-
-            if not os.path.exists(os.path.join(user_name,repo_name)):
-                self.logger.info("clone repo: %s", repo_name)
-                os.system("git clone %s %s" % (repo.strip(), os.path.join(user_name,repo_name)))
-
-            repoModel:BaseRepo = None
-            if self.args.type == "github":
-                repoModel = GitHubRepo(user_name, repo_name, self.logger)
-            elif self.args.type == "coding":
-                repoModel = CodingRepo(user_name, repo_name)
-            elif self.args.type == "gitlab":
-                repoModel = GitlabRepo(user_name, repo_name)
-            elif self.args.type == "gitee":
-                repoModel = GiteeRepo(user_name, repo_name)
-            repoModel.sync()
-
-    def init_logger(self, debug:bool):
-        '''
-        init logger
-        '''
-        self.logger = logging.getLogger(__name__)
-        if debug:
-            self.logger.setLevel(logging.DEBUG)
-        else:
-            self.logger.setLevel(logging.INFO)
-        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-        console_handler = logging.StreamHandler()
-        console_handler.setLevel(logging.DEBUG)
-        console_handler.setFormatter(formatter)
-        self.logger.addHandler(console_handler)
-
-    def init_args(self):
-        '''
-        init args
-        '''
-        parser = argparse.ArgumentParser()
-        parser.add_argument('-d', '--debug', action='store_true', help='debug mode')
-        parser.add_argument('-type', '--type', action='store_true',default="github", help='github,gitlab,gitee,coding')
-        parser.add_argument('-repo', '--repository', action='store_true', default="github", help='run repo')
-        self.args = parser.parse_args()

repo_sync/utils/str_util.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+@Contact :   liuyuqi.gov@msn.cn
+@Time    :   2023/10/31 17:06:37
+@License :   Copyright © 2017-2022 liuyuqi. All Rights Reserved.
+@Desc    :   字符串工具类
+"""
+import argparse
+import locale
+import re
+import sys
+
+
+def compat_register_utf8():
+    """win 兼容utf-8编码"""
+    if sys.platform == 'win32':
+        from codecs import register, lookup
+
+        register(lambda name: lookup('utf-8') if name == 'cp65001' else None)
+
+
+def preferredencoding():
+    """Get preferred encoding.
+
+    Returns the best encoding scheme for the system, based on
+    locale.getpreferredencoding() and some further tweaks.
+    """
+    try:
+        pref = locale.getpreferredencoding()
+        'TEST'.encode(pref)
+    except Exception:
+        pref = 'UTF-8'
+
+    return pref
+
+
+def SpCharReplace(char):
+    """特殊字符替换"""
+    temp = str(char)
+    for i in temp:
+        if '<' == i:
+            char = char.replace('<', '《')
+        if '>' == i:
+            char = char.replace('>', '》')
+        if "'" == i:
+            char = char.replace("'", '')  # 处理单引号
+        if '\\' == i:
+            char = char.replace('\\', '')  # 处理反斜杠\
+        if '"' == i:
+            char = char.replace('"', '`')  # 处理双引号"
+        if '&' == i:
+            char = char.replace('&', '-')  # 处理&号"
+        if '|' == i:
+            char = char.replace('|', '')  # 处理&号
+        if '@' == i:
+            char = char.replace('@', '.')  # 处理@号
+        if '%' == i:
+            char = char.replace('%', '`')  # 处理单引号
+        if '*' == i:
+            char = char.replace('*', '`')  # 处理反斜杠\
+        if '("' == i:
+            char = char.replace('"', '`')  # 处理双引号"
+        if ')"' == i:
+            char = char.replace(')"', '`')
+        if '-' == i:
+            char = char.replace('-', '`')  # 处理&号
+        if 'ÐÂÎÅ' == i:
+            char = char.replace('ÐÂÎÅ', '`')  # 处理ÐÂÎÅ
+        # 在后面扩展其他特殊字符
+    return char

repo_sync/version.py

@@ -0,0 +1 @@
+__version__ = '1.2.1'

requirements.txt

@@ -1 +1,2 @@
-requests==2.27.1
+requests==2.27.1
+python-dotenv==1.0.0

scripts/sync.xys

@@ -0,0 +1,17 @@
+$InitialPath = "<curpath>";
+    $GETResult = urldecode(html('<html><style>body{text-align:center}.selection{text-align:left;margin-left:10%;margin-right:10%;padding-bottom:20px}.submtbtn{padding-left:20px;padding-right:20px;padding-top:10px;padding-bottom:10px}</style><body><div><h1>repo_sync tools v1.12</h1></div><form method="get"action="xys:"><div class="selection"><span>Repo:&nbsp;&nbsp;</span><span>' . $InitialPath . '</span></div><div class="selection">Operation:<input type=radio checked name="operation"value="create">Create&nbsp;&nbsp;<input type=radio name="operation"value="push">Push&nbsp;&nbsp;<input type=radio name="operation"value="pull">Pull&nbsp;&nbsp;<input type=radio name="operation"value="delete">Delete&nbsp;&nbsp;</div><div class="selection">Platform:<input type=radio checked name="platform"value="github">Github&nbsp;&nbsp;<input type=radio name="platform"value="gitlab">Gitlab&nbsp;&nbsp;<input type=radio name="platform"value="gitee">Gitee&nbsp;&nbsp;<input type=radio name="platform"value="gogs">git.yoq.me&nbsp;&nbsp;<input type=radio name="platform"value="coding">Coding&nbsp;&nbsp;</div><div><input class="submtbtn"type="submit"name="mysubmit"value="run it"></div></form></body></html>'));
+        IF ("$GETResult"=="") {sub "_Cancel";}
+        // substr $GETResult, $GETResult, 1;
+        $operation1=gettoken($GETResult, "1", "&");
+        $platform1=gettoken($GETResult, "2", "&");
+        $ok=gettoken($a, "3", "&");
+                $operation=gettoken($operation1,2,"=");
+                $platform=gettoken($platform1,2,"=");
+                //run "cd F:\dist\Develop\Python\crawl_github\dist\ && main "+$platform+" --platform +"github"+ --repo_path <curpath>"
+                msg "cd /d F:\dist\Develop\Python\crawl_github\dist\ && main $operation --platform $platform --repo_path <curpath>"
+"_Cancel"
+//   msg "Cancled by user";   
+	sub "_EOF";  
+"_EOF"
+	END 1==1;
+//EOF

setup.cfg

@@ -0,0 +1 @@
+[meta]

setup.py

@@ -0,0 +1,6 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='repo_sync',
+    enter_points={'console_scripts': ['repo_sync = repo_sync:main']},
+)