crawl_mrdx/shell/tar.py

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@Author  :   liuyuqi
@Contact :   liuyuqi.gov@msn.cn
@Time    :   2019/11/11 20:18:09
@Version :   1.0
@License :   (C)Copyright 2019
@Desc    :   200个文件夹打包一次
'''

import os

rootPath = r"/home/twtech/github/python/crawl-mrdx/data"
os.chdir(rootPath)

size = 100
count = 1
count2 = 30
currentDir=""

for path in os.listdir(rootPath):
    if os.path.isdir(os.path.join(rootPath,path)):
        currentDir = currentDir + " " + path
        if count % size == 0:
            shellStr = "tar -zcf /ossfs/crawl-mrdx/mrdx%s.tar.gz" % (
                count2) + currentDir
            print(shellStr)
            os.system(shellStr)
            currentDir=""
            count2 = count2 + 1
        count = count + 1

# Check if there are remaining directories to include in a tar.gz file
if currentDir != "":
    shellStr = "tar -zcf /ossfs/crawl-mrdx/mrdx%s.tar.gz" % (
        count2) + currentDir
    print(shellStr)
    os.system(shellStr)