腾讯云COS上传、批量删除工具(Python)

作者
文章目录 展开

腾讯云对象存储COS是类似于阿里云OSS，相比OSS，COS提供每月免费额度：存储空间50G、外网访问流量10G（内网免费）、免费读请求100万次、写请求10万次。对网站备份来说不错，但是，腾讯云提供的工具太low，参考阿里云OSS，写了一个cos信息配置、创建目录、上传、批量删除工具（coscmd）

安装Python2.7

此工具在Python2.7上测试通过，建议用《OneinStack》安装Python2.7，安装路径为:/usr/local/python，命令如下：

  1. cd ~/oneinstack
  2. ./addons.sh #选择7，install Let’s Encrypt client

配置COS

登陆腾讯云管理后台https://console.qcloud.com/cos/bucket创建bucket，并获取API密钥，在下面配置中一一对应。

  1. cd oneinstack/tools #必须进入该目录执行
  2. wget http://mirrors.linuxeye.com/oneinstack/tools/cos.tgz #v4版本
  3. wget http://mirrors.linuxeye.com/oneinstack/tools/cos_v3.tgz #v3版本，可提交工单升v4
  4. tar xzf cos.tgz
  5. /usr/local/python/bin/python ./coscmd config –appid=[appid] —id=[secret_id] –key=[secret_key] –region=[region] –bucket=[bucket]  #v4配置
  6. /usr/local/python/bin/python ./coscmd config –appid=[appid] —id=[secret_id] –key=[secret_key] –bucket=[bucket]  #v3配置

COSv4版本配置

腾讯云COS上传、批量删除工具(Python)

COSv3版本配置

腾讯云COS上传、批量删除工具(Python)

执行后会将相关信息写到~/.coscredentials，执行其它动作会自动加在改配置。

创建目录

腾讯云COS上传、批量删除工具(Python)

上传文件

腾讯云COS上传、批量删除工具(Python)

批量删除

腾讯云COS上传、批量删除工具(Python)

coscmd代码(COSv4)

  1. #!/usr/bin/env python
  2. #coding:utf-8
  4. import sys,os
  5. import datetime
  6. import random
  7. import threading
  8. import time
  9. import datetime
  10. import logging
  11. import ConfigParser
  12. from optparse import OptionParser
  13. from logging.handlers import RotatingFileHandler
  14. from time import strftime, localtime
  15. from time import sleep
  16. from datetime import date
  17. from datetime import timedelta
  18. from cos import CosClient
  19. from cos import UploadFileRequest
  20. from cos import CreateFolderRequest
  21. from cos import DelFileRequest
  22. from cos import DelFolderRequest
  23. from cos import ListFolderRequest
  24. from cos import threadpool
  26. MAX_RETRY_TIMES = 3
  27. LOG_SAVE_EVERY_NUM = 1024
  28. ONE_TASK_DEL_FILE_NUMS = 50
  29. log_level = 1
  30. log_file_name = “del_file.log”
  31. dir_thread_num = 2
  32. file_thread_num = 5
  33. log_out_to_screen = 1
  34. delete_folder_fail_exist = 0
  36. CONFIGFILE = “%s/.coscredentials” % os.path.expanduser(‘~’)
  37. CONFIGSECTION = ‘COSCredentials’
  39. HAS_FORK = hasattr(os, ‘fork’)
  41. HELP = \
  42. ”’coscmd:
  43.     config         –appid=[appid] –id=[secret_id] –key=[secret_key] –region=[region] –bucket=[bucket] 
  44.     ls             cosdir
  45.     mkdir          dirname
  46.     put            localfile  cosdir 
  47.     rm(delete,del) object
  48.     ”’
  50. CMD_LIST = {}
  51. def cmd_configure(args, options):
  52.     if options.appid is None or options.secret_id is None or options.secret_key is None  or options.region is None or options.bucket is None:
  53.         print(“%s miss parameters, use –appid=[appid] —id=[secret_id] –key=[secret_key] –region=[region] –bucket=[bucket] to specify appid/id/key/region/bucket pair” % args[0])
  54.         sys.exit(-1)
  55.     config = ConfigParser.RawConfigParser()
  56.     config.add_section(CONFIGSECTION)
  57.     config.set(CONFIGSECTION, ‘appid’, options.appid)
  58.     config.set(CONFIGSECTION, ‘secret_id’, options.secret_id)
  59.     config.set(CONFIGSECTION, ‘secret_key’, options.secret_key)
  60.     if options.region in [‘sh’,’gz’,’tj’,’sgp’]:
  61.         config.set(CONFIGSECTION, ‘region’, options.region)
  62.     else:
  63.         print(“input region error, setup use : –region={sh,gz,tj,sgp}”)
  64.         sys.exit(-1)
  65.     config.set(CONFIGSECTION, ‘bucket’, options.bucket)
  66.     cfgfile = open(CONFIGFILE, ‘w+’)
  67.     config.write(cfgfile)
  68.     print(“Your configuration is saved into %s .” % CONFIGFILE)
  69.     cfgfile.close()
  70.     import stat
  71.     os.chmod(CONFIGFILE, stat.S_IREAD | stat.S_IWRITE)
  73. def cmd_loadconfigure():
  74.     config = ConfigParser.ConfigParser()
  75.     config.read(CONFIGFILE)
  76.     global appid
  77.     global secret_id
  78.     global secret_key
  79.     global region
  80.     global bucket
  81.     appid = int(config.get(CONFIGSECTION, ‘appid’))
  82.     secret_id = config.get(CONFIGSECTION, ‘secret_id’).decode(‘utf-8’)
  83.     secret_key = config.get(CONFIGSECTION, ‘secret_key’).decode(‘utf-8’)
  84.     region = config.get(CONFIGSECTION, ‘region’)
  85.     bucket = config.get(CONFIGSECTION, ‘bucket’).decode(‘utf-8’)
  86.     if len(secret_id) == 0 or len(secret_key) == 0 or len(region) == 0 or len(bucket) == 0:
  87.         print(“can’t get appid/secret_id/secret_key/region/bucket, setup use : config –appid=[appid] —id=[secret_id] –key=[secret_key] –region=[region] –bucket=[bucket]”)
  88.         sys.exit(1)
  90. def cmd_lsdir(COSDIR):
  91.     cosdir = COSDIR.decode(‘utf-8’)
  92.     request = ListFolderRequest(bucket, cosdir)
  93.     list_folder_ret = cos_client.list_folder(request)
  94.     if list_folder_ret[u’code’] == 0:
  95.         print(True)
  96.     else:
  97.         print(“%s, appid/secret_id/secret_key/region/bucket invalid”% list_folder_ret[u’message’])
  99. def cmd_mkdir(COSDIR):
  100.     cosdir = COSDIR.decode(‘utf-8’)
  101.     request = CreateFolderRequest(bucket, cosdir)
  102.     create_folder_ret = cos_client.create_folder(request)
  103.     if create_folder_ret[u’code’] == 0:
  104.         print(“mkdir cos://%s%s OK” % (bucket,COSDIR))
  105.     else:
  106.         print(create_folder_ret[u’message’])
  108. def cmd_put(LOCALFILE,COSFILE):
  109.     localfile = LOCALFILE.decode(‘utf-8’)
  110.     cosfile = COSFILE.decode(‘utf-8’)
  111.     request = UploadFileRequest(bucket, cosfile, localfile)
  112.     request.set_insert_only(0)
  113.     upload_file_ret = cos_client.upload_file(request)
  114.     if upload_file_ret[u’code’] == 0:
  115.         print(“put cos://%s%s OK” % (bucket,COSFILE))
  116.     else:
  117.         print(upload_file_ret[u’message’])
  119. def loginit():
  120.     global config
  121.     if (log_file_name == “”):
  122.         return
  123.     log_level = logging.ERROR
  124.     if log_level == 0:
  125.         log_level = logging.DEBUG
  126.     if log_level == 1:
  127.         log_level = logging.INFO
  128.     if log_level == 2:
  129.         log_level = logging.WARNING
  131.         #定义一个RotatingFileHandler，最多备份5个日志文件，每个日志文件最大20M
  132.     logger = logging.getLogger(“”)
  133.     Rthandler = RotatingFileHandler(log_file_name, maxBytes= 20*1024*1024,backupCount=5)
  134.     Rthandler.setLevel(log_level)
  135.     formatter = logging.Formatter(‘%(asctime)s %(levelname)s %(message)s’)
  136.     Rthandler.setFormatter(formatter)
  137.     logger.addHandler(Rthandler)
  138.         #输出日志到屏幕
  139.     console = logging.StreamHandler()
  140.     console.setFormatter(formatter)
  141.     if (log_out_to_screen == 1):
  142.         logger.addHandler(console)
  144.     logger.setLevel(log_level)
  145.     return logger
  147. #日期相关操作
  148. class Dateop():
  149.     @staticmethod
  150.     def isValidDate(str):
  151.         try:
  152.             time.strptime(str, “%Y””%m””%d”)
  153.             return True
  154.         except:
  155.             return False
  157.     @staticmethod
  158.     def getdaystr(n=0):
  159.         dt = date.today()-timedelta(days=n)
  160.         tt = dt.timetuple()
  161.         daystr = strftime(“%Y””%m””%d”,tt)
  162.         return daystr
  164.     @staticmethod
  165.     def cmpDateAgo(t1,t2):
  166.         if (Dateop.isValidDate(t1)==False or Dateop.isValidDate(t2)==False):
  167.             return False
  168.         if (int(t1) <= int (t2)):
  169.             return True
  170.         return False
  172.     @staticmethod
  173.     def isNeedDeleteDir(dirname, n=0):
  174.         if (len(dirname) != 8):
  175.             return False
  176.         if Dateop.isValidDate(dirname) == False:
  177.             return False
  178.         d2 = Dateop.getdaystr(n);
  179.         if Dateop.cmpDateAgo(dirname, d2):
  180.             return True
  181.         return False
  182. #删除文件统计
  183. class FileStat():
  184.     global cos_log
  185.     def __init__(self):
  186.         self.delfilesuccnum = 0
  187.         self.deldirsuccnum = 0
  188.         self.delfilefailnum = 0
  189.         self.deldirfailnum = 0
  190.         self.lock = threading.Lock()
  192.     def addDelFileFailNum(self,num=1):
  193.         self.lock.acquire(1)
  194.         self.delfilefailnum += num
  195.         self.lock.release()
  196.     def addDelDirFailNum(self,num=1):
  197.         self.lock.acquire(1)
  198.         self.deldirfailnum += num
  199.         self.lock.release()
  200.     def addDelDirSuccNum(self, num=1):
  201.         self.lock.acquire(1)
  202.         self.deldirsuccnum += num
  203.         self.lock.release()
  204.     def addDelFileSuccNum(self, num=1):
  205.         self.lock.acquire(1)
  206.         self.delfilesuccnum += num
  207.         self.lock.release()
  208.     def printStat(self):
  209.         msg =””.join([“delfilesuccnum=”,str(self.delfilesuccnum),
  210.                 “,delfilefailnum=”,str(self.delfilefailnum),
  211.                 “,deldirsuccnum=”,str(self.deldirsuccnum),
  212.                 “,deldirfailnum=”,str(self.deldirfailnum)])
  213.         print(msg)
  214.     def logStat(self):
  215.         curtime = time.strftime(“%Y-%m-%d %H:%M:%S”, time.localtime())
  216.         log = ”.join([“delfilenum=”,str(self.delfilesuccnum),
  217.             “,deldirnum=”,str(self.deldirsuccnum),”,delfilefailnum=”,
  218.             str(self.delfilefailnum),”,deldirfailnum=”,str(self.deldirfailnum)])
  219.         cos_log.info(log)
  221. #执行时间统计
  222. class TimeStat(object):
  223.     global cos_log
  224.     def __init__(self):
  225.         self.start()
  226.     def start(self):
  227.         self.start = datetime.datetime.now()
  228.         self.t1 = time.time()
  229.         msg = “delete task started  ………..”
  230.         cos_log.info(msg)
  231.     def end(self):
  232.         self.end = datetime.datetime.now()
  233.         self.t2 = time.time()
  234.         msg = “delete task ended\n\nrm task finished,\ntimecost:”+str(self.t2-self.t1) + ” (s)”
  235.         cos_log.info(msg)
  237. #删除文件列表中的文件
  238. def delfiles(cos_client, bucket, filelist):
  239.     for f in filelist:
  240.         delfile(cos_client, bucket, f)
  242. def delfolders(cos_client, bucket, folderlist):
  243.     for f in folderlist:
  244.         delfolder(cos_client, bucket, f)
  245. #文件夹删除
  246. def delfolder(cos_client, bucket, folder):
  247.     global stat
  248.     global cos_log
  249.     if not folder:
  250.         return 0
  251.     delfolderreq = DelFolderRequest(bucket, folder)
  252.     retry = 0
  253.     while (retry < MAX_RETRY_TIMES):
  254.         ret = cos_client.del_folder(delfolderreq)
  255.         msg = “delfolder fail, bucket=”+bucket+”,folder=”+folder+ret[‘message’]
  256.         if (ret[‘code’] == 0):
  257.             break
  258.         elif (ret[‘code’] == -166):
  259.             cos_log.warning(msg)
  260.             break
  261.         #操作太频繁,频控
  262.         elif (ret[‘code’] == -71):
  263.             sleep(random.randint(1,5))
  264.             cos_log.warning(msg)
  265.             retry += 1
  266.             continue
  267.         #文件夹非空
  268.         elif (ret[‘code’] == -173):
  269.             break
  270.         else:
  271.             cos_log.warning(msg)
  272.             retry += 1
  273.     if (ret[‘code’] != 0 and  ret[‘code’] != -166):
  274.         stat.addDelDirFailNum()
  275.         cos_log.error(“delfolder fail, bucket=”+bucket+”,folder=”+folder+ret[‘message’])
  276.         return ret[‘code’]
  277.     if (ret[‘code’] == 0):
  278.         stat.addDelDirSuccNum()
  279.         msg = “delfolder success, bucket=”+bucket+”,folder=”+folder
  280.         cos_log.info(msg)
  281.     return 0
  283. #文件删除
  284. def delfile(cos_client, bucket, filepath):
  285.     global stat
  286.     global cos_log
  287.     delfilereq = DelFileRequest(bucket, filepath)
  288.     retry = 0
  289.     while (retry < MAX_RETRY_TIMES):
  290.         ret = cos_client.del_file(delfilereq)
  291.         msg = “delfile fail bucket=”+bucket+”,file=”+filepath+ret[‘message’]
  292.         if (ret[‘code’] == 0):
  293.             break
  294.         #文件不存在
  295.         elif (ret[‘code’] == -166):
  296.             cos_log.warning(msg)
  297.             break
  298.         #单目录写操作过快
  299.         elif (ret[‘code’] == -143):
  300.             sleep(random.randint(1,5))
  301.             cos_log.warning(msg)
  302.             retry += 1
  303.             continue
  304.         #操作太频繁,频控
  305.         elif (ret[‘code’] == -71):
  306.             sleep(random.randint(1,5))
  307.             cos_log.warning(msg)
  308.             retry += 1
  309.             continue
  310.         else:
  311.             cos_log.warning(msg)
  312.             retry += 1
  313.             continue
  314.     if (ret[‘code’] != 0 and  ret[‘code’] != -166):
  315.         stat.addDelFileFailNum()
  316.         cos_log.error(“delfile fail, bucket=”+bucket+”,file=”+filepath+ret[‘message’])
  317.         return ret[‘code’]
  318.     if (ret[‘code’] == 0):
  319.         stat.addDelFileSuccNum()
  320.         msg = “delfile success, bucket=”+bucket+”,file=”+filepath
  321.         cos_log.info(msg)
  322.     return 0
  324. #递归文件夹进行文件删除
  325. def delete_r(cos_client, bucket, path, thread_pool_file):
  326.     global stat
  327.     global config
  328.     global cos_log
  329.     cos_log.debug(“delete_r bucket:”+bucket+”,path:”+path)
  330.     context = u””
  331.     #递归文件夹
  332.     while True:
  333.         listfolderreq = ListFolderRequest(bucket, path, 1000, u”, context)
  334.         retry = 0
  335.         while (retry < MAX_RETRY_TIMES):
  336.             listret = cos_client.list_folder(listfolderreq)
  337.             if listret[‘code’] != 0 :
  338.                 retry += 1
  339.                 sleep(random.randint(1,3))
  340.                 continue
  341.             else:
  342.                 break
  343.         if (listret[‘code’] != 0):
  344.             cos_log.error(“delete_r: list folder fail:”+path +”,return msg:”+ listret[‘message’])
  345.             return listret[‘code’]
  346.         if (len(listret[‘data’][‘infos’]) == 0):
  347.             break;
  348.         filelist = []
  349.         dirlist = []
  350.         for info in listret[‘data’][‘infos’]:
  351.             fullname = path + info[‘name’]
  352.             #list出来的文件列表中文件夹和文件本身是混杂一起的
  353.             if info.has_key(‘filesize’):
  354.                 filelist.append(fullname)
  355.                 if (len(filelist) >= ONE_TASK_DEL_FILE_NUMS):
  356.                     args = [cos_client, bucket, filelist]
  357.                     args_tuple = (args,None)
  358.                     args_list = [args_tuple]
  359.                     requests = threadpool.makeRequests(delfiles, args_list)
  360.                     for req in requests:
  361.                         thread_pool_file.putRequest(req)
  362.                         filelist = []
  363.                         continue
  364.                 else:
  365.                     pass
  366.             else:
  367.                 dirlist.append(fullname)
  368.                 if (len(dirlist) >= ONE_TASK_DEL_FILE_NUMS):
  369.                     args = [cos_client, bucket, dirlist]
  370.                     args_tuple = (args,None)
  371.                     args_list = [args_tuple]
  372.                     requests = threadpool.makeRequests(delfolders, args_list)
  373.                     for req in requests:
  374.                         thread_pool_file.putRequest(req)
  375.                         dirlist = []
  376.                         continue
  377.                 else:
  378.                     pass
  379.                 pass
  381.         if (len(filelist) > 0):
  382.             args = [cos_client, bucket, filelist]
  383.             args_tuple = (args,None)
  384.             args_list = [args_tuple]
  385.             requests = threadpool.makeRequests(delfiles, args_list)
  386.             for req in requests:
  387.                 thread_pool_file.putRequest(req)
  388.                 filelist = []
  389.         else:
  390.             pass
  392.         if (len(dirlist) > 0):
  393.             args = [cos_client, bucket, dirlist]
  394.             args_tuple = (args,None)
  395.             args_list = [args_tuple]
  396.             requests = threadpool.makeRequests(delfolders, args_list)
  397.             for req in requests:
  398.                 thread_pool_file.putRequest(req)
  399.                 filelist = []
  400.         else:
  401.             pass
  403.         cos_log.debug(“delete_r thread pool file waiting\n”)
  404.         thread_pool_file.wait()
  405.         cos_log.debug(“delete_r thread pool file waiting end\n”)
  407.         if (listret[‘data’][‘listover’] == False):
  408.             context = listret[‘data’][‘context’]
  409.             continue
  410.         else:
  411.             break
  413.     stat.logStat()
  414.     return 0
  415. #支持Ctrl+C终止程序
  416. class Watcher():
  418.     def __init__(self):
  419.         self.child = os.fork()
  420.         if self.child == 0:
  421.             return
  422.         else:
  423.             self.watch()
  425.     def watch(self):
  426.         global cos_log
  427.         try:
  428.             os.wait()
  429.         except KeyboardInterrupt:
  430.             cos_log.ERROR(“ctrl+c terminated rm_recursive.py, exiting…”)
  431.             self.kill()
  432.         sys.exit()
  433.     def kill(self):
  434.         try:
  435.             os.kill(self.child, signal.SIGKILL)
  436.         except OSError:
  437.             pass
  438. def cmd_rm(COSDIR):
  439.     global thread_pool
  440.     global cos_log
  441.     global stat
  442.     cos_log = loginit()
  443.     stat = FileStat()
  444.     timestat = TimeStat()
  445.     if HAS_FORK:
  446.       Watcher()
  447.     path = COSDIR.decode(‘utf-8’)
  448.     thread_pool_dir = threadpool.ThreadPool(dir_thread_num)
  449.     thread_pool_file = threadpool.ThreadPool(file_thread_num)
  450.     cos_log.debug(“bucket:”+bucket +”,path:”+path)
  451.     args = [cos_client, bucket, path, thread_pool_file]
  452.     args_tuple = (args, None)
  453.     args_list = [args_tuple]
  454.     requests = threadpool.makeRequests(delete_r, args_list)
  455.     for req in requests:
  456.         thread_pool_dir.putRequest(req)
  458.     cos_log.debug(“thread_pool_dir waiting…..\n”)
  459.     thread_pool_dir.wait()
  460.     thread_pool_dir.dismissWorkers(dir_thread_num, True)
  461.     cos_log.debug(“thread_pool_dir wait end…..\n”)
  463.     timestat.end()
  464.     stat.logStat()
  466. if sys.argv[1] in [‘config’,’ls’,’mkdir’,’put’,’rm’,’delete’,’del‘] and len(sys.argv) >= 3:
  467.     if sys.argv[1] == ‘config’:
  468.         parser = OptionParser()
  469.         parser.add_option(“-a”, “–appid”, dest=”appid”, help=”specify appid”)
  470.         parser.add_option(“-i”, “–id“, dest=”secret_id”, help=”specify secret id“)
  471.         parser.add_option(“-k”, “–key”, dest=”secret_key”, help=”specify secret key”)
  472.         parser.add_option(“-r”, “–region”, dest=”region”, help=”specify region”)
  473.         parser.add_option(“-b”, “–bucket”, dest=”bucket”, help=”specify bucket”)
  474.         (options, args) = parser.parse_args()
  475.         CMD_LIST[‘config’] = cmd_configure
  476.         CMD_LIST[‘config’](args, options)
  477.     if sys.argv[1] == ‘ls’:
  478.         cmd_loadconfigure()
  479.         cos_client = CosClient(appid, secret_id, secret_key, region)
  480.         COSDIR = sys.argv[2]
  481.         cmd_lsdir(COSDIR)
  482.     if sys.argv[1] == ‘mkdir’:
  483.         cmd_loadconfigure()
  484.         cos_client = CosClient(appid, secret_id, secret_key, region)
  485.         COSDIR = sys.argv[2]
  486.         cmd_mkdir(COSDIR)
  487.     if sys.argv[1] == ‘put’ and len(sys.argv) == 4:
  488.         cmd_loadconfigure()
  489.         cos_client = CosClient(appid, secret_id, secret_key, region)
  490.         LOCALFILE = sys.argv[2]
  491.         COSFILE = sys.argv[3]
  492.         cmd_put(LOCALFILE,COSFILE)
  493.     if sys.argv[1] in (‘rm’,’delete’,’del‘):
  494.         cmd_loadconfigure()
  495.         cos_client = CosClient(appid, secret_id, secret_key, region)
  496.         COSDIR = sys.argv[2]
  497.         path = COSDIR.decode(‘utf-8’)
  498.         cmd_rm(path)
  499. else:
  500.     print(HELP)
  501.     exit()

Fri Mar 31 21:51:20 CST 2017


【AD】美国洛杉矶CN2 VPS/香港CN2 VPS/日本CN2 VPS推荐，延迟低、稳定性高、免费备份_搬瓦工vps

【AD】RackNerd 推出的 KVM VPS 特价优惠，在纽约、西雅图、圣何塞和阿什本每年仅需 12.88 美元！