|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# coding=utf-8 |
| 3 | + |
| 4 | +import getopt, json, logging, re, sys, time |
| 5 | +from aliyunsdkcdn.request.v20180510.DescribeRefreshQuotaRequest import DescribeRefreshQuotaRequest |
| 6 | +from aliyunsdkcdn.request.v20180510.DescribeRefreshTasksRequest import DescribeRefreshTasksRequest |
| 7 | +from aliyunsdkcdn.request.v20180510.PushObjectCacheRequest import PushObjectCacheRequest |
| 8 | +from aliyunsdkcdn.request.v20180510.RefreshObjectCachesRequest import RefreshObjectCachesRequest |
| 9 | +from aliyunsdkcore.client import AcsClient |
| 10 | + |
| 11 | +# 初始化日志记录 |
| 12 | +logging.basicConfig(level=logging.DEBUG, filename='./RefreshAndPredload.log') |
| 13 | + |
| 14 | + |
| 15 | +# 定义全局变量类，存储AK、SK、FD等信息 |
| 16 | +class Envariable(object): |
| 17 | + LISTS = [] |
| 18 | + REGION = 'cn-zhangzhou' |
| 19 | + AK = None |
| 20 | + SK = None |
| 21 | + FD = None |
| 22 | + CLI = None |
| 23 | + TASK_TYPE = None |
| 24 | + TASK_AREA = None |
| 25 | + TASK_OTYPE = None |
| 26 | + |
| 27 | + # 设置AK |
| 28 | + @staticmethod |
| 29 | + def set_ak(ak): |
| 30 | + Envariable.AK = ak |
| 31 | + |
| 32 | + # 获取AK |
| 33 | + @staticmethod |
| 34 | + def get_ak(): |
| 35 | + return Envariable.AK |
| 36 | + |
| 37 | + # 设置SK |
| 38 | + @staticmethod |
| 39 | + def set_sk(sk): |
| 40 | + Envariable.SK = sk |
| 41 | + |
| 42 | + # 获取SK |
| 43 | + @staticmethod |
| 44 | + def get_sk(): |
| 45 | + return Envariable.SK |
| 46 | + |
| 47 | + # 设置FD |
| 48 | + @staticmethod |
| 49 | + def set_fd(fd): |
| 50 | + Envariable.FD = fd |
| 51 | + |
| 52 | + # 获取FD |
| 53 | + @staticmethod |
| 54 | + def get_fd(): |
| 55 | + return Envariable.FD |
| 56 | + |
| 57 | + # 设置任务类型 |
| 58 | + @staticmethod |
| 59 | + def set_task_type(task_type): |
| 60 | + Envariable.TASK_TYPE = task_type |
| 61 | + |
| 62 | + # 获取任务类型 |
| 63 | + @staticmethod |
| 64 | + def get_task_type(): |
| 65 | + return Envariable.TASK_TYPE |
| 66 | + |
| 67 | + # 设置任务区域 |
| 68 | + @staticmethod |
| 69 | + def set_task_area(task_area): |
| 70 | + Envariable.TASK_AREA = task_area |
| 71 | + |
| 72 | + # 获取任务区域 |
| 73 | + @staticmethod |
| 74 | + def get_task_area(): |
| 75 | + return Envariable.TASK_AREA |
| 76 | + |
| 77 | + # 设置任务对象类型 |
| 78 | + @staticmethod |
| 79 | + def set_task_otype(task_otype): |
| 80 | + Envariable.TASK_OTYPE = task_otype |
| 81 | + |
| 82 | + # 获取任务对象类型 |
| 83 | + @staticmethod |
| 84 | + def get_task_otype(): |
| 85 | + return Envariable.TASK_OTYPE |
| 86 | + |
| 87 | + # 创建AcsClient |
| 88 | + @staticmethod |
| 89 | + def set_acs_client(): |
| 90 | + Envariable.CLI = AcsClient(Envariable.get_ak(), Envariable.get_sk(), Envariable.REGION) |
| 91 | + |
| 92 | + # 获取AcsClient |
| 93 | + @staticmethod |
| 94 | + def get_acs_client(): |
| 95 | + return Envariable.CLI |
| 96 | + |
| 97 | + |
| 98 | +class InitHandler(object): |
| 99 | + def __init__(self, ak, sk, region): |
| 100 | + try: |
| 101 | + self.client = AcsClient(ak, sk, region) |
| 102 | + except Exception: |
| 103 | + logging.info('[error]: initial AcsClient failed') |
| 104 | + exit(1) |
| 105 | + |
| 106 | + |
| 107 | +class BaseCheck(object): |
| 108 | + def __init__(self): |
| 109 | + self.invalidurl = '' |
| 110 | + self.lines = 0 |
| 111 | + self.urllist = Envariable.get_fd() |
| 112 | + |
| 113 | + # 检查配额 |
| 114 | + def printQuota(self): |
| 115 | + try: |
| 116 | + if Envariable.get_acs_client(): |
| 117 | + client = Envariable.get_acs_client() |
| 118 | + else: |
| 119 | + Envariable.set_acs_client() |
| 120 | + client = Envariable.get_acs_client() |
| 121 | + quotas = DescribeRefreshQuotaRequest() |
| 122 | + quotaResp = json.loads(client.do_action_with_exception(quotas)) |
| 123 | + except Exception: |
| 124 | + logging.info('\n[error]: initial AcsClient failed\n') |
| 125 | + sys.exit(1) |
| 126 | + |
| 127 | + if Envariable.TASK_TYPE: |
| 128 | + if Envariable.TASK_TYPE == 'push': |
| 129 | + if self.lines > int(quotaResp['PreloadRemain']): |
| 130 | + sys.exit('\n[error]：PreloadRemain is not enough {0}'.format(quotaResp['PreloadRemain'])) |
| 131 | + return True |
| 132 | + if Envariable.TASK_TYPE == 'clear': |
| 133 | + if Envariable.get_task_otype() == 'File' and self.lines > int(quotaResp['UrlRemain']): |
| 134 | + sys.exit('\n[error]：UrlRemain is not enough {0}'.format(quotaResp['UrlRemain'])) |
| 135 | + elif Envariable.get_task_otype() == 'Directory' and self.lines > int(quotaResp['DirRemain']): |
| 136 | + sys.exit('\n[error]：DirRemain is not enough {0}'.format(quotaResp['DirRemain'])) |
| 137 | + else: |
| 138 | + return True |
| 139 | + |
| 140 | + # 验证URL格式 |
| 141 | + def urlFormat(self): |
| 142 | + with open(self.urllist, 'r') as f: |
| 143 | + for line in f.readlines(): |
| 144 | + self.lines += 1 |
| 145 | + if not re.match(r'^((https)|(http))', line): |
| 146 | + self.invalidurl = line + '\n' + self.invalidurl |
| 147 | + if self.invalidurl != '': |
| 148 | + sys.exit('\n[error]: URL format is illegal \n{0}'.format(self.invalidurl)) |
| 149 | + return True |
| 150 | + |
| 151 | + |
| 152 | +# 批量处理类，将URL列表按指定数量分成多个批次 |
| 153 | +class doTask(object): |
| 154 | + @staticmethod |
| 155 | + def urlencode_pl(inputs_str): |
| 156 | + len_str = len(inputs_str) |
| 157 | + if inputs_str == '' or len_str <= 0: |
| 158 | + return '' |
| 159 | + result_end = '' |
| 160 | + for chs in inputs_str: |
| 161 | + if chs.isalnum() or chs in {':', '/', '.', '-', '_', '*'}: |
| 162 | + result_end += chs |
| 163 | + elif chs == ' ': |
| 164 | + result_end += '+' |
| 165 | + else: |
| 166 | + result_end += f'%{ord(chs):02X}' |
| 167 | + return result_end |
| 168 | + |
| 169 | + # 分批处理URL |
| 170 | + @staticmethod |
| 171 | + def doProd(): |
| 172 | + gop = 20 # 这里定义了每个批次的最大URL数量 |
| 173 | + mins = 1 |
| 174 | + maxs = gop |
| 175 | + with open(Envariable.get_fd(), 'r') as f: |
| 176 | + for line in f.readlines(): |
| 177 | + line = doTask.urlencode_pl(line.strip()) + '\n' |
| 178 | + Envariable.LISTS.append(line) |
| 179 | + if mins >= maxs: |
| 180 | + yield Envariable.LISTS |
| 181 | + Envariable.LISTS = [] |
| 182 | + mins = 1 |
| 183 | + else: |
| 184 | + mins += 1 |
| 185 | + if Envariable.LISTS: |
| 186 | + yield Envariable.LISTS |
| 187 | + |
| 188 | + # 执行刷新或预热任务 |
| 189 | + @staticmethod |
| 190 | + def doRefresh(lists): |
| 191 | + try: |
| 192 | + if Envariable.get_acs_client(): |
| 193 | + client = Envariable.get_acs_client() |
| 194 | + else: |
| 195 | + Envariable.set_acs_client() |
| 196 | + client = Envariable.get_acs_client() |
| 197 | + |
| 198 | + if Envariable.get_task_type() == 'clear': |
| 199 | + taskID = 'RefreshTaskId' |
| 200 | + request = RefreshObjectCachesRequest() |
| 201 | + if Envariable.get_task_otype(): |
| 202 | + request.set_ObjectType(Envariable.get_task_otype()) |
| 203 | + elif Envariable.get_task_type() == 'push': |
| 204 | + taskID = 'PushTaskId' |
| 205 | + request = PushObjectCacheRequest() |
| 206 | + if Envariable.get_task_area(): |
| 207 | + request.set_Area(Envariable.get_task_area()) |
| 208 | + |
| 209 | + taskreq = DescribeRefreshTasksRequest() |
| 210 | + request.set_accept_format('json') |
| 211 | + request.set_ObjectPath(lists) |
| 212 | + response = json.loads(client.do_action_with_exception(request)) |
| 213 | + print(response) |
| 214 | + |
| 215 | + timeout = 0 |
| 216 | + while True: |
| 217 | + count = 0 |
| 218 | + taskreq.set_accept_format('json') |
| 219 | + taskreq.set_TaskId(response[taskID]) |
| 220 | + taskresp = json.loads(client.do_action_with_exception(taskreq)) |
| 221 | + print(f'[{response[taskID]}] is doing... ...') |
| 222 | + for t in taskresp['Tasks']['CDNTask']: |
| 223 | + if t['Status'] != 'Complete': |
| 224 | + count += 1 |
| 225 | + if count == 0: |
| 226 | + logging.info(f'[{response[taskID]}] is finish') |
| 227 | + break |
| 228 | + elif timeout > 5: |
| 229 | + logging.info(f'[{response[taskID]}] timeout') |
| 230 | + break |
| 231 | + else: |
| 232 | + timeout += 1 |
| 233 | + time.sleep(5) |
| 234 | + continue |
| 235 | + except Exception as e: |
| 236 | + logging.info(f'\n[error]：{e}') |
| 237 | + sys.exit(1) |
| 238 | + |
| 239 | + |
| 240 | +class Refresh(object): |
| 241 | + def main(self, argv): |
| 242 | + if len(argv) < 1: |
| 243 | + sys.exit(f'\n[usage]: {sys.argv[0]} -h ') |
| 244 | + try: |
| 245 | + opts, args = getopt.getopt(argv, 'hi:k:n:r:t:a:o:') |
| 246 | + except getopt.GetoptError: |
| 247 | + sys.exit(f'\n[usage]: {sys.argv[0]} -h ') |
| 248 | + |
| 249 | + for opt, arg in opts: |
| 250 | + if opt == '-h': |
| 251 | + self.help() |
| 252 | + sys.exit() |
| 253 | + elif opt == '-i': |
| 254 | + Envariable.set_ak(arg) |
| 255 | + elif opt == '-k': |
| 256 | + Envariable.set_sk(arg) |
| 257 | + elif opt == '-r': |
| 258 | + Envariable.set_fd(arg) |
| 259 | + elif opt == '-t': |
| 260 | + Envariable.set_task_type(arg) |
| 261 | + elif opt == '-a': |
| 262 | + Envariable.set_task_area(arg) |
| 263 | + elif opt == '-o': |
| 264 | + Envariable.set_task_otype(arg) |
| 265 | + else: |
| 266 | + sys.exit(f'\n[usage]: {sys.argv[0]} -h ') |
| 267 | + |
| 268 | + try: |
| 269 | + if not (Envariable.get_ak() and Envariable.get_sk() and Envariable.get_fd() and Envariable.get_task_type()): |
| 270 | + sys.exit("\n[error]: Must be by parameter '-i', '-k', '-r', '-t'\n") |
| 271 | + if Envariable.get_task_type() not in {'push', 'clear'}: |
| 272 | + sys.exit("\n[error]: taskType Error, '-t' option in 'push' or 'clear'\n") |
| 273 | + if Envariable.get_task_area() and Envariable.get_task_otype(): |
| 274 | + sys.exit('\n[error]: -a and -o cannot exist at same time\n') |
| 275 | + if Envariable.get_task_area(): |
| 276 | + if Envariable.get_task_area() not in {'domestic', 'overseas'}: |
| 277 | + sys.exit("\n[error]: Area value Error, '-a' option in 'domestic' or 'overseas'\n") |
| 278 | + if Envariable.get_task_otype(): |
| 279 | + if Envariable.get_task_otype() not in {'File', 'Directory'}: |
| 280 | + sys.exit("\n[error]: ObjectType value Error, '-a' options in 'File' or 'Directory'\n") |
| 281 | + if Envariable.get_task_type() == 'push': |
| 282 | + sys.exit("\n[error]: -t must be clear and 'push' -a use together\n") |
| 283 | + except Exception as e: |
| 284 | + logging.info(f'\n[error]: Parameter {e} error\n') |
| 285 | + sys.exit(1) |
| 286 | + |
| 287 | + handler = BaseCheck() |
| 288 | + if handler.urlFormat() and handler.printQuota(): |
| 289 | + for g in doTask.doProd(): |
| 290 | + doTask.doRefresh(''.join(g)) |
| 291 | + time.sleep(1) |
| 292 | + |
| 293 | + def help(self): |
| 294 | + print( |
| 295 | + '\nscript options explain: \ |
| 296 | + \n\t -i <AccessKey> 访问阿里云凭证，访问控制台上可以获得； \ |
| 297 | + \n\t -k <AccessKeySecret> 访问阿里云密钥，访问控制台上可以获得； \ |
| 298 | + \n\t -r <filename> filename指“文件所在的路径+文件名称”，自动化脚本运行后将会读取文件内记录的URL；文件内的URL记录方式为每行一条URL，有特殊字符先做URLencode，以http或https开头； \ |
| 299 | + \n\t -t <taskType> 任务类型，clear：刷新，push：预热； \ |
| 300 | + \n\t -a [String,<domestic|overseas>] 可选项，预热范围，不传默认是全球；\ |
| 301 | + \n\t domestic 仅中国内地； \ |
| 302 | + \n\t overseas 全球（不包含中国内地）； \ |
| 303 | + \n\t -o [String,<File|Directory>] 可选项，刷新的类型； \ |
| 304 | + \n\t File 文件刷新（默认值）； \ |
| 305 | + \n\t Directory 目录刷新' |
| 306 | + ) |
| 307 | + |
| 308 | + |
| 309 | +if __name__ == '__main__': |
| 310 | + fun = Refresh() |
| 311 | + fun.main(sys.argv[1:]) |
0 commit comments