From dd802dccbbf59634dd15e449c71bbdfa2606f4a2 Mon Sep 17 00:00:00 2001 From: isummer Date: Sat, 12 Nov 2022 22:27:10 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fir_ser/api/utils/modelutils.py | 2 +- fir_ser/api/views/download.py | 18 ++- fir_ser/common/base/magic.py | 180 ++++++++++++++++++++++++-- fir_ser/common/core/decorators.py | 146 --------------------- fir_ser/xsign/utils/supersignutils.py | 8 +- 5 files changed, 188 insertions(+), 166 deletions(-) delete mode 100644 fir_ser/common/core/decorators.py diff --git a/fir_ser/api/utils/modelutils.py b/fir_ser/api/utils/modelutils.py index 3344206..947cc8b 100644 --- a/fir_ser/api/utils/modelutils.py +++ b/fir_ser/api/utils/modelutils.py @@ -259,7 +259,7 @@ def check_app_access_token(app_id, access_token, only_check, udid): return AppDownloadToken.objects.filter(app_id__app_id=app_id, bind_udid=udid).count() -@MagicCacheData.make_cache(60 * 60 * 24, key=lambda x: x.app_id) +@MagicCacheData.make_cache(timeout=60 * 60 * 24, key_func=lambda x: x.app_id) def get_app_storage_used(app_obj): binary_size_sum = 0 for release_obj in AppReleaseInfo.objects.filter(app_id=app_obj).all(): diff --git a/fir_ser/api/views/download.py b/fir_ser/api/views/download.py index 10b2179..4f703ef 100644 --- a/fir_ser/api/views/download.py +++ b/fir_ser/api/views/download.py @@ -18,7 +18,8 @@ from api.utils.response import BaseResponse from api.utils.serializer import AppsShortSerializer, AppAdInfoSerializer from api.utils.signalutils import run_get_xsign_binary_file from common.base.baseutils import get_origin_domain_name, format_get_uri, make_random_uuid, make_resigned -from common.core.decorators import cache_response +from common.base.magic import cache_response +from common.cache.storage import AppDownloadShortShowCache from common.core.response import mobileprovision_file_response, file_response, ApiResponse from common.core.sysconfig import Config from common.core.throttle import VisitShortThrottle, InstallShortThrottle, InstallThrottle1, InstallThrottle2 @@ -97,7 +98,7 @@ class ShortDownloadView(APIView): 根据下载短链接，获取应用信息 ''' - @cache_response(timeout=600 - 60, cache="default", key_func='calculate_cache_key', cache_errors=False) + @cache_response(timeout=600 - 60, key_func='calculate_cache_key', callback_func='set_short_show_cache') def get(self, request, short): res = BaseResponse() release_id = request.query_params.get("release_id", None) @@ -145,6 +146,19 @@ class ShortDownloadView(APIView): res.ad = AppAdInfoSerializer(ad_obj, context={"key": "ShortDownloadView", "short": short}).data return Response(res.dict) + def set_short_show_cache(self, view_instance, view_method, + request, args, kwargs, cache_key): + short = kwargs.get("short", '') + short_show_cache = AppDownloadShortShowCache("ShortDownloadView".lower(), short) + key_list = short_show_cache.get_storage_cache() + + if key_list and isinstance(key_list, list): + key_list.append(cache_key) + key_list = list(set(key_list)) + else: + key_list = [cache_key] + short_show_cache.set_storage_cache(key_list, 600) + # key的设置 def calculate_cache_key(self, view_instance, view_method, request, args, kwargs): diff --git a/fir_ser/common/base/magic.py b/fir_ser/common/base/magic.py index 6724fa0..7f842cc 100644 --- a/fir_ser/common/base/magic.py +++ b/fir_ser/common/base/magic.py @@ -6,10 +6,11 @@ import datetime import logging import time -from functools import wraps +from functools import wraps, WRAPPER_ASSIGNMENTS from importlib import import_module from django.core.cache import cache +from django.http import HttpResponse logger = logging.getLogger(__name__) @@ -173,38 +174,51 @@ def magic_call_in_times(call_time=24 * 3600, call_limit=6, key=None): class MagicCacheData(object): @staticmethod - def make_cache(cache_time=60 * 10, invalid_time=0, key=None): + def make_cache(timeout=60 * 10, invalid_time=0, key_func=None, timeout_func=None): """ - :param cache_time: 数据缓存的时候，单位秒 + :param timeout_func: + :param timeout: 数据缓存的时候，单位秒 :param invalid_time: 数据缓存提前失效时间，单位秒。该cache有效时间为 cache_time-invalid_time - :param key: cache唯一标识，默认为所装饰函数名称 + :param key_func: cache唯一标识，默认为所装饰函数名称 :return: """ def decorator(func): @wraps(func) def wrapper(*args, **kwargs): - cache_key = f'magic_cache_data' - if key: - cache_key = f'{cache_key}_{key(*args, **kwargs)}' - else: - cache_key = f'{cache_key}_{func.__name__}' + cache_key = f'magic_cache_data_{func.__name__}' + if key_func: + cache_key = f'{cache_key}_{key_func(*args, **kwargs)}' + + cache_time = timeout + if timeout_func: + cache_time = timeout_func(*args, **kwargs) n_time = time.time() res = cache.get(cache_key) + if res: + while res.get('status') != 'ok': + time.sleep(0.5) + logger.warning( + f'exec {func} wait. data status is not ok. cache_time:{cache_time} cache_key:{cache_key} cache data exist result:{res}') + res = cache.get(cache_key) + if res and n_time - res.get('c_time', n_time) < cache_time - invalid_time: - logger.info(f"exec {func} finished. cache_key:{cache_key} cache data exist result:{res}") + logger.info( + f"exec {func} finished. cache_time:{cache_time} cache_key:{cache_key} cache data exist result:{res}") return res['data'] else: + res = {'c_time': n_time, 'data': '', 'status': 'ready'} + cache.set(cache_key, res, cache_time) try: - data = func(*args, **kwargs) - res = {'c_time': n_time, 'data': data} + res['data'] = func(*args, **kwargs) + res['status'] = 'ok' cache.set(cache_key, res, cache_time) logger.info( - f"exec {func} finished. time:{time.time() - n_time} cache_key:{cache_key} result:{res}") + f"exec {func} finished. time:{time.time() - n_time} cache_time:{cache_time} cache_key:{cache_key} result:{res}") except Exception as e: logger.error( - f"exec {func} failed. time:{time.time() - n_time} cache_key:{cache_key} Exception:{e}") + f"exec {func} failed. time:{time.time() - n_time} cache_time:{cache_time} cache_key:{cache_key} Exception:{e}") return res['data'] @@ -217,3 +231,141 @@ class MagicCacheData(object): cache_key = f'magic_cache_data_{key}' res = cache.delete(cache_key) logger.warning(f"invalid_cache cache_key:{cache_key} result:{res}") + + +class MagicCacheResponse(object): + def __init__(self, timeout=60 * 10, invalid_time=0, key_func=None, callback_func=None): + self.timeout = timeout + self.key_func = key_func + self.invalid_time = invalid_time + self.callback_func = callback_func + + @staticmethod + def invalid_cache(key): + cache_key = f'magic_cache_response_{key}' + res = cache.delete(cache_key) + logger.warning(f"invalid_response_cache cache_key:{cache_key} result:{res}") + + def __call__(self, func): + this = self + + @wraps(func, assigned=WRAPPER_ASSIGNMENTS) + def inner(self, request, *args, **kwargs): + return this.process_cache_response( + view_instance=self, + view_method=func, + request=request, + args=args, + kwargs=kwargs, + ) + + return inner + + def process_cache_response(self, + view_instance, + view_method, + request, + args, + kwargs): + func_key = self.calculate_key( + view_instance=view_instance, + view_method=view_method, + request=request, + args=args, + kwargs=kwargs + ) + func_name = f'{view_instance.__class__.__name__}_{view_method.__name__}' + cache_key = f'magic_cache_response_{func_name}' + if func_key: + cache_key = f'{cache_key}_{func_key}' + + timeout = self.calculate_timeout(view_instance=view_instance) + n_time = time.time() + res = cache.get(cache_key) + if res and n_time - res.get('c_time', n_time) < timeout - self.invalid_time: + logger.info(f"exec {func_name} finished. cache_key:{cache_key} cache data exist result:{res}") + content, status, headers = res['data'] + response = HttpResponse(content=content, status=status) + for k, v in headers.values(): + response[k] = v + else: + response = view_method(view_instance, request, *args, **kwargs) + response = view_instance.finalize_response(request, response, *args, **kwargs) + response.render() + + if not response.status_code >= 400: + # django 3.0 has not .items() method, django 3.2 has not ._headers + if hasattr(response, '_headers'): + headers = response._headers.copy() + else: + headers = {k: (k, v) for k, v in response.items()} + data = ( + response.rendered_content, + response.status_code, + headers + ) + res = {'c_time': n_time, 'data': data} + cache.set(cache_key, res, timeout) + + self.callback_check(view_instance=view_instance, + view_method=view_method, + request=request, + args=args, + kwargs=kwargs, + cache_key=cache_key) + + logger.info( + f"exec {func_name} finished. time:{time.time() - n_time} cache_key:{cache_key} result:{res}") + + if not hasattr(response, '_closable_objects'): + response._closable_objects = [] + + return response + + def calculate_key(self, + view_instance, + view_method, + request, + args, + kwargs): + if isinstance(self.key_func, str): + key_func = getattr(view_instance, self.key_func) + else: + key_func = self.key_func + if key_func: + return key_func( + view_instance=view_instance, + view_method=view_method, + request=request, + args=args, + kwargs=kwargs, + ) + + def calculate_timeout(self, view_instance, **_): + if isinstance(self.timeout, str): + self.timeout = getattr(view_instance, self.timeout) + return self.timeout + + def callback_check(self, + view_instance, + view_method, + request, + args, + kwargs, + cache_key): + if isinstance(self.callback_func, str): + callback_func = getattr(view_instance, self.callback_func) + else: + callback_func = self.callback_func + if callback_func: + return callback_func( + view_instance=view_instance, + view_method=view_method, + request=request, + args=args, + kwargs=kwargs, + cache_key=cache_key + ) + + +cache_response = MagicCacheResponse diff --git a/fir_ser/common/core/decorators.py b/fir_ser/common/core/decorators.py deleted file mode 100644 index 07e46ad..0000000 --- a/fir_ser/common/core/decorators.py +++ /dev/null @@ -1,146 +0,0 @@ -from functools import wraps, WRAPPER_ASSIGNMENTS - -from django.http.response import HttpResponse - -from common.cache.storage import AppDownloadShortShowCache - - -def get_cache(alias): - from django.core.cache import caches - return caches[alias] - - -def set_short_show_cache(short, cache_key): - short_show_cache = AppDownloadShortShowCache("ShortDownloadView".lower(), short) - key_list = short_show_cache.get_storage_cache() - - if key_list and isinstance(key_list, list): - key_list.append(cache_key) - key_list = list(set(key_list)) - else: - key_list = [cache_key] - short_show_cache.set_storage_cache(key_list, 600) - - -class CacheResponse: - """ - Store/Receive and return cached `HttpResponse` based on DRF response. - .. note:: - This decorator will render and discard the original DRF response in - favor of Django's `HttpResponse`. The allows the cache to retain a - smaller memory footprint and eliminates the need to re-render - responses on each request. Furthermore it eliminates the risk for users - to unknowingly cache whole Serializers and QuerySets. - """ - - def __init__(self, - timeout=None, - key_func=None, - cache=None, - cache_errors=None): - if timeout is None: - self.timeout = None - else: - self.timeout = timeout - - if key_func is None: - self.key_func = '' - else: - self.key_func = key_func - - if cache_errors is None: - self.cache_errors = True - else: - self.cache_errors = cache_errors - - self.cache = get_cache(cache or 'default') - - def __call__(self, func): - this = self - - @wraps(func, assigned=WRAPPER_ASSIGNMENTS) - def inner(self, request, *args, **kwargs): - return this.process_cache_response( - view_instance=self, - view_method=func, - request=request, - args=args, - kwargs=kwargs, - ) - - return inner - - def process_cache_response(self, - view_instance, - view_method, - request, - args, - kwargs): - - key = self.calculate_key( - view_instance=view_instance, - view_method=view_method, - request=request, - args=args, - kwargs=kwargs - ) - - timeout = self.calculate_timeout(view_instance=view_instance) - - response_triple = self.cache.get(key) - if not response_triple: - # render response to create and cache the content byte string - response = view_method(view_instance, request, *args, **kwargs) - response = view_instance.finalize_response(request, response, *args, **kwargs) - response.render() - - if not response.status_code >= 400 or self.cache_errors: - # django 3.0 has not .items() method, django 3.2 has not ._headers - if hasattr(response, '_headers'): - headers = response._headers.copy() - else: - headers = {k: (k, v) for k, v in response.items()} - response_triple = ( - response.rendered_content, - response.status_code, - headers - ) - short = kwargs.get("short", '') - set_short_show_cache(short, key) - self.cache.set(key, response_triple, timeout) - else: - # build smaller Django HttpResponse - content, status, headers = response_triple - response = HttpResponse(content=content, status=status) - for k, v in headers.values(): - response[k] = v - if not hasattr(response, '_closable_objects'): - response._closable_objects = [] - - return response - - def calculate_key(self, - view_instance, - view_method, - request, - args, - kwargs): - if isinstance(self.key_func, str): - key_func = getattr(view_instance, self.key_func) - else: - key_func = self.key_func - return key_func( - view_instance=view_instance, - view_method=view_method, - request=request, - args=args, - kwargs=kwargs, - ) - - def calculate_timeout(self, view_instance, **_): - if isinstance(self.timeout, str): - self.timeout = getattr(view_instance, self.timeout) - return self.timeout - - -cache_response = CacheResponse diff --git a/fir_ser/xsign/utils/supersignutils.py b/fir_ser/xsign/utils/supersignutils.py index 38181fd..75c638b 100644 --- a/fir_ser/xsign/utils/supersignutils.py +++ b/fir_ser/xsign/utils/supersignutils.py @@ -626,9 +626,11 @@ class IosUtils(object): sign_status=SignStatus.SIGNATURE_PACKAGE_COMPLETE) base_app_udid = AppUDID.objects.filter(app_id=app_obj, udid__developerid_id=developer_obj_id) if base_app_udid.filter(sign_status__lt=SignStatus.SIGNATURE_PACKAGE_COMPLETE).count(): - c_time = base_app_udid.order_by('-created_time').first() - u_time = base_app_udid.order_by('-updated_time').first() - if u_time.updated_time > c_time.created_time: + c_time = base_app_udid.filter(sign_status__lt=SignStatus.SIGNATURE_PACKAGE_COMPLETE).order_by( + '-created_time').first() + u_time = base_app_udid.filter(sign_status=SignStatus.SIGNATURE_PACKAGE_COMPLETE).order_by( + '-updated_time').first() + if c_time and u_time and u_time.updated_time > c_time.created_time: base_app_udid.update(sign_status=SignStatus.SIGNATURE_PACKAGE_COMPLETE) del_cache_response_by_short(app_obj.app_id)