优化代码

fir_ser/api/utils/modelutils.py

@@ -259,7 +259,7 @@ def check_app_access_token(app_id, access_token, only_check, udid):
         return AppDownloadToken.objects.filter(app_id__app_id=app_id, bind_udid=udid).count()
 
 
-@MagicCacheData.make_cache(60 * 60 * 24, key=lambda x: x.app_id)
+@MagicCacheData.make_cache(timeout=60 * 60 * 24, key_func=lambda x: x.app_id)
 def get_app_storage_used(app_obj):
     binary_size_sum = 0
     for release_obj in AppReleaseInfo.objects.filter(app_id=app_obj).all():

fir_ser/api/views/download.py

@@ -18,7 +18,8 @@ from api.utils.response import BaseResponse
 from api.utils.serializer import AppsShortSerializer, AppAdInfoSerializer
 from api.utils.signalutils import run_get_xsign_binary_file
 from common.base.baseutils import get_origin_domain_name, format_get_uri, make_random_uuid, make_resigned
-from common.core.decorators import cache_response
+from common.base.magic import cache_response
+from common.cache.storage import AppDownloadShortShowCache
 from common.core.response import mobileprovision_file_response, file_response, ApiResponse
 from common.core.sysconfig import Config
 from common.core.throttle import VisitShortThrottle, InstallShortThrottle, InstallThrottle1, InstallThrottle2
@@ -97,7 +98,7 @@ class ShortDownloadView(APIView):
     根据下载短链接，获取应用信息
     '''
 
-    @cache_response(timeout=600 - 60, cache="default", key_func='calculate_cache_key', cache_errors=False)
+    @cache_response(timeout=600 - 60, key_func='calculate_cache_key', callback_func='set_short_show_cache')
     def get(self, request, short):
         res = BaseResponse()
         release_id = request.query_params.get("release_id", None)
@@ -145,6 +146,19 @@ class ShortDownloadView(APIView):
                 res.ad = AppAdInfoSerializer(ad_obj, context={"key": "ShortDownloadView", "short": short}).data
         return Response(res.dict)
 
+    def set_short_show_cache(self, view_instance, view_method,
+                             request, args, kwargs, cache_key):
+        short = kwargs.get("short", '')
+        short_show_cache = AppDownloadShortShowCache("ShortDownloadView".lower(), short)
+        key_list = short_show_cache.get_storage_cache()
+
+        if key_list and isinstance(key_list, list):
+            key_list.append(cache_key)
+            key_list = list(set(key_list))
+        else:
+            key_list = [cache_key]
+        short_show_cache.set_storage_cache(key_list, 600)
+
     # key的设置
     def calculate_cache_key(self, view_instance, view_method,
                             request, args, kwargs):

fir_ser/common/base/magic.py

@@ -6,10 +6,11 @@
 import datetime
 import logging
 import time
-from functools import wraps
+from functools import wraps, WRAPPER_ASSIGNMENTS
 from importlib import import_module
 
 from django.core.cache import cache
+from django.http import HttpResponse
 
 logger = logging.getLogger(__name__)
 
@@ -173,38 +174,51 @@ def magic_call_in_times(call_time=24 * 3600, call_limit=6, key=None):
 
 class MagicCacheData(object):
     @staticmethod
-    def make_cache(cache_time=60 * 10, invalid_time=0, key=None):
+    def make_cache(timeout=60 * 10, invalid_time=0, key_func=None, timeout_func=None):
         """
-        :param cache_time:  数据缓存的时候，单位秒
+        :param timeout_func:
+        :param timeout:  数据缓存的时候，单位秒
         :param invalid_time: 数据缓存提前失效时间，单位秒。该cache有效时间为 cache_time-invalid_time
-        :param key: cache唯一标识，默认为所装饰函数名称
+        :param key_func: cache唯一标识，默认为所装饰函数名称
         :return:
         """
 
         def decorator(func):
             @wraps(func)
             def wrapper(*args, **kwargs):
-                cache_key = f'magic_cache_data'
-                if key:
-                    cache_key = f'{cache_key}_{key(*args, **kwargs)}'
-                else:
-                    cache_key = f'{cache_key}_{func.__name__}'
+                cache_key = f'magic_cache_data_{func.__name__}'
+                if key_func:
+                    cache_key = f'{cache_key}_{key_func(*args, **kwargs)}'
+
+                cache_time = timeout
+                if timeout_func:
+                    cache_time = timeout_func(*args, **kwargs)
 
                 n_time = time.time()
                 res = cache.get(cache_key)
+                if res:
+                    while res.get('status') != 'ok':
+                        time.sleep(0.5)
+                        logger.warning(
+                            f'exec {func} wait. data status is not ok. cache_time:{cache_time} cache_key:{cache_key}  cache data exist result:{res}')
+                        res = cache.get(cache_key)
+
                 if res and n_time - res.get('c_time', n_time) < cache_time - invalid_time:
-                    logger.info(f"exec {func} finished. cache_key:{cache_key}  cache data exist result:{res}")
+                    logger.info(
+                        f"exec {func} finished. cache_time:{cache_time} cache_key:{cache_key} cache data exist result:{res}")
                     return res['data']
                 else:
+                    res = {'c_time': n_time, 'data': '', 'status': 'ready'}
+                    cache.set(cache_key, res, cache_time)
                     try:
-                        data = func(*args, **kwargs)
-                        res = {'c_time': n_time, 'data': data}
+                        res['data'] = func(*args, **kwargs)
+                        res['status'] = 'ok'
                         cache.set(cache_key, res, cache_time)
                         logger.info(
-                            f"exec {func} finished. time:{time.time() - n_time}  cache_key:{cache_key} result:{res}")
+                            f"exec {func} finished. time:{time.time() - n_time} cache_time:{cache_time} cache_key:{cache_key} result:{res}")
                     except Exception as e:
                         logger.error(
-                            f"exec {func} failed. time:{time.time() - n_time}  cache_key:{cache_key} Exception:{e}")
+                            f"exec {func} failed. time:{time.time() - n_time}  cache_time:{cache_time} cache_key:{cache_key} Exception:{e}")
 
                     return res['data']
 
@@ -217,3 +231,141 @@ class MagicCacheData(object):
         cache_key = f'magic_cache_data_{key}'
         res = cache.delete(cache_key)
         logger.warning(f"invalid_cache cache_key:{cache_key} result:{res}")
+
+
+class MagicCacheResponse(object):
+    def __init__(self, timeout=60 * 10, invalid_time=0, key_func=None, callback_func=None):
+        self.timeout = timeout
+        self.key_func = key_func
+        self.invalid_time = invalid_time
+        self.callback_func = callback_func
+
+    @staticmethod
+    def invalid_cache(key):
+        cache_key = f'magic_cache_response_{key}'
+        res = cache.delete(cache_key)
+        logger.warning(f"invalid_response_cache cache_key:{cache_key} result:{res}")
+
+    def __call__(self, func):
+        this = self
+
+        @wraps(func, assigned=WRAPPER_ASSIGNMENTS)
+        def inner(self, request, *args, **kwargs):
+            return this.process_cache_response(
+                view_instance=self,
+                view_method=func,
+                request=request,
+                args=args,
+                kwargs=kwargs,
+            )
+
+        return inner
+
+    def process_cache_response(self,
+                               view_instance,
+                               view_method,
+                               request,
+                               args,
+                               kwargs):
+        func_key = self.calculate_key(
+            view_instance=view_instance,
+            view_method=view_method,
+            request=request,
+            args=args,
+            kwargs=kwargs
+        )
+        func_name = f'{view_instance.__class__.__name__}_{view_method.__name__}'
+        cache_key = f'magic_cache_response_{func_name}'
+        if func_key:
+            cache_key = f'{cache_key}_{func_key}'
+
+        timeout = self.calculate_timeout(view_instance=view_instance)
+        n_time = time.time()
+        res = cache.get(cache_key)
+        if res and n_time - res.get('c_time', n_time) < timeout - self.invalid_time:
+            logger.info(f"exec {func_name} finished. cache_key:{cache_key}  cache data exist result:{res}")
+            content, status, headers = res['data']
+            response = HttpResponse(content=content, status=status)
+            for k, v in headers.values():
+                response[k] = v
+        else:
+            response = view_method(view_instance, request, *args, **kwargs)
+            response = view_instance.finalize_response(request, response, *args, **kwargs)
+            response.render()
+
+            if not response.status_code >= 400:
+                # django 3.0 has not .items() method, django 3.2 has not ._headers
+                if hasattr(response, '_headers'):
+                    headers = response._headers.copy()
+                else:
+                    headers = {k: (k, v) for k, v in response.items()}
+                data = (
+                    response.rendered_content,
+                    response.status_code,
+                    headers
+                )
+                res = {'c_time': n_time, 'data': data}
+                cache.set(cache_key, res, timeout)
+
+                self.callback_check(view_instance=view_instance,
+                                    view_method=view_method,
+                                    request=request,
+                                    args=args,
+                                    kwargs=kwargs,
+                                    cache_key=cache_key)
+
+                logger.info(
+                    f"exec {func_name} finished. time:{time.time() - n_time}  cache_key:{cache_key} result:{res}")
+
+        if not hasattr(response, '_closable_objects'):
+            response._closable_objects = []
+
+        return response
+
+    def calculate_key(self,
+                      view_instance,
+                      view_method,
+                      request,
+                      args,
+                      kwargs):
+        if isinstance(self.key_func, str):
+            key_func = getattr(view_instance, self.key_func)
+        else:
+            key_func = self.key_func
+        if key_func:
+            return key_func(
+                view_instance=view_instance,
+                view_method=view_method,
+                request=request,
+                args=args,
+                kwargs=kwargs,
+            )
+
+    def calculate_timeout(self, view_instance, **_):
+        if isinstance(self.timeout, str):
+            self.timeout = getattr(view_instance, self.timeout)
+        return self.timeout
+
+    def callback_check(self,
+                       view_instance,
+                       view_method,
+                       request,
+                       args,
+                       kwargs,
+                       cache_key):
+        if isinstance(self.callback_func, str):
+            callback_func = getattr(view_instance, self.callback_func)
+        else:
+            callback_func = self.callback_func
+        if callback_func:
+            return callback_func(
+                view_instance=view_instance,
+                view_method=view_method,
+                request=request,
+                args=args,
+                kwargs=kwargs,
+                cache_key=cache_key
+            )
+
+
+cache_response = MagicCacheResponse

fir_ser/common/core/decorators.py

@@ -1,146 +0,0 @@
-from functools import wraps, WRAPPER_ASSIGNMENTS
-
-from django.http.response import HttpResponse
-
-from common.cache.storage import AppDownloadShortShowCache
-
-
-def get_cache(alias):
-    from django.core.cache import caches
-    return caches[alias]
-
-
-def set_short_show_cache(short, cache_key):
-    short_show_cache = AppDownloadShortShowCache("ShortDownloadView".lower(), short)
-    key_list = short_show_cache.get_storage_cache()
-
-    if key_list and isinstance(key_list, list):
-        key_list.append(cache_key)
-        key_list = list(set(key_list))
-    else:
-        key_list = [cache_key]
-    short_show_cache.set_storage_cache(key_list, 600)
-
-
-class CacheResponse:
-    """
-    Store/Receive and return cached `HttpResponse` based on DRF response.
-    .. note::
-        This decorator will render and discard the original DRF response in
-        favor of Django's `HttpResponse`. The allows the cache to retain a
-        smaller memory footprint and eliminates the need to re-render
-        responses on each request. Furthermore it eliminates the risk for users
-        to unknowingly cache whole Serializers and QuerySets.
-    """
-
-    def __init__(self,
-                 timeout=None,
-                 key_func=None,
-                 cache=None,
-                 cache_errors=None):
-        if timeout is None:
-            self.timeout = None
-        else:
-            self.timeout = timeout
-
-        if key_func is None:
-            self.key_func = ''
-        else:
-            self.key_func = key_func
-
-        if cache_errors is None:
-            self.cache_errors = True
-        else:
-            self.cache_errors = cache_errors
-
-        self.cache = get_cache(cache or 'default')
-
-    def __call__(self, func):
-        this = self
-
-        @wraps(func, assigned=WRAPPER_ASSIGNMENTS)
-        def inner(self, request, *args, **kwargs):
-            return this.process_cache_response(
-                view_instance=self,
-                view_method=func,
-                request=request,
-                args=args,
-                kwargs=kwargs,
-            )
-
-        return inner
-
-    def process_cache_response(self,
-                               view_instance,
-                               view_method,
-                               request,
-                               args,
-                               kwargs):
-
-        key = self.calculate_key(
-            view_instance=view_instance,
-            view_method=view_method,
-            request=request,
-            args=args,
-            kwargs=kwargs
-        )
-
-        timeout = self.calculate_timeout(view_instance=view_instance)
-
-        response_triple = self.cache.get(key)
-        if not response_triple:
-            # render response to create and cache the content byte string
-            response = view_method(view_instance, request, *args, **kwargs)
-            response = view_instance.finalize_response(request, response, *args, **kwargs)
-            response.render()
-
-            if not response.status_code >= 400 or self.cache_errors:
-                # django 3.0 has not .items() method, django 3.2 has not ._headers
-                if hasattr(response, '_headers'):
-                    headers = response._headers.copy()
-                else:
-                    headers = {k: (k, v) for k, v in response.items()}
-                response_triple = (
-                    response.rendered_content,
-                    response.status_code,
-                    headers
-                )
-                short = kwargs.get("short", '')
-                set_short_show_cache(short, key)
-                self.cache.set(key, response_triple, timeout)
-        else:
-            # build smaller Django HttpResponse
-            content, status, headers = response_triple
-            response = HttpResponse(content=content, status=status)
-            for k, v in headers.values():
-                response[k] = v
-        if not hasattr(response, '_closable_objects'):
-            response._closable_objects = []
-
-        return response
-
-    def calculate_key(self,
-                      view_instance,
-                      view_method,
-                      request,
-                      args,
-                      kwargs):
-        if isinstance(self.key_func, str):
-            key_func = getattr(view_instance, self.key_func)
-        else:
-            key_func = self.key_func
-        return key_func(
-            view_instance=view_instance,
-            view_method=view_method,
-            request=request,
-            args=args,
-            kwargs=kwargs,
-        )
-
-    def calculate_timeout(self, view_instance, **_):
-        if isinstance(self.timeout, str):
-            self.timeout = getattr(view_instance, self.timeout)
-        return self.timeout
-
-
-cache_response = CacheResponse

fir_ser/xsign/utils/supersignutils.py

@@ -626,9 +626,11 @@ class IosUtils(object):
                         sign_status=SignStatus.SIGNATURE_PACKAGE_COMPLETE)
                     base_app_udid = AppUDID.objects.filter(app_id=app_obj, udid__developerid_id=developer_obj_id)
                     if base_app_udid.filter(sign_status__lt=SignStatus.SIGNATURE_PACKAGE_COMPLETE).count():
-                        c_time = base_app_udid.order_by('-created_time').first()
-                        u_time = base_app_udid.order_by('-updated_time').first()
-                        if u_time.updated_time > c_time.created_time:
+                        c_time = base_app_udid.filter(sign_status__lt=SignStatus.SIGNATURE_PACKAGE_COMPLETE).order_by(
+                            '-created_time').first()
+                        u_time = base_app_udid.filter(sign_status=SignStatus.SIGNATURE_PACKAGE_COMPLETE).order_by(
+                            '-updated_time').first()
+                        if c_time and u_time and u_time.updated_time > c_time.created_time:
                             base_app_udid.update(sign_status=SignStatus.SIGNATURE_PACKAGE_COMPLETE)
 
             del_cache_response_by_short(app_obj.app_id)