From 58e7f8fa6ac0184f89e0acea9e5da3e6b831ae7c Mon Sep 17 00:00:00 2001 From: youngS Date: Thu, 29 Jul 2021 20:06:48 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E5=90=8D=E8=AE=A4=E8=AF=81=E7=85=A7?= =?UTF-8?q?=E7=89=87=E5=AD=98=E6=94=BE=E5=88=B0=E6=9C=8D=E5=8A=A1=E5=99=A8?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=AD=98=E5=82=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fir_ser/api/utils/serializer.py | 4 +- fir_ser/api/utils/storage/storage.py | 155 ++++++++++++++------------- fir_ser/api/views/login.py | 3 +- fir_ser/api/views/uploads.py | 5 +- 4 files changed, 90 insertions(+), 77 deletions(-) diff --git a/fir_ser/api/utils/serializer.py b/fir_ser/api/utils/serializer.py index 13923b3..09ef732 100644 --- a/fir_ser/api/utils/serializer.py +++ b/fir_ser/api/utils/serializer.py @@ -21,12 +21,14 @@ def get_download_url_from_context(self, obj, key, url, force_new=False): if self.context.get("storage", None) and self.context.get("storage") != "undefined": storage = self.context.get("storage", None) else: - if isinstance(obj, models.Apps) or isinstance(obj, models.UserCertificationInfo): + if isinstance(obj, models.Apps): storage = Storage(obj.user_id) elif isinstance(obj, models.AppReleaseInfo): storage = Storage(obj.app_id.user_id) elif isinstance(obj, models.UserInfo): storage = Storage(obj) + elif isinstance(obj, models.UserCertificationInfo) or isinstance(obj, models.CertificationInfo): + storage = Storage(obj.user_id, None, True) else: storage = None if storage: diff --git a/fir_ser/api/utils/storage/storage.py b/fir_ser/api/utils/storage/storage.py index 4c10f26..5da1eb2 100644 --- a/fir_ser/api/utils/storage/storage.py +++ b/fir_ser/api/utils/storage/storage.py @@ -17,11 +17,62 @@ import logging logger = logging.getLogger(__file__) +def get_storage_auth(storage_obj): + filter_fields = ['access_key', 'secret_key', 'bucket_name', 'domain_name', 'is_https', 'endpoint', + 'sts_role_arn', 'cnd_auth_key', 'download_auth_type'] + return get_dict_from_filter_fields(filter_fields, storage_obj.__dict__) + + +def get_storage(user, assigned_storage_obj, use_default_storage): + if use_default_storage: + return get_storage_form_conf(user) + if assigned_storage_obj: + storage_obj = assigned_storage_obj + else: + storage_obj = user.storage + + if storage_obj: + auth = get_storage_auth(storage_obj) + storage_key = "_".join([CACHE_KEY_TEMPLATE.get('user_storage_key'), user.uid, + base64.b64encode(json.dumps(auth).encode("utf-8")).decode("utf-8")[0:64]]) + storage_type = storage_obj.storage_type + new_storage_obj = cache.get(storage_key) + if new_storage_obj and not assigned_storage_obj: + logger.info("user %s get storage obj %s cache %s" % (user, storage_key, new_storage_obj)) + return new_storage_obj + else: + if storage_type == 1: + new_storage_obj = QiNiuOss(**auth) + elif storage_type == 2: + new_storage_obj = AliYunOss(**auth) + else: + new_storage_obj = LocalStorage(**auth) + logger.warning("user %s make storage obj key:%s obj: %s" % (user, storage_key, new_storage_obj)) + new_storage_obj.storage_type = storage_type + cache.set(storage_key, new_storage_obj, 600) + return new_storage_obj + else: + logger.info("user %s has not storage obj, so get default" % user) + # return self.get_default_storage(user, storage_obj, False) + # 不需要管理存储，直接从配置文件获取默认存储 + return get_storage_form_conf(user) + + +def get_default_storage(user, storage_obj, use_default_storage): + admin_obj = UserInfo.objects.filter(is_superuser=True).order_by('pk').first() + if admin_obj and admin_obj.storage and admin_obj.pk != user.pk: + logger.info("user %s has not storage obj, from admin " + "get default storage" % user) + return get_storage(admin_obj, storage_obj, use_default_storage) + else: + return get_storage_form_conf(user) + + class Storage(object): - def __init__(self, user, storage_obj=None, use_default_storage=False): + def __init__(self, user, assigned_storage_obj=None, use_default_storage=False): try: with cache.lock("%s_%s" % ('make_storage_cache', user.uid), timeout=10, blocking_timeout=6): - self.storage = self.get_storage(user, storage_obj, use_default_storage) + self.storage = get_storage(user, assigned_storage_obj, use_default_storage) except Exception as e: logger.error("get %s storage failed Exception:%s" % (user, e)) self.storage = None @@ -80,81 +131,10 @@ class Storage(object): except Exception as e: logger.error("oss download %s failed Exception %s" % (local_file_full_path, e)) - def get_storage(self, user, storage_obj, use_default_storage): - if storage_obj: - self.storage_obj = storage_obj - else: - self.storage_obj = user.storage - if use_default_storage: - self.storage_obj = None - - if self.storage_obj: - auth = self.get_storage_auth() - storage_key = "_".join([CACHE_KEY_TEMPLATE.get('user_storage_key'), user.uid, - base64.b64encode(json.dumps(auth).encode("utf-8")).decode("utf-8")[0:64]]) - storage_type = self.storage_obj.storage_type - new_storage_obj = cache.get(storage_key) - if new_storage_obj and not storage_obj: - logger.info("user %s get storage obj cache %s" % (user, new_storage_obj)) - return new_storage_obj - else: - if storage_type == 1: - new_storage_obj = QiNiuOss(**auth) - elif storage_type == 2: - new_storage_obj = AliYunOss(**auth) - else: - new_storage_obj = LocalStorage(**auth) - logger.warning("user %s make storage obj %s" % (user, new_storage_obj)) - new_storage_obj.storage_type = storage_type - cache.set(storage_key, new_storage_obj, 600) - return new_storage_obj - else: - logger.info("user %s has not storage obj, so get default" % user) - return self.get_default_storage(user, storage_obj, False) - - def get_default_storage(self, user, storage_obj, use_default_storage): - admin_obj = UserInfo.objects.filter(is_superuser=True).order_by('pk').first() - if admin_obj and admin_obj.storage and admin_obj.pk != user.pk: - logger.info("user %s has not storage obj, from admin " - "get default storage" % user) - return self.get_storage(admin_obj, storage_obj, use_default_storage) - else: - storage_lists = THIRD_PART_CONFIG.get('storage') - for storage in storage_lists: - if storage.get("active", None): - storage_type = storage.get('type', None) - auth = storage.get('auth', {}) - storage_key = "_".join([CACHE_KEY_TEMPLATE.get('user_storage_key'), 'default', - base64.b64encode(json.dumps(auth).encode("utf-8")).decode("utf-8")[0:64]]) - new_storage_obj = cache.get(storage_key) - if new_storage_obj: - logger.info("user %s get default storage obj cache %s" % (user, new_storage_obj)) - return new_storage_obj - else: - if storage_type == 1: - new_storage_obj = QiNiuOss(**auth) - new_storage_obj.storage_type = 1 - elif storage_type == 2: - new_storage_obj = AliYunOss(**auth) - new_storage_obj.storage_type = 2 - else: - new_storage_obj = LocalStorage(**auth) - new_storage_obj.storage_type = 3 - cache.set(storage_key, new_storage_obj, 600) - logger.warning("user %s has not storage obj, admin already has not storage obj, from settings " - "get default storage %s" % (user, new_storage_obj)) - return new_storage_obj - return None - def get_storage_type(self): if self.storage: return self.storage.storage_type - def get_storage_auth(self): - filter_fields = ['access_key', 'secret_key', 'bucket_name', 'domain_name', 'is_https', 'endpoint', - 'sts_role_arn', 'cnd_auth_key', 'download_auth_type'] - return get_dict_from_filter_fields(filter_fields, self.storage_obj.__dict__) - def get_local_storage(clean_cache=False): storage_lists = THIRD_PART_CONFIG.get('storage') @@ -178,3 +158,32 @@ def get_local_storage(clean_cache=False): logger.info("system get local storage obj, from settings " "storage %s" % new_storage_obj) return new_storage_obj + + +def get_storage_form_conf(user): + storage_lists = THIRD_PART_CONFIG.get('storage', []) + for storage in storage_lists: + if storage.get("active", None): + storage_type = storage.get('type', None) + auth = storage.get('auth', {}) + storage_key = "_".join([CACHE_KEY_TEMPLATE.get('user_storage_key'), 'default', + base64.b64encode(json.dumps(auth).encode("utf-8")).decode("utf-8")[0:64]]) + new_storage_obj = cache.get(storage_key) + if new_storage_obj: + logger.info("user %s get default storage %s obj cache %s " % (user, storage_key, new_storage_obj)) + return new_storage_obj + else: + if storage_type == 1: + new_storage_obj = QiNiuOss(**auth) + new_storage_obj.storage_type = 1 + elif storage_type == 2: + new_storage_obj = AliYunOss(**auth) + new_storage_obj.storage_type = 2 + else: + new_storage_obj = LocalStorage(**auth) + new_storage_obj.storage_type = 3 + cache.set(storage_key, new_storage_obj, 600) + logger.warning("user %s has not storage obj, from settings " + "get default storage key:%s obj:%s" % (user, storage_key, new_storage_obj)) + return new_storage_obj + return None diff --git a/fir_ser/api/views/login.py b/fir_ser/api/views/login.py index f92d344..e10fcc2 100644 --- a/fir_ser/api/views/login.py +++ b/fir_ser/api/views/login.py @@ -617,8 +617,7 @@ class CertificationView(APIView): certification_obj = CertificationInfo.objects.filter(user_id=request.user).all() if certification_obj: storage = Storage(request.user) - certification_serializer = CertificationSerializer(certification_obj, many=True, - context={"storage": storage}) + certification_serializer = CertificationSerializer(certification_obj, many=True) res.data["certification"] = certification_serializer.data return Response(res.dict) diff --git a/fir_ser/api/views/uploads.py b/fir_ser/api/views/uploads.py index 4b13d95..6f79e9a 100644 --- a/fir_ser/api/views/uploads.py +++ b/fir_ser/api/views/uploads.py @@ -6,7 +6,7 @@ from api.utils.app.apputils import get_random_short, SaveAppInfos from api.utils.baseutils import get_app_domain_name -from api.utils.storage.storage import Storage +from api.utils.storage.storage import Storage, get_local_storage from api.utils.storage.caches import upload_file_tmp_name, del_cache_response_by_short from api.models import Apps, AppReleaseInfo, UserInfo, AppScreenShot, CertificationInfo from api.utils.app.randomstrings import make_app_uuid @@ -177,6 +177,8 @@ class UploadView(APIView): res.code = 1006 res.msg = '类型不允许' else: + if ftype and ftype == 'certification': + storage = Storage(request.user, None, True) upload_key = make_from_user_uuid(request.user) + '.' + app_type + settings.FILE_UPLOAD_TMP_KEY upload_token = storage.get_upload_token(upload_key) storage_type = storage.get_storage_type() @@ -347,6 +349,7 @@ class UploadView(APIView): if ext: ptype = ext.get('ptype', None) if ptype is not None and ptype in [1, 2, 3]: + storage = Storage(request.user, None, True) certification_obj = CertificationInfo.objects.filter(user_id=request.user, type=ptype).first() if certification_obj: