From e7858a840e30cd05f2cd54fbd721b3309d450f7e Mon Sep 17 00:00:00 2001 From: nineven Date: Sat, 14 May 2022 19:41:13 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E5=99=A8=E9=9C=80=E8=A6=81?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E9=98=BF=E9=87=8C=E4=BA=91=EF=BC=8C=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E5=99=A8=E5=92=8Coss=E5=AD=98=E5=82=A8=E5=86=85?= =?UTF-8?q?=E7=BD=91=E4=BA=92=E9=80=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/user/FirUserStorage.vue | 4 +- fir_client/vue.config.js | 2 +- fir_ser/api/utils/modelutils.py | 14 ++++- fir_ser/api/utils/utils.py | 45 +++++++++++----- fir_ser/common/libs/storage/aliyunApi.py | 19 +------ fir_ser/common/utils/storage.py | 24 +++++---- fir_ser/xsign/utils/supersignutils.py | 52 +++++++++++-------- 7 files changed, 93 insertions(+), 67 deletions(-) diff --git a/fir_client/src/components/user/FirUserStorage.vue b/fir_client/src/components/user/FirUserStorage.vue index 74e910f..1a322e8 100644 --- a/fir_client/src/components/user/FirUserStorage.vue +++ b/fir_client/src/components/user/FirUserStorage.vue @@ -263,7 +263,7 @@ + label-width="80px" style="width: 50%;margin:0 auto;"> 存储最大容量空间 {{ diskSize(storageinfo.max_storage_capacity) }} @@ -1231,7 +1231,7 @@ export default { }, filters: {}, watch: { watchObj: function () { - if (this.editstorageinfo.bucket_name && this.editstorageinfo.download_auth_type === 1) { + if (this.editstorageinfo.bucket_name && this.editstorageinfo.download_auth_type === 1 && this.isaddflag) { this.editstorageinfo.domain_name = this.editstorageinfo.bucket_name + '.' + this.editstorageinfo.endpoint.replace("-internal", "") } } diff --git a/fir_client/vue.config.js b/fir_client/vue.config.js index 038654f..0973f1e 100644 --- a/fir_client/vue.config.js +++ b/fir_client/vue.config.js @@ -82,7 +82,7 @@ if (page) { } } -const version='2.3.6'; +const version='2.3.8'; const pro_base_env = { baseUrl: 'https://flyapps.cn', diff --git a/fir_ser/api/utils/modelutils.py b/fir_ser/api/utils/modelutils.py index 510223c..b2966eb 100644 --- a/fir_ser/api/utils/modelutils.py +++ b/fir_ser/api/utils/modelutils.py @@ -13,7 +13,7 @@ from django.db.models import Count from rest_framework.pagination import PageNumberPagination from api.models import AppReleaseInfo, UserDomainInfo, DomainCnameInfo, UserAdDisplayInfo, RemoteClientInfo, \ - AppBundleIdBlackList, NotifyReceiver, WeChatInfo, AppDownloadToken, Apps, StorageShareInfo, UserInfo + AppBundleIdBlackList, NotifyReceiver, WeChatInfo, AppDownloadToken, Apps, StorageShareInfo, UserInfo, AppStorage from common.base.baseutils import get_server_domain_from_request, get_user_default_domain_name, get_real_ip_address, \ get_origin_domain_name from common.base.magic import MagicCacheData @@ -295,3 +295,15 @@ def get_user_storage_capacity(user_obj): storage_capacity = Config.STORAGE_FREE_CAPACITY return storage_capacity + + +def get_user_storage_obj(user_obj, default=False): + storage = user_obj.storage + if storage and not default: + return storage + for storage in Config.STORAGE: + if storage.get("active", None): + storage_type = storage.get('type', None) + auth = storage.get('auth', {}) + storage = AppStorage(user_id=user_obj, storage_type=storage_type, **auth) + return storage diff --git a/fir_ser/api/utils/utils.py b/fir_ser/api/utils/utils.py index ec74e67..8b55ce6 100644 --- a/fir_ser/api/utils/utils.py +++ b/fir_ser/api/utils/utils.py @@ -11,7 +11,7 @@ import os from django.core.cache import cache from api.models import UserInfo, AppReleaseInfo, AppScreenShot, Token, UserAdDisplayInfo -from api.utils.modelutils import get_app_d_count_by_app_id +from api.utils.modelutils import get_app_d_count_by_app_id, get_user_storage_obj from api.utils.signalutils import run_xsign_migrate_data, run_xsign_clean_data from common.base.baseutils import get_real_ip_address from common.cache.storage import UserTokenCache @@ -108,40 +108,57 @@ def check_storage_is_new_storage(user_obj, new_storage_obj): def migrating_storage_file_data(user_obj, filename, new_storage_obj, clean_old_data=True): local_file_full_path = os.path.join(MEDIA_ROOT, filename) - old_storage_obj = Storage(user_obj) + if filename == 'head_img.jpeg': + return True + storage_obj = get_user_storage_obj(user_obj) + if storage_obj.storage_type == 2: + storage_obj.download_auth_type = 2 + old_oss_storage_obj = Storage(user_obj, storage_obj, prefix='migrate_storage') + else: + old_oss_storage_obj = Storage(user_obj, prefix='migrate_storage') + if not new_storage_obj: - new_storage_obj = Storage(user_obj, None, True) + storage_obj = get_user_storage_obj(user_obj, default=True) + if storage_obj.storage_type == 2: + storage_obj.download_auth_type = 2 + new_oss_storage_obj = Storage(user_obj, storage_obj, prefix='migrate_storage') + else: + new_oss_storage_obj = Storage(user_obj, None, True, prefix='migrate_storage') else: - new_storage_obj = Storage(user_obj, new_storage_obj) + if new_storage_obj.storage_type == 2: + new_storage_obj.download_auth_type = 2 + new_oss_storage_obj = Storage(user_obj, new_storage_obj, prefix='migrate_storage') + else: + new_oss_storage_obj = Storage(user_obj, new_storage_obj, prefix='migrate_storage') - if old_storage_obj.get_storage_uuid() == new_storage_obj.get_storage_uuid(): + if old_oss_storage_obj.get_storage_uuid() == new_oss_storage_obj.get_storage_uuid(): # 同一个存储，无需迁移数据 return True - if old_storage_obj.get_storage_type() == 3: - if new_storage_obj.get_storage_type() == 3: + if old_oss_storage_obj.get_storage_type() == 3: + if new_oss_storage_obj.get_storage_type() == 3: # 都是本地存储，无需操作 return True else: # 本地向云存储上传,并删除本地数据 - res = new_storage_obj.upload_file(local_file_full_path) + res = new_oss_storage_obj.upload_file(local_file_full_path) if clean_old_data: return delete_local_files(filename) return res else: - if new_storage_obj.get_storage_type() == 3: + if new_oss_storage_obj.get_storage_type() == 3: # 云存储下载 本地，并删除云存储 - if download_files_form_oss(old_storage_obj, local_file_full_path, True): + if download_files_form_oss(old_oss_storage_obj, local_file_full_path, True): if clean_old_data: - return old_storage_obj.delete_file(filename) + return old_oss_storage_obj.delete_file(filename) return True else: # 云存储互传，先下载本地，然后上传新云存储，删除本地和老云存储 - if download_files_form_oss(old_storage_obj, local_file_full_path, True): - res1 = new_storage_obj.upload_file(local_file_full_path) + if download_files_form_oss(old_oss_storage_obj, local_file_full_path, True): + res1 = new_oss_storage_obj.upload_file(local_file_full_path) res2 = delete_local_files(filename) if clean_old_data: - return old_storage_obj.delete_file(filename) + return old_oss_storage_obj.delete_file(filename) return res1 and res2 diff --git a/fir_ser/common/libs/storage/aliyunApi.py b/fir_ser/common/libs/storage/aliyunApi.py index e16407c..da6e0a9 100644 --- a/fir_ser/common/libs/storage/aliyunApi.py +++ b/fir_ser/common/libs/storage/aliyunApi.py @@ -200,23 +200,6 @@ class AliYunOss(object): def upload_file(self, local_file_full_path): return self.multipart_upload_file(local_file_full_path) - if os.path.isfile(local_file_full_path): - filename = os.path.basename(local_file_full_path) - headers = { - 'Content-Disposition': f'attachment; filename="{filename}"', - 'Cache-Control': '' - } - self.bucket.put_object_from_file(filename, local_file_full_path, headers) - # with open(local_file_full_path, 'rb') as fileobj: - # # Seek方法用于指定从第1000个字节位置开始读写。上传时会从您指定的第1000个字节位置开始上传，直到文件结束。 - # fileobj.seek(1000, os.SEEK_SET) - # # Tell方法用于返回当前位置。 - # # current = fileobj.tell() - # self.bucket.put_object(os.path.basename(local_file_full_path), fileobj) - return True - else: - logger.error(f"file {local_file_full_path} is not file") - def download_file(self, name, local_file_full_path): dir_path = os.path.dirname(local_file_full_path) if not os.path.exists(dir_path): @@ -236,7 +219,7 @@ class AliYunOss(object): if os.path.isfile(local_file_full_path): total_size = os.path.getsize(local_file_full_path) # determine_part_size方法用于确定分片大小。 - part_size = determine_part_size(total_size, preferred_size=1024 * 1024 * 10) + part_size = determine_part_size(total_size, preferred_size=1024 * 1024 * 100) filename = os.path.basename(local_file_full_path) headers = { 'Content-Disposition': 'attachment; filename="%s"' % get_filename_form_file(filename).encode( diff --git a/fir_ser/common/utils/storage.py b/fir_ser/common/utils/storage.py index eeeefae..fe92771 100644 --- a/fir_ser/common/utils/storage.py +++ b/fir_ser/common/utils/storage.py @@ -25,9 +25,9 @@ def get_storage_auth(storage_obj): @run_function_by_locker() -def get_storage(user, assigned_storage_obj, use_default_storage): +def get_oss_storage(user, assigned_storage_obj, use_default_storage, prefix): if use_default_storage: - return get_storage_form_conf(user) + return get_storage_form_conf(user, prefix) if assigned_storage_obj: storage_obj = assigned_storage_obj else: @@ -36,7 +36,7 @@ def get_storage(user, assigned_storage_obj, use_default_storage): if storage_obj: auth = get_storage_auth(storage_obj) storage_type = storage_obj.storage_type - storage_cache = CloudStorageCache(auth, user.uid) + storage_cache = CloudStorageCache(auth, f'{user.uid}_{prefix}') storage_key, new_storage_obj = storage_cache.get_storage_key_and_cache() if new_storage_obj and not assigned_storage_obj: @@ -56,16 +56,20 @@ def get_storage(user, assigned_storage_obj, use_default_storage): else: logger.info(f"user {user} has not storage obj, so get default") # 不需要管理存储，直接从配置文件获取默认存储 - return get_storage_form_conf(user) + return get_storage_form_conf(user, prefix) class Storage(object): - def __init__(self, user, assigned_storage_obj=None, use_default_storage=False): + def __init__(self, user, assigned_storage_obj=None, use_default_storage=False, prefix='default'): + self.user = user + self.prefix = prefix + self.assigned_storage_obj = assigned_storage_obj + self.use_default_storage = use_default_storage try: locker = { - 'locker_key': f"make_storage_cache_{user.uid}", - "timeout": 60 * 5, "blocking_timeout": 6} - self.storage = get_storage(user, assigned_storage_obj, use_default_storage, locker=locker) + 'locker_key': f"make_storage_cache_{user.uid}_{prefix}", + "timeout": 60, "blocking_timeout": 6} + self.storage = get_oss_storage(user, assigned_storage_obj, use_default_storage, prefix, locker=locker) except Exception as e: logger.error(f"get {user} storage failed Exception:{e}") self.storage = None @@ -167,12 +171,12 @@ def get_local_storage(clean_cache=False): return new_storage_obj -def get_storage_form_conf(user): +def get_storage_form_conf(user, prefix): for storage in Config.STORAGE: if storage.get("active", None): storage_type = storage.get('type', None) auth = storage.get('auth', {}) - storage_cache = CloudStorageCache(auth, 'default') + storage_cache = CloudStorageCache(auth, f'system_{prefix}') storage_key, new_storage_obj = storage_cache.get_storage_key_and_cache() if new_storage_obj: logger.info(f"user {user} get default storage {storage_key} obj cache {new_storage_obj} ") diff --git a/fir_ser/xsign/utils/supersignutils.py b/fir_ser/xsign/utils/supersignutils.py index bec1b8e..9299dec 100644 --- a/fir_ser/xsign/utils/supersignutils.py +++ b/fir_ser/xsign/utils/supersignutils.py @@ -17,6 +17,7 @@ from django.core.cache import cache from django.db.models import Count, F from api.models import UserInfo, AppReleaseInfo, Apps +from api.utils.modelutils import get_user_storage_obj from api.utils.response import BaseResponse from api.utils.utils import delete_local_files, download_files_form_oss from common.base.baseutils import file_format_path, delete_app_profile_file, get_profile_full_path, format_apple_date, \ @@ -44,6 +45,16 @@ from xsign.utils.utils import delete_app_to_dev_and_file logger = logging.getLogger(__name__) +def get_sign_oss_storage(user_obj): + storage_obj = get_user_storage_obj(user_obj) + if storage_obj.storage_type == 2: + storage_obj.download_auth_type = 2 + oss_storage_obj = Storage(user_obj, storage_obj, prefix='super_sign') + else: + oss_storage_obj = Storage(user_obj, prefix='super_sign') + return oss_storage_obj + + def check_org_file(user_obj, org_file): if not os.path.isdir(os.path.dirname(org_file)): os.makedirs(os.path.dirname(org_file)) @@ -51,8 +62,7 @@ def check_org_file(user_obj, org_file): if os.path.isfile(org_file): return True - storage_obj = Storage(user_obj) - return download_files_form_oss(storage_obj, org_file) + return download_files_form_oss(get_sign_oss_storage(user_obj), org_file) def get_abnormal_queryset(user_obj, udid): @@ -581,26 +591,26 @@ class IosUtils(object): @staticmethod def update_sign_file_name(user_obj, app_obj, developer_obj_id, release_obj, random_file_name): apptodev_obj = APPToDeveloper.objects.filter(developerid_id=developer_obj_id, app_id=app_obj).first() - storage_obj = Storage(user_obj) - + storage_obj = get_sign_oss_storage(user_obj) logger.info(f"update sign file end, now upload {storage_obj.storage} {random_file_name}.ipa file") - if storage_obj.upload_file(os.path.join(MEDIA_ROOT, random_file_name + ".ipa")): - if apptodev_obj: - delete_local_files(apptodev_obj.binary_file + ".ipa") - storage_obj.delete_file(apptodev_obj.binary_file + ".ipa") - apptodev_obj.binary_file = random_file_name - old_release_file = apptodev_obj.release_file - apptodev_obj.release_file = release_obj.release_id - apptodev_obj.save(update_fields=['binary_file', 'release_file', 'updated_time']) - if storage_obj.get_storage_type() in [1, 2] and old_release_file != release_obj.release_id: - logger.warning(f"update sign file , now clean ole {old_release_file}.ipa file") - delete_local_files(old_release_file + ".ipa") - else: - APPToDeveloper.objects.create(developerid_id=developer_obj_id, app_id=app_obj, - binary_file=random_file_name, release_file=release_obj.release_id) - if storage_obj.get_storage_type() in [1, 2]: - delete_local_files(random_file_name + ".ipa") - return True + with cache.lock(f"upload_files_form_oss_{random_file_name}", timeout=60 * 30): + if storage_obj.upload_file(os.path.join(MEDIA_ROOT, random_file_name + ".ipa")): + if apptodev_obj: + delete_local_files(apptodev_obj.binary_file + ".ipa") + storage_obj.delete_file(apptodev_obj.binary_file + ".ipa") + apptodev_obj.binary_file = random_file_name + old_release_file = apptodev_obj.release_file + apptodev_obj.release_file = release_obj.release_id + apptodev_obj.save(update_fields=['binary_file', 'release_file', 'updated_time']) + if storage_obj.get_storage_type() in [1, 2] and old_release_file != release_obj.release_id: + logger.warning(f"update sign file , now clean ole {old_release_file}.ipa file") + delete_local_files(old_release_file + ".ipa") + else: + APPToDeveloper.objects.create(developerid_id=developer_obj_id, app_id=app_obj, + binary_file=random_file_name, release_file=release_obj.release_id) + if storage_obj.get_storage_type() in [1, 2]: + delete_local_files(random_file_name + ".ipa") + return True @staticmethod def update_sign_data(user_obj, app_obj, developer_obj_id, random_file_name, release_obj, udid_list):