修复存储切换下载token过期问题

增加私有云存储下载点计算
修复配置文件获取异常
修改云存储切换为任务执行

fir_client/src/components/user/FirUserStorage.vue

@@ -193,8 +193,8 @@
     </el-dialog>
 
     <el-tabs v-model="activeName" tab-position="top" type="border-card" @tab-click="handleClick">
-      <el-tab-pane label="存储选择" name="change">'
-        <el-row :gutter="24">
+      <el-tab-pane label="存储选择" name="change">
+        <el-row :gutter="24" style="margin-top: 15px">
           <el-col :span="14">
             存储选择：
             <el-select v-model="use_storage_id" :placeholder="selectlabel" filterable style="width: 400px"
@@ -235,8 +235,6 @@
                 type="line"/>
           </el-col>
         </el-row>
-
-
         <div v-if="use_storage_id!==org_storage_id" style="margin-top: 20px">
           <el-button style="margin-left: 10px"
                      type="info"
@@ -262,7 +260,7 @@
           <!--          </el-button>-->
         </div>
         <el-divider/>
-        <el-form ref="storageinfoform" :model="storageinfo"
+        <el-form v-if="storageinfo" ref="storageinfoform" :model="storageinfo"
                  label-width="80px" style="width: 50%;margin:0 auto;">
 
           <el-form-item label="存储最大容量" label-width="110px" style="text-align: left">
@@ -666,26 +664,48 @@
         </el-table>
       </el-tab-pane>
       <el-tab-pane label="存储设置" name="setting" style="text-align: center">
-        <el-tag style="margin: 20px 0"> 应用版本数设置，当应用历史版本超过该限制，将会自动清理较老的版本</el-tag>
-        <div style="margin: auto;width: 700px;height: 100%">
-          <el-form ref="form" :model="storage_config" label-width="180px">
-            <el-form-item label="应用历史版本数" style="text-align: left">
-              <el-input-number v-model="storage_config.user_history_limit" :min="1"
-                               style="width: 300px;margin: 0 10px"></el-input-number>
-              <el-button @click="updateConfig('update')">保存修改</el-button>
-            </el-form-item>
-            <el-form-item label="当前存储空间大小" style="text-align: left">
-              <el-tag>{{ diskSize(storage_config.user_max_storage_capacity) }}</el-tag>
-              已经使用
-              <el-tag>{{ diskSize(storage_config.user_used_storage_capacity) }}</el-tag>
-              还剩
-              <el-tag>
-                {{ diskSize(storage_config.user_max_storage_capacity - storage_config.user_used_storage_capacity) }}
-              </el-tag>
-            </el-form-item>
+        <div v-if="migrate_progress && migrate_progress.s_time" style="text-align: left">
+          <el-card>
+            <div slot="header">
+              <el-tag type="danger">存储迁移中，请耐心等待</el-tag>
+            </div>
+            <el-row :gutter="24">
+              <el-col :span="4">存储迁移进度</el-col>
+              <el-col :span="20">
+                <el-progress
+                    :color="storage_usedColor"
+                    :percentage="parseInt(migrate_progress.migrate_num*100/migrate_progress.release_num)"
+                    :stroke-width="18"
+                    :text-inside="true" status="success" style="width: 80%"
+                    type="line"/>
+              </el-col>
+            </el-row>
+            <el-row :gutter="24" style="margin-top: 30px">
+              <el-col :span="4">总迁移应用数</el-col>
+              <el-col :span="20">
+                <el-tag>{{ migrate_progress.release_num }}</el-tag>
+              </el-col>
+            </el-row>
+            <el-row :gutter="24" style="margin-top: 20px">
+              <el-col :span="4">已经迁移成功应用数</el-col>
+              <el-col :span="20">
+                <el-tag>{{ migrate_progress.migrate_num }}</el-tag>
+              </el-col>
+            </el-row>
+            <el-row :gutter="24" style="margin-top: 20px">
+              <el-col :span="4">存储迁移开始时间</el-col>
+              <el-col :span="20">
+                <el-tag>{{ getFormatDate(migrate_progress.s_time) }}</el-tag>
+              </el-col>
+            </el-row>
+          </el-card>
 
+          <el-divider/>
+        </div>
 
-            <el-form-item label="存储迁移状态" style="text-align: left">
+        <div style="margin: auto;width: 700px;height: 100%">
+          <el-form ref="form" :model="storage_config" label-width="180px">
+            <el-form-item v-if="storage_config.storage_status" label="存储迁移状态" style="text-align: left">
               <el-tag v-if="!storage_config.storage_status">未迁移，状态正常</el-tag>
               <div v-else>
                 <el-tag>迁移中，迁移时间 {{ getFormatDate(storage_config.storage_status) }}</el-tag>
@@ -695,20 +715,39 @@
                 </el-tag>
               </div>
             </el-form-item>
-            <el-form-item label="清理所有应用数据" style="text-align: left">
-              <el-button style="margin-left: 10px"
-                         type="danger"
-                         @click="clean_storage_data('all')">
-                清理所有应用数据
-              </el-button>
-            </el-form-item>
-            <el-form-item label="清理应用历史数据" style="text-align: left">
-              <el-button style="margin-left: 10px"
-                         type="danger"
-                         @click="clean_storage_data('history')">
-                清理应用历史版本数据，只保留最新版本数据
-              </el-button>
-            </el-form-item>
+
+            <div v-if="!(migrate_progress && migrate_progress.s_time)">
+              <el-tag style="margin: 20px 0"> 应用版本数设置，当应用历史版本超过该限制，将会自动清理较老的版本</el-tag>
+              <el-form-item label="应用历史版本数" style="text-align: left">
+                <el-input-number v-model="storage_config.user_history_limit" :min="1"
+                                 style="width: 300px;margin: 0 10px"></el-input-number>
+                <el-button @click="updateConfig('update')">保存修改</el-button>
+              </el-form-item>
+              <el-form-item label="当前存储空间大小" style="text-align: left">
+                <el-tag>{{ diskSize(storage_config.user_max_storage_capacity) }}</el-tag>
+                已经使用
+                <el-tag>{{ diskSize(storage_config.user_used_storage_capacity) }}</el-tag>
+                还剩
+                <el-tag>
+                  {{ diskSize(storage_config.user_max_storage_capacity - storage_config.user_used_storage_capacity) }}
+                </el-tag>
+              </el-form-item>
+              <el-form-item label="清理所有应用数据" style="text-align: left">
+                <el-button style="margin-left: 10px"
+                           type="danger"
+                           @click="clean_storage_data('all')">
+                  清理所有应用数据
+                </el-button>
+              </el-form-item>
+              <el-form-item label="清理应用历史数据" style="text-align: left">
+                <el-button style="margin-left: 10px"
+                           type="danger"
+                           @click="clean_storage_data('history')">
+                  清理应用历史版本数据，只保留最新版本数据
+                </el-button>
+              </el-form-item>
+            </div>
+
           </el-form>
         </div>
 
@@ -761,6 +800,7 @@ export default {
       operatestatus: '',
       dialogstorageVisible: false,
       editstorageinfo: {},
+      migrate_progress: {},
       selectlabel: "",
       storageinfo: {'used_number': 0, 'max_storage_capacity': 1},
       storage_list: [],
@@ -993,6 +1033,13 @@ export default {
           this.fstorage_lists = data.storage_group_list;
           this.org_storage_id = this.use_storage_id = data.storage;
           this.storage_list = data.storage_list;
+          this.migrate_progress = data.migrate_progress;
+          if (this.migrate_progress && this.migrate_progress.s_time) {
+            this.activeName = 'setting'
+            this.$message.warning("存储迁移中，请耐心等待")
+            this.refreshactiveFun()
+            return
+          }
           this.getstorageinfobyid(this.use_storage_id)
         } else {
           this.$message.error('存储类别获取失败,' + data.msg);
@@ -1207,6 +1254,7 @@ export default {
         configStorageData(res => {
           if (res.code === 1000) {
             this.storage_config = res.data
+            this.migrate_progress = res.migrate_progress
           } else {
             this.$message.error("获取数据失败了，" + res.msg)
           }

fir_ser/admin/views/storage.py

@@ -14,8 +14,8 @@ from rest_framework.views import APIView
 
 from admin.utils.serializer import AdminStorageSerializer
 from admin.utils.utils import AppsPageNumber, BaseModelSet
-from api.base_views import storage_change
 from api.models import UserInfo, AppStorage
+from api.tasks import migrate_storage_job
 from common.base.baseutils import format_storage_selection
 from common.core.auth import AdminTokenAuthentication
 from common.core.response import ApiResponse
@@ -64,8 +64,13 @@ class StorageChangeView(APIView):
             use_storage_id = data.get("use_storage_id", None)
             force = data.get("force", None)
             if use_storage_id:
-                if not storage_change(use_storage_id, obj, force):
-                    ApiResponse(code=1006, msg="修改失败")
+                c_task = migrate_storage_job.apply_async((use_storage_id, obj.pk, force))
+                msg = c_task.get(propagate=False)
+                logger.info(f"run migrate storage task {obj} msg:{msg}")
+                if c_task.successful():
+                    c_task.forget()
+                else:
+                    ApiResponse(code=1006, msg=msg)
                 return ApiResponse()
 
         return ApiResponse(code=1004, msg="数据校验失败")

fir_ser/api/base_views.py

@@ -65,44 +65,6 @@ def app_screen_delete(screen_id, apps_obj, storage):
         del_cache_response_by_short(apps_obj.app_id)
 
 
-def app_release_delete(app_obj, release_id, storage):
-    res = BaseResponse()
-    user_obj = app_obj.user_id
-    if app_obj:
-        apprelease_count = AppReleaseInfo.objects.filter(app_id=app_obj).values("release_id").count()
-        appreleaseobj = AppReleaseInfo.objects.filter(app_id=app_obj, release_id=release_id).first()
-        if not appreleaseobj.is_master:
-            logger.info(f"delete app release {appreleaseobj}")
-            storage.delete_file(appreleaseobj.release_id, appreleaseobj.release_type)
-            delete_local_files(appreleaseobj.release_id, appreleaseobj.release_type)
-            storage.delete_file(appreleaseobj.icon_url)
-
-            appreleaseobj.delete()
-        elif appreleaseobj.is_master and apprelease_count < 2:
-            logger.info(f"delete app master release {appreleaseobj} and clean app {app_obj} ")
-            count = APPToDeveloper.objects.filter(app_id=app_obj).count()
-            if app_obj.issupersign or count > 0:
-                logger.info(f"app_id:{app_obj.app_id} is supersign ,delete this app need clean IOS developer")
-                IosUtils.clean_app_by_user_obj(app_obj)
-
-            storage.delete_file(appreleaseobj.release_id, appreleaseobj.release_type)
-            delete_local_files(appreleaseobj.release_id, appreleaseobj.release_type)
-            storage.delete_file(appreleaseobj.icon_url)
-            del_cache_by_delete_app(app_obj.app_id)
-
-            appreleaseobj.delete()
-            delete_app_screenshots_files(storage, app_obj)
-            has_combo = app_obj.has_combo
-            if has_combo:
-                app_obj.has_combo.has_combo = None
-            app_obj.delete()
-        else:
-            pass
-        del_cache_response_by_short(app_obj.app_id)
-
-    return res
-
-
 def check_storage_ok(user_obj, new_storage_obj):
     if migrating_storage_file_data(user_obj, 'head_img.jpeg', new_storage_obj, False):
         return True

fir_ser/api/migrations/0006_auto_20220513_0843.py

@@ -21,6 +21,11 @@ class Migration(migrations.Migration):
             name='storage_capacity',
             field=models.BigIntegerField(default=0, verbose_name='存储容量，单位byte'),
         ),
+        migrations.AlterField(
+            model_name='userinfo',
+            name='download_times',
+            field=models.PositiveBigIntegerField(default=0, verbose_name='可用下载点次数,需要用户充值'),
+        ),
         migrations.AlterField(
             model_name='userinfo',
             name='storage',

fir_ser/api/models.py

@@ -39,7 +39,7 @@ class UserInfo(AbstractUser):
 
     memo = models.TextField('备注', blank=True, null=True, default=None, )
     date_joined = models.DateTimeField(auto_now_add=True, verbose_name="注册时间")
-    download_times = models.PositiveIntegerField(default=0, verbose_name="可用下载次数,需要用户充值")
+    download_times = models.PositiveBigIntegerField(default=0, verbose_name="可用下载点次数,需要用户充值")
     storage_capacity = models.BigIntegerField(default=0, verbose_name="存储容量，单位byte")
     all_download_times = models.BigIntegerField(default=0, verbose_name="总共下载次数")
     default_domain_name = models.ForeignKey(to="DomainCnameInfo", verbose_name="默认下载页域名", on_delete=models.CASCADE)

fir_ser/api/tasks.py

@@ -9,7 +9,8 @@ import logging
 from captcha.models import CaptchaStore
 
 from api.utils.ctasks import sync_download_times, auto_clean_upload_tmp_file, auto_clean_remote_client_log, \
-    notify_check_user_download_times, notify_check_apple_developer_devices, notify_check_apple_developer_cert
+    notify_check_user_download_times, notify_check_apple_developer_devices, notify_check_apple_developer_cert, \
+    migrate_user_oss_storage
 from api.views.login import get_login_type
 from common.core.sysconfig import Config, ConfigCacheBase
 from common.libs.geetest.geetest_utils import check_bypass_status
@@ -31,57 +32,62 @@ def start_api_sever_do_clean():
 
 
 def clean_config_cache(key):
-    ConfigCacheBase().invalid_config_cache(key)
+    return ConfigCacheBase().invalid_config_cache(key)
 
 
 @app.task
 def sync_download_times_job():
-    sync_download_times()
+    return sync_download_times()
 
 
 @app.task
 def check_bypass_status_job():
     if Config.LOGIN.get("geetest") or Config.CHANGER.get('geetest') or Config.REGISTER.get(
             'geetest') or Config.REPORT.get('geetest'):
-        check_bypass_status()
+        return check_bypass_status()
 
 
 @app.task
 def auto_clean_upload_tmp_file_job():
-    auto_clean_upload_tmp_file()
+    return auto_clean_upload_tmp_file()
 
 
 @app.task
 def auto_clean_captcha_store_job():
-    CaptchaStore.remove_expired()
+    return CaptchaStore.remove_expired()
 
 
 @app.task
 def auto_delete_tmp_file_job():
-    auto_delete_ios_mobile_tmp_file()
+    return auto_delete_ios_mobile_tmp_file()
 
 
 @app.task
 def sync_wx_access_token_job():
     if get_login_type().get('third', '').get('wxp'):
-        sync_wx_access_token()
+        return sync_wx_access_token()
 
 
 @app.task
 def auto_clean_remote_client_job():
-    auto_clean_remote_client_log()
+    return auto_clean_remote_client_log()
 
 
 @app.task
 def download_times_notify_check_job():
-    notify_check_user_download_times()
+    return notify_check_user_download_times()
 
 
 @app.task
 def apple_developer_devices_check_job():
-    notify_check_apple_developer_devices()
+    return notify_check_apple_developer_devices()
 
 
 @app.task
 def apple_developer_cert_notify_check_job():
-    notify_check_apple_developer_cert()
+    return notify_check_apple_developer_cert()
+
+
+@app.task
+def migrate_storage_job(use_storage_id, user_pk, force):
+    return migrate_user_oss_storage(use_storage_id, user_pk, force)

fir_ser/api/utils/ctasks.py

@@ -9,7 +9,9 @@ import time
 
 from django.core.cache import cache
 
+from api.base_views import storage_change
 from api.models import Apps, UserInfo, RemoteClientInfo
+from common.cache.state import MigrateStorageState
 from common.notify.ntasks import check_user_download_times, check_apple_developer_devices, check_apple_developer_cert
 from common.utils.storage import Storage
 from fir_ser.settings import CACHE_KEY_TEMPLATE
@@ -66,3 +68,16 @@ def notify_check_apple_developer_devices():
 def notify_check_apple_developer_cert():
     for user_obj in UserInfo.objects.filter(is_active=True, supersign_active=True).all():
         check_apple_developer_cert(user_obj, expire_day=7, days=[0, 1, 3, 7])
+
+
+def migrate_user_oss_storage(use_storage_id, user_pk, force):
+    msg = 'success'
+    user_obj = UserInfo.objects.filter(pk=user_pk).first()
+    if user_obj:
+        with MigrateStorageState(user_obj.uid) as state:
+            if state:
+                if not storage_change(use_storage_id, user_obj, force):
+                    msg = '数据迁移失败'
+            else:
+                msg = "数据迁移中,请耐心等待"
+    return msg

fir_ser/api/utils/modelutils.py

@@ -22,12 +22,12 @@ from common.core.sysconfig import Config
 logger = logging.getLogger(__name__)
 
 
-def get_app_d_count_by_app_id(app_id):
+def get_app_d_count_by_app_id(app_id, is_oss=False):
     d_count = 1
     binary_size = AppReleaseInfo.objects.filter(is_master=True, app_id__app_id=app_id).values('binary_size').first()
     if binary_size and binary_size.get('binary_size', 0) > 0:
         d_count += binary_size.get('binary_size') // 1024 // 1024 // 100
-    return d_count
+    return d_count * (Config.APP_USE_BASE_DOWNLOAD_TIMES if not is_oss else Config.PRIVATE_OSS_DOWNLOAD_TIMES)
 
 
 def get_user_domain_name(obj, domain_type=1):

fir_ser/api/utils/serializer.py

@@ -70,6 +70,11 @@ class UserInfoSerializer(serializers.ModelSerializer):
     def get_storage_used_capacity(self, obj):
         return get_user_storage_used(obj)
 
+    download_times = serializers.SerializerMethodField()
+
+    def get_download_times(self, obj):
+        return obj.download_times // Config.APP_USE_BASE_DOWNLOAD_TIMES
+
     storage_used = serializers.SerializerMethodField()
 
     def get_storage_used(self, obj):
@@ -83,7 +88,8 @@ class UserInfoSerializer(serializers.ModelSerializer):
     free_download_times = serializers.SerializerMethodField()
 
     def get_free_download_times(self, obj):
-        return get_user_free_download_times(obj.id, auth_status=get_user_cert_auth_status(obj.id))
+        return get_user_free_download_times(obj.id, auth_status=get_user_cert_auth_status(
+            obj.id)) // Config.APP_USE_BASE_DOWNLOAD_TIMES
 
     certification = serializers.SerializerMethodField()
 

fir_ser/api/utils/utils.py

@@ -7,6 +7,7 @@ import binascii
 import datetime
 import logging
 import os
+import time
 
 from django.core.cache import cache
 
@@ -14,7 +15,7 @@ from api.models import UserInfo, AppReleaseInfo, AppScreenShot, Token, UserAdDis
 from api.utils.modelutils import get_app_d_count_by_app_id, get_user_storage_obj
 from api.utils.signalutils import run_xsign_migrate_data, run_xsign_clean_data
 from common.base.baseutils import get_real_ip_address
-from common.cache.storage import UserTokenCache
+from common.cache.storage import UserTokenCache, TaskProgressCache
 from common.libs.storage.localApi import LocalStorage
 from common.utils.caches import consume_user_download_times
 from common.utils.storage import Storage
@@ -164,12 +165,24 @@ def migrating_storage_file_data(user_obj, filename, new_storage_obj, clean_old_d
 
 def migrating_storage_data(user_obj, new_storage_obj, clean_old_data):
     with cache.lock(f"migrating_storage_data_{user_obj.uid}", timeout=60 * 60 * 24):
+        task_cache = TaskProgressCache('migrating_storage_data', user_obj.uid)
+        app_release_queryset = AppReleaseInfo.objects.filter(app_id__user_id=user_obj)
+        cache_data = task_cache.get_storage_cache()
+        if not cache_data:
+            cache_default_data = {
+                's_time': time.time(),
+                'release_num': app_release_queryset.count(),
+                'migrate_num': 0,
+                'target_oss_id': -1 if not new_storage_obj else new_storage_obj.pk
+            }
+        else:
+            logger.warning(f"migrate {user_obj} oss progress is exist. may be is migrating data {cache_data}")
 
         auth_status = False
         certification = getattr(user_obj, 'certification', None)
         if certification and certification.status == 1:
             auth_status = True
-        for app_release_obj in AppReleaseInfo.objects.filter(app_id__user_id=user_obj).all():
+        for app_release_obj in app_release_queryset.all():
             # 迁移APP数据
             filename = get_filename_from_apptype(app_release_obj.release_id, app_release_obj.release_type)
             migrating_storage_file_data(user_obj, filename, new_storage_obj, clean_old_data)
@@ -183,8 +196,11 @@ def migrating_storage_data(user_obj, new_storage_obj, clean_old_data):
             #     filename = get_filename_from_apptype(apptodev_obj.binary_file, app_release_obj.release_type)
             #     migrating_storage_file_data(user_obj, filename, new_storage_obj, clean_old_data)
             # 消费下载次数
-            amount = get_app_d_count_by_app_id(app_release_obj.app_id.app_id)
+            cache_default_data['migrate_num'] += 1
+            task_cache.set_storage_cache(cache_default_data)
+            amount = get_app_d_count_by_app_id(app_release_obj.app_id.app_id, False)
             consume_user_download_times(user_obj.pk, app_release_obj.app_id, amount, auth_status)
+        task_cache.del_storage_cache()
         return True
 
 

fir_ser/api/views/storage.py

@@ -10,8 +10,9 @@ from django.db.models import Q
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-from api.base_views import storage_change, app_delete
+from api.base_views import app_delete
 from api.models import AppStorage, Apps, StorageShareInfo, UserInfo
+from api.tasks import migrate_storage_job
 from api.utils.apputils import clean_history_apps
 from api.utils.modelutils import PageNumber, get_user_storage_capacity, get_user_storage_used
 from api.utils.response import BaseResponse
@@ -19,6 +20,7 @@ from api.utils.serializer import StorageSerializer, StorageShareSerializer
 from api.utils.utils import upload_oss_default_head_img
 from common.base.baseutils import get_choices_dict, get_choices_name_from_key, get_real_ip_address
 from common.cache.state import MigrateStorageState
+from common.cache.storage import TaskProgressCache
 from common.core.auth import ExpiringTokenAuthentication, StoragePermission
 from common.core.sysconfig import Config
 from common.utils.caches import del_cache_storage
@@ -86,6 +88,8 @@ class StorageView(APIView):
 
         if act == 'storage_group':
             res.storage_group_list = get_storage_group(request, res, self.storage_type, is_default == 'true')
+            migrate_progress = TaskProgressCache('migrating_storage_data', request.user.uid).get_storage_cache()
+            res.migrate_progress = migrate_progress if migrate_progress else {}
             return Response(res.dict)
         if pk:
             if pk == '-1':
@@ -188,23 +192,24 @@ class StorageView(APIView):
                     res.msg = "数据异常，请重试"
                     return Response(res.dict)
 
-            with MigrateStorageState(request.user.uid) as state:
-                if state:
-                    if not storage_change(use_storage_id, request.user, force):
-                        res.code = 1006
-                        res.msg = '修改失败'
-                else:
-                    res.code = 1007
-                    res.msg = "数据迁移中,请耐心等待"
+            c_task = migrate_storage_job.apply_async((use_storage_id, request.user.pk, force))
+            msg = c_task.get(propagate=False)
+            logger.info(f"run migrate storage task {request.user} msg:{msg}")
+            if c_task.successful():
+                c_task.forget()
+            else:
+                res.code = 1006
+                res.msg = msg
 
             return Response(res.dict)
 
         storage_id = data.get("id", None)
         if storage_id:
-            # if request.user.storage and storage_id == request.user.storage.id:
-            #     res.msg = '存储正在使用中，无法修改'
-            #     res.code = 1007
-            #     return Response(res.dict)
+            if request.user.storage and storage_id == request.user.storage.id and MigrateStorageState(
+                    request.user.uid).get_state():
+                res.msg = '存储迁移中，无法修改'
+                res.code = 1007
+                return Response(res.dict)
             storage_obj = AppStorage.objects.filter(id=storage_id, user_id=request.user).first()
             storage_obj_bak = AppStorage.objects.filter(id=storage_id, user_id=request.user).first()
             serializer = StorageSerializer(instance=storage_obj, data=data, context={'user_obj': request.user},
@@ -241,6 +246,10 @@ class StorageView(APIView):
         storage_id = request.query_params.get("id", None)
         if storage_id:
             try:
+                if request.user.storage and storage_id == request.user.storage.id:
+                    res.msg = '存储使用中，无法删除'
+                    res.code = 1007
+                    return Response(res.dict)
                 AppStorage.objects.filter(user_id=request.user, id=storage_id).delete()
                 logger.error(f"user {request.user} delete storage id:{storage_id} success")
             except Exception as e:
@@ -418,12 +427,14 @@ class StorageConfigView(APIView):
 
     def get(self, request):
         res = BaseResponse()
+        migrate_progress = TaskProgressCache('migrating_storage_data', request.user.uid).get_storage_cache()
         res.data = {
             'user_max_storage_capacity': get_user_storage_capacity(request.user),
             'user_used_storage_capacity': get_user_storage_used(request.user),
             'user_history_limit': UserInfo.objects.filter(pk=request.user.pk).first().history_release_limit,
-            'storage_status': MigrateStorageState(request.user.uid).get_state()
+            'storage_status': MigrateStorageState(request.user.uid).get_state(),
         }
+        res.migrate_progress = migrate_progress if migrate_progress else {}
         return Response(res.dict)
 
     def put(self, request):

fir_ser/common/cache/invalid.py

@@ -48,16 +48,20 @@ def invalid_short_cache(app_obj, key='ShortDownloadView'.lower()):
     """
     if not app_obj:
         return
-    master_release_dict = AppReleaseInfo.objects.filter(app_id=app_obj, is_master=True).values('icon_url',
-                                                                                               'release_id').first()
+    master_release_dict = AppReleaseInfo.objects.filter(app_id=app_obj, is_master=True).values('icon_url', 'release_id',
+                                                                                               'release_type').first()
     if master_release_dict:
         # 1.清理图片缓存
         DownloadUrlCache(key, master_release_dict.get('icon_url')).del_storage_cache()
 
         release_id = master_release_dict.get('release_id')
+        release_type = master_release_dict.get('release_type')
         # 2.清理下载token缓存
         TokenManagerCache(key, release_id).del_storage_cache()
+        TokenManagerCache(key, f'{release_id}.{"apk" if release_type == 0 else "ipa"}').del_storage_cache()
+        DownloadUrlCache(key, f'{release_id}.{"apk" if release_type == 0 else "ipa"}').del_storage_cache()
         TokenManagerCache('', f"{release_id}.plist").del_storage_cache()
+        DownloadUrlCache('', f"{release_id}.plist").del_storage_cache()
 
     # 3.清理广告缓存
     invalid_ad_pic_cache(key, app_obj.short)

fir_ser/common/cache/storage.py

@@ -217,6 +217,12 @@ class TaskStateCache(RedisCacheBase):
         super().__init__(self.cache_key)
 
 
+class TaskProgressCache(RedisCacheBase):
+    def __init__(self, prefix, task_id):
+        self.cache_key = f"{CACHE_KEY_TEMPLATE.get('task_progress_key')}_{prefix}_{task_id}"
+        super().__init__(self.cache_key)
+
+
 class PendingStateCache(RedisCacheBase):
     def __init__(self, locker_key):
         self.cache_key = f"{CACHE_KEY_TEMPLATE.get('pending_state_key')}_{locker_key}"

fir_ser/common/core/sysconfig.py

@@ -8,7 +8,7 @@ import json
 import logging
 import re
 
-from django.template import Context, Template
+from django.template import Context, Template, TemplateSyntaxError
 from django.template.base import VariableNode
 from rest_framework import serializers
 
@@ -58,10 +58,24 @@ class ConfigCacheBase(object):
                         logger.warning(f"get same render key. so continue")
                         continue
                     context_dict[sys_obj_dict['key']] = sys_obj_dict['value']
-
-                value = get_render_context(value, context_dict)
+                try:
+                    value = get_render_context(value, context_dict)
+                except TemplateSyntaxError as e:
+                    res_list = re.findall("Could not parse the remainder: '{{(.*?)}}'", str(e))
+                    for res in res_list:
+                        r_value = get_render_context(f'{{{{{res}}}}}', context_dict)
+                        value = value.replace(f'{{{{{res}}}}}', r_value)
+                    value = get_render_context(value, context_dict)
+                except Exception as e:
+                    logger.warning(f"db config - render failed {e}")
             except Exception as e:
                 logger.warning(f"db config - render failed {e}")
+        if isinstance(value, str):
+            v_group = re.findall('"(.*?)"', value)
+            if v_group and len(v_group) == 1 and v_group[0].isdigit():
+                return int(v_group[0])
+        if value.isdigit():
+            return int(value)
         return value
 
     def get_value_from_db(self, key):
@@ -297,6 +311,14 @@ class UserDownloadTimesCache(ConfigCacheBase):
     def SIGN_EXTRA_MULTIPLE(self):
         return super().get_value('SIGN_EXTRA_MULTIPLE', DOWNLOADTIMESCONF.SIGN_EXTRA_MULTIPLE)
 
+    @property
+    def APP_USE_BASE_DOWNLOAD_TIMES(self):
+        return super().get_value('APP_USE_BASE_DOWNLOAD_TIMES', DOWNLOADTIMESCONF.APP_USE_BASE_DOWNLOAD_TIMES)
+
+    @property
+    def PRIVATE_OSS_DOWNLOAD_TIMES(self):
+        return super().get_value('PRIVATE_OSS_DOWNLOAD_TIMES', DOWNLOADTIMESCONF.PRIVATE_OSS_DOWNLOAD_TIMES)
+
 
 class OssStorageConfCache(ConfigCacheBase):
     def __init__(self, *args, **kwargs):

fir_ser/common/utils/caches.py

@@ -230,10 +230,15 @@ def get_user_cert_auth_status(user_id):
 def check_user_has_all_download_times(app_obj, user_obj):
     if user_obj:
         user_id = user_obj.pk
+        storage = user_obj.storage
     else:
         user_id = app_obj.user_id_id
+        storage = app_obj.user_id.storage
     auth_status = get_user_cert_auth_status(user_id)
-    d_count = get_app_d_count_by_app_id(app_obj.app_id)
+    d_count = get_app_d_count_by_app_id(app_obj.app_id, storage)
+    a = get_user_free_download_times(user_id, auth_status=auth_status)
+    b = d_count
+    c = check_user_can_download(user_id)
     return get_user_free_download_times(user_id, auth_status=auth_status) - d_count >= 0 or check_user_can_download(
         user_id)
 
@@ -241,8 +246,9 @@ def check_user_has_all_download_times(app_obj, user_obj):
 def consume_user_download_times_by_app_obj(app_obj):
     user_id = app_obj.user_id_id
     auth_status = get_user_cert_auth_status(user_id)
-    amount = get_app_d_count_by_app_id(app_obj.app_id)
-    if consume_user_download_times(user_id, app_obj.app_id, int(amount * Config.SIGN_EXTRA_MULTIPLE), auth_status):
+    amount = get_app_d_count_by_app_id(app_obj.app_id, False)
+    if consume_user_download_times(user_id, app_obj.app_id, int(amount * (Config.SIGN_EXTRA_MULTIPLE - 1)),
+                                   auth_status):
         return False
     return True
 
@@ -290,7 +296,7 @@ def update_order_info(user_id, out_trade_no, payment_number, payment_type, descr
                         order_obj.save(
                             update_fields=["status", "payment_type", "payment_number", "pay_time",
                                            "description"])
-                        add_user_download_times(user_id, download_times)
+                        add_user_download_times(user_id, download_times * Config.APP_USE_BASE_DOWNLOAD_TIMES)
                         logger.info(f"{user_obj} 订单 {out_trade_no} msg：{order_obj.description}")
                         pay_success_notify(user_obj, order_obj)
                         return True
@@ -340,6 +346,8 @@ def get_wx_ticket_login_info_cache(ticket):
 
 
 def add_download_times_free_base(user_obj, amount, payment_name, description, order_type=1):
+    if amount <= 0:
+        return True
     order_number = get_order_num()
     order_obj = Order.objects.create(payment_type=2, order_number=order_number, payment_number=order_number,
                                      user_id=user_obj, status=1, order_type=order_type, actual_amount=0,

fir_ser/common/utils/download.py

@@ -68,10 +68,10 @@ def get_app_instance_by_cache(app_id, limit):
     app_obj_cache = app_instance_cache.get_storage_cache()
     if not app_obj_cache:
         app_obj_cache = Apps.objects.filter(app_id=app_id).values("pk", 'user_id', 'type', 'need_password',
-                                                                  'issupersign',
+                                                                  'issupersign', 'user_id__storage',
                                                                   'user_id__certification__status').first()
         if app_obj_cache:
-            app_obj_cache['d_count'] = get_app_d_count_by_app_id(app_id)
+            app_obj_cache['d_count'] = get_app_d_count_by_app_id(app_id, app_obj_cache.get('user_id__storage', False))
             app_instance_cache.set_storage_cache(app_obj_cache, limit)
     if not app_obj_cache:
         return False, '应用不存在'

fir_ser/config.py

@@ -391,10 +391,14 @@ bIX1aWjPxirQX9mzaL3oEQI=
 
 class DOWNLOADTIMESCONF(object):
     SIGN_EXTRA_MULTIPLE = 2  # 超级签名消耗额外倍数，超级签名需要占用的服务大量资源
-    USER_FREE_DOWNLOAD_TIMES = 5
-    AUTH_USER_FREE_DOWNLOAD_TIMES = 10
-    NEW_USER_GIVE_DOWNLOAD_TIMES = 100
-    AUTH_USER_GIVE_DOWNLOAD_TIMES = 200
+
+    APP_USE_BASE_DOWNLOAD_TIMES = 100  # 单个应用下载消费点数
+    PRIVATE_OSS_DOWNLOAD_TIMES = 4  # 私有存储单个应用下载消费点数
+
+    USER_FREE_DOWNLOAD_TIMES = "{% widthratio 5 1 {{APP_USE_BASE_DOWNLOAD_TIMES}}  %}"  # 用户每日免费下载点数
+    AUTH_USER_FREE_DOWNLOAD_TIMES = '{% widthratio 10 1 {{APP_USE_BASE_DOWNLOAD_TIMES}}  %}'  # 认证用户每日免费下载点数
+    NEW_USER_GIVE_DOWNLOAD_TIMES = "{% widthratio 100 1 {{APP_USE_BASE_DOWNLOAD_TIMES}}  %}"  # 新用户注册赠送下载点数
+    AUTH_USER_GIVE_DOWNLOAD_TIMES = "{% widthratio 200 1 {{APP_USE_BASE_DOWNLOAD_TIMES}}  %}"  # 用户认证成功赠送下载点数
 
 
 class APPLEDEVELOPERCONF(object):

fir_ser/fir_ser/settings.py

@@ -261,6 +261,7 @@ CORS_ALLOW_HEADERS = (
 CACHE_KEY_TEMPLATE = {
     'config_key': 'config',
     'task_state_key': 'task_state',
+    'task_progress_key': 'task_progress',
     'pending_state_key': 'pending_state',
     'wx_login_bind_key': 'wx_login_bind',
     'notify_loop_msg_key': 'notify_loop_msg',

fir_ser/xsign/tasks.py

@@ -101,14 +101,14 @@ def run_resign_task_do(app_id, developer_id, developer_app_id, need_download_pro
 
 @app.task
 def auto_check_ios_developer_active_job():
-    auto_check_ios_developer_active()
+    return auto_check_ios_developer_active()
 
 
 @app.task
 def get_best_proxy_ips_job():
-    get_best_proxy_ips()
+    return get_best_proxy_ips()
 
 
 @app.task
 def auto_clean_sign_log_job():
-    auto_clean_sign_log()
+    return auto_clean_sign_log()