让CTFd自动备份，摆脱数据丢失烦恼

让CTFd自动备份，摆脱数据丢失烦恼

IShirai_KurokoI
  2024-01-12 2024-01-12 创建   2024-01-12 21:25:13 2024-01-12 21:25:13 更新      1.7k 字  9 分钟  

这一切都要起源于某次经历:我们平台数据呢？导入失败了，数据备份呢？没了？ 所以就有了自动备份的想法，首先我们需要做一个定时的执行器，既然使用的是flask，那么就用flask的定时执行器？然而发现并不行，因为只能设置间隔，无法定时（在夜间无人使用时进行备份），于是转而使用scheduler。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
logger = logging.getLogger('backup')


def config(app):
    if not get_config("backup:setup"):
        for key, val in {
            'enabled': 'false',
            'interval': '7',
            'time': '3',
            'max': '8',
            'setup': 'true'
        }.items():
            set_config('backup:' + key, val)


interval = 7
time = 3


def load(app):
    config(app)
    plugin_name = __name__.split('.')[-1]

    # 初始化日志
    logger_backup = logging.getLogger("backup")
    logger_backup.setLevel(logging.INFO)

    log_dir = app.config["LOG_FOLDER"]
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)
    logs = {
        "backup": os.path.join(log_dir, "backup.log"),
    }
    try:
        for log in logs.values():
            if not os.path.exists(log):
                open(log, "a").close()
        backup_log = logging.handlers.RotatingFileHandler(
            logs["backup"], maxBytes=10485760, backupCount=5
        )
        logger_backup.addHandler(backup_log)
    except IOError:
        pass
    stdout = logging.StreamHandler(stream=sys.stdout)
    logger_backup.addHandler(stdout)
    logger_backup.propagate = 0

    global interval, time
    interval = get_config("backup:interval")
    time = get_config("backup:time")

    register_plugin_assets_directory(
        app,
        base_path=f"/plugins/{plugin_name}/assets",
        endpoint='plugins.backup.assets')
    register_admin_plugin_menu_bar(title='Backup',
                                   route='/plugins/backup/admin/settings')

    page_blueprint = Blueprint("backup",
                               __name__,
                               template_folder="templates",
                               static_folder="static",
                               url_prefix="/plugins/backup")
    CTFd_API_v1.add_namespace(Namespace("backup-admin"),
                              path="/plugins/backup/admin")

    def format_size(size):
        # Convert bytes to human-readable format
        for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
            if size < 1024.0:
                break
            size /= 1024.0
        return "{:.2f} {}".format(size, unit)

    @page_blueprint.route('/admin/settings')
    @admins_only
    def admin_list_configs():
        global interval, time
        if get_config("backup:interval") is not interval or get_config("backup:time") is not time:
            interval = get_config("backup:interval")
            time = get_config("backup:time")
            log_simple("backup", "[{date}] [Auto Backup] 自动备份配置已更新，正在重设计划任务！")
            update_schedule(interval, time)

        upload_folder = os.path.join(
            os.path.normpath(app.root_path), app.config.get("UPLOAD_FOLDER")
        )
        auto_backups_folder = os.path.join(upload_folder, "autoBackups")
        os.makedirs(auto_backups_folder, exist_ok=True)
        backup_files = []
        for root, dirs, files in os.walk(auto_backups_folder):
            for file in files:
                file_path = os.path.join(root, file)
                file_size = os.path.getsize(file_path)
                file_date = os.path.getctime(file_path)
                backup_files.append({
                    'name': file,
                    'size': format_size(file_size),
                    'date': datetime.utcfromtimestamp(file_date)
                })
        return render_template('backup_config.html', backup_files=backup_files)

    @page_blueprint.route("/admin/download")
    @admins_only
    def admin_download_backup():
        upload_folder = os.path.join(
            os.path.normpath(app.root_path), app.config.get("UPLOAD_FOLDER")
        )
        auto_backups_folder = os.path.join(upload_folder, "autoBackups")
        os.makedirs(auto_backups_folder, exist_ok=True)

        backup_name = request.args.get("name")
        # 构造备份文件的完整路径
        backup_path = os.path.join(auto_backups_folder, backup_name)

        # 检查文件是否存在
        if os.path.exists(backup_path):
            # 使用send_file发送文件
            return send_file(
                backup_path,
                as_attachment=True
            )
        else:
            # 文件不存在，可以返回404或其他适当的响应
            return "File not found", 404

    @page_blueprint.route("/admin/delete")
    @admins_only
    def admin_delete_backup():
        upload_folder = os.path.join(
            os.path.normpath(app.root_path), app.config.get("UPLOAD_FOLDER")
        )
        auto_backups_folder = os.path.join(upload_folder, "autoBackups")
        os.makedirs(auto_backups_folder, exist_ok=True)

        backup_name = request.args.get("name")
        # 构造备份文件的完整路径
        backup_path = os.path.join(auto_backups_folder, backup_name)

        # 检查文件是否存在
        if os.path.exists(backup_path):
            os.remove(backup_path)
            return {
                'success': True,
                'message': '删除成功！'
            }, 200
        else:
            return {
                'success': False,
                'message': '文件不存在！'
            }, 200

    def custom_export_ctf():
        db = dataset.connect(get_app_config("SQLALCHEMY_DATABASE_URI"))
        backup = tempfile.NamedTemporaryFile()
        backup_zip = zipfile.ZipFile(backup, "w")
        tables = db.tables
        for table in tables:
            result = db[table].all()
            result_file = BytesIO()
            freeze_export(result, fileobj=result_file)
            result_file.seek(0)
            backup_zip.writestr("db/{}.json".format(table), result_file.read())
        if "alembic_version" not in tables:
            result = {
                "count": 1,
                "results": [{"version_num": get_current_revision()}],
                "meta": {},
            }
            result_file = BytesIO()
            json.dump(result, result_file)
            result_file.seek(0)
            backup_zip.writestr("db/alembic_version.json", result_file.read())
        uploader = get_uploader()
        uploader.sync()
        upload_folder = os.path.join(
            os.path.normpath(app.root_path), app.config.get("UPLOAD_FOLDER")
        )
        for root, _dirs, files in os.walk(upload_folder):
            for file in files:
                parent_dir = os.path.basename(root)
                if not "autoBackups" in parent_dir:
                    backup_zip.write(
                        os.path.join(root, file),
                        arcname=os.path.join("uploads", parent_dir, file),
                    )

        backup_zip.close()
        backup.seek(0)
        return backup

    def delete_oldest_file(folder_path):
        # 获取文件夹内所有文件
        files = [f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))]

        # 如果文件数大于8个，进行删除操作
        if len(files) > get_config("backup:max"):
            # 按文件的修改时间进行排序
            files = sorted(files, key=lambda x: os.path.getmtime(os.path.join(folder_path, x)))

            # 删除最旧的文件
            file_to_delete = os.path.join(folder_path, files[0])
            os.remove(file_to_delete)
            log_simple("backup", "[{date}] [Auto Backup] 删除了最旧的备份文件：{file_to_delete}",
                       file_to_delete=file_to_delete)

    def write_backup():
        backup_file = custom_export_ctf()

        # 如果不存在则创建目录
        upload_folder = os.path.join(
            os.path.normpath(app.root_path), app.config.get("UPLOAD_FOLDER")
        )
        auto_backups_folder = os.path.join(upload_folder, "autoBackups")
        os.makedirs(auto_backups_folder, exist_ok=True)

        # 写入备份文件
        name = ctf_name()
        day = current_backend_time().strftime("%Y-%m-%d_%T")
        full_name = os.path.join(auto_backups_folder, f"{name}.{day}.zip")
        with open(full_name, "wb") as target:
            shutil.copyfileobj(backup_file, target)
        log_simple("backup", "[{date}] [Auto Backup] 备份完成：{name}.{day}.zip", name=name, day=day)
        # 删除旧备份
        delete_oldest_file(auto_backups_folder)

    @page_blueprint.route("/admin/backupNow")
    @admins_only
    def admin_backup_now():
        write_backup()
        return {
            'success': True,
            'message': '备份完成！'
        }, 200

    def single_task(task, t):
        def wrap(func):
            @wraps(func)
            def inner(*args, **kwargs):
                add_result = cache.get(key=task)
                if not add_result:
                    cache.set(key=task, value=True, timeout=t)
                    try:
                        result = func(*args, **kwargs)
                        return result
                    except Exception as e:
                        raise e
                else:
                    return

            return inner

        return wrap

    def convert_hours_to_time_string(hours):
        return f"{hours:02d}:00"

    @single_task("backup", 1800)
    def backup():
        with app.app_context():
            if get_config("backup:enabled"):
                write_backup()
                log_simple("backup", "[{date}] [Auto Backup] 自动备份完成！")
            else:
                log_simple("backup", "[{date}] [Auto Backup] 自动备份未启用。")

    def check():
        global interval, time
        with app.app_context():
            if get_config("backup:interval") is not interval or get_config("backup:time") is not time:
                interval = get_config("backup:interval")
                time = get_config("backup:time")
                log_simple("backup", "[{date}] [Auto Backup] 自动备份配置已更新，正在重设计划任务！")
                update_schedule(interval, time)
            schedule.run_pending()

    def update_schedule(it, t):
        schedule.clear()
        schedule.every(it).days.at(convert_hours_to_time_string(t),
                                   pytz.timezone(get_config("backend_timezone", "Asia/Shanghai"))).do(backup)
        log_simple("backup", "[{date}] [Auto Backup] 计划任务重设完成！")
        log_simple("backup", '[{date}] [Auto Backup] 计划任务内容{jobs}', jobs=schedule.get_jobs())

    scheduler = APScheduler()
    scheduler.init_app(app)
    scheduler.start()
    scheduler.add_job(id='auto-backup-check',
                      func=check,
                      trigger="interval",
                      seconds=10)

    schedule.every(interval).days.at(convert_hours_to_time_string(time),
                                     pytz.timezone(get_config("backend_timezone", "Asia/Shanghai"))).do(backup)
    log_simple("backup", "[{date}] [Auto Backup] 计划任务内容{jobs}", jobs=schedule.get_jobs())

    app.register_blueprint(page_blueprint)

这里并没有使用原生的备份方法，因为原生的备份方法会将自动备份文件本身一同备份，增加磁盘占用，所以重写了方法排除了自动备份保存的目录。

另外singletask是通过redis缓存保证多工作线程的情况下备份不会被重复执行。

各工作线程的APScheduler每十秒执行一次检查，尝试运行任务（其实可以延长，降低资源占用，但是可能会导致一些问题，各位大佬可以测试一下）

20240112210515

插件地址：https://github.com/IShiraiKurokoI/CTFd-backup-plugin

  • 标题: 让CTFd自动备份，摆脱数据丢失烦恼
  • 作者: IShirai_KurokoI
  • 创建于 : 2024-01-12 00:00:00
  • 更新于 : 2024-01-12 21:25:13
  • 链接: https://ishiraikurokoi.top/2024-01-12-CTFd-Backup-Plugin/
  • 版权声明: 本文章采用 CC BY-NC-SA 4.0 进行许可。
评论
目录
让CTFd自动备份，摆脱数据丢失烦恼