import datetime import time import click from flask import current_app from werkzeug.exceptions import NotFound import app from extensions.ext_database import db from models.dataset import Embedding @app.celery.task(queue='dataset') def clean_embedding_cache_task(): click.echo(click.style('Start clean embedding cache.', fg='green')) clean_days = int(current_app.config.get('CLEAN_DAY_SETTING')) start_at = time.perf_counter() thirty_days_ago = datetime.datetime.now() - datetime.timedelta(days=clean_days) page = 1 while True: try: embeddings = db.session.query(Embedding).filter(Embedding.created_at < thirty_days_ago) \ .order_by(Embedding.created_at.desc()).paginate(page=page, per_page=100) except NotFound: break for embedding in embeddings: db.session.delete(embedding) db.session.commit() page += 1 end_at = time.perf_counter() click.echo(click.style('Cleaned embedding cache from db success latency: {}'.format(end_at - start_at), fg='green'))