AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-10 04:09:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

add clean unused dataset command (#609)

Browse Source
This commit is contained in:
Jyong 2023-07-20 11:08:28 +08:00 committed by GitHub
parent 026f0bfce9
commit 52c84da051
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 15 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
api/commands.py
View File

@ -225,26 +225,22 @@ def clean_unused_dataset_indexes():
).all() ).all()
if not documents or len(documents) == 0: if not documents or len(documents) == 0:
try: try:
all_documents = db.session.query(Document).filter( # remove index
Document.dataset_id == dataset.id, vector_index = IndexBuilder.get_index(dataset, 'high_quality')
Document.indexing_status == 'completed', kw_index = IndexBuilder.get_index(dataset, 'economy')
Document.enabled == True, # delete from vector index
Document.archived == False, if vector_index:
).all() vector_index.delete()
if all_documents and len(all_documents)>0: kw_index.delete()
update_params = { # update document
Document.enabled: False update_params = {
} Document.enabled: False
}
Document.query.filter_by(dataset_id=dataset.id).update(update_params) Document.query.filter_by(dataset_id=dataset.id).update(update_params)
db.session.commit() db.session.commit()
# remove index click.echo(click.style('Cleaned unused dataset {} from db success!'.format(dataset.id),
vector_index = IndexBuilder.get_index(dataset, 'high_quality') fg='green'))
kw_index = IndexBuilder.get_index(dataset, 'economy')
# delete from vector index
if vector_index:
vector_index.delete()
kw_index.delete()
except Exception as e: except Exception as e:
click.echo( click.echo(
click.style('clean dataset index error: {} {}'.format(e.__class__.__name__, str(e)), click.style('clean dataset index error: {} {}'.format(e.__class__.__name__, str(e)),