|
|
|
|
@ -139,13 +139,17 @@ class AnalyticdbVectorBySql:
|
|
|
|
|
)
|
|
|
|
|
if embedding_dimension is not None:
|
|
|
|
|
index_name = f"{self._collection_name}_embedding_idx"
|
|
|
|
|
cur.execute(f"ALTER TABLE {self.table_name} ALTER COLUMN vector SET STORAGE PLAIN")
|
|
|
|
|
cur.execute(
|
|
|
|
|
f"CREATE INDEX {index_name} ON {self.table_name} USING ann(vector) "
|
|
|
|
|
f"WITH(dim='{embedding_dimension}', distancemeasure='{self.config.metrics}', "
|
|
|
|
|
f"pq_enable=0, external_storage=0)"
|
|
|
|
|
)
|
|
|
|
|
cur.execute(f"CREATE INDEX ON {self.table_name} USING gin(to_tsvector)")
|
|
|
|
|
try:
|
|
|
|
|
cur.execute(f"ALTER TABLE {self.table_name} ALTER COLUMN vector SET STORAGE PLAIN")
|
|
|
|
|
cur.execute(
|
|
|
|
|
f"CREATE INDEX {index_name} ON {self.table_name} USING ann(vector) "
|
|
|
|
|
f"WITH(dim='{embedding_dimension}', distancemeasure='{self.config.metrics}', "
|
|
|
|
|
f"pq_enable=0, external_storage=0)"
|
|
|
|
|
)
|
|
|
|
|
cur.execute(f"CREATE INDEX ON {self.table_name} USING gin(to_tsvector)")
|
|
|
|
|
except Exception as e:
|
|
|
|
|
if "already exists" not in str(e):
|
|
|
|
|
raise e
|
|
|
|
|
redis_client.set(collection_exist_cache_key, 1, ex=3600)
|
|
|
|
|
|
|
|
|
|
def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs):
|
|
|
|
|
@ -177,9 +181,11 @@ class AnalyticdbVectorBySql:
|
|
|
|
|
return cur.fetchone() is not None
|
|
|
|
|
|
|
|
|
|
def delete_by_ids(self, ids: list[str]) -> None:
|
|
|
|
|
if not ids:
|
|
|
|
|
return
|
|
|
|
|
with self._get_cursor() as cur:
|
|
|
|
|
try:
|
|
|
|
|
cur.execute(f"DELETE FROM {self.table_name} WHERE ref_doc_id IN %s", (tuple(ids),))
|
|
|
|
|
cur.execute(f"DELETE FROM {self.table_name} WHERE ref_doc_id = ANY(%s)", (ids,))
|
|
|
|
|
except Exception as e:
|
|
|
|
|
if "does not exist" not in str(e):
|
|
|
|
|
raise e
|
|
|
|
|
@ -240,7 +246,7 @@ class AnalyticdbVectorBySql:
|
|
|
|
|
ts_rank(to_tsvector, to_tsquery_from_text(%s, 'zh_cn'), 32) AS score
|
|
|
|
|
FROM {self.table_name}
|
|
|
|
|
WHERE to_tsvector@@to_tsquery_from_text(%s, 'zh_cn') {where_clause}
|
|
|
|
|
ORDER BY score DESC
|
|
|
|
|
ORDER BY (score,id) DESC
|
|
|
|
|
LIMIT {top_k}""",
|
|
|
|
|
(f"'{query}'", f"'{query}'"),
|
|
|
|
|
)
|
|
|
|
|
|