diff --git a/api/extensions/ext_storage.py b/api/extensions/ext_storage.py index 4c811c66ba..bd35278544 100644 --- a/api/extensions/ext_storage.py +++ b/api/extensions/ext_storage.py @@ -102,6 +102,9 @@ class Storage: def delete(self, filename): return self.storage_runner.delete(filename) + def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]: + return self.storage_runner.scan(path, files=files, directories=directories) + storage = Storage() diff --git a/api/extensions/storage/base_storage.py b/api/extensions/storage/base_storage.py index 0dedd7ff8c..7c7316a107 100644 --- a/api/extensions/storage/base_storage.py +++ b/api/extensions/storage/base_storage.py @@ -30,3 +30,7 @@ class BaseStorage(ABC): @abstractmethod def delete(self, filename): raise NotImplementedError + + @abstractmethod + def scan(self, path, files, directories) -> list[str]: + raise NotImplementedError diff --git a/api/extensions/storage/opendal_storage.py b/api/extensions/storage/opendal_storage.py index ee8cfa9179..12e2738e9d 100644 --- a/api/extensions/storage/opendal_storage.py +++ b/api/extensions/storage/opendal_storage.py @@ -80,3 +80,20 @@ class OpenDALStorage(BaseStorage): logger.debug(f"file {filename} deleted") return logger.debug(f"file {filename} not found, skip delete") + + def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]: + if not self.exists(path): + raise FileNotFoundError("Path not found") + + all_files = self.op.scan(path=path) + if files and directories: + logger.debug(f"files and directories on {path} scanned") + return [f.path for f in all_files] + if files: + logger.debug(f"files on {path} scanned") + return [f.path for f in all_files if not f.path.endswith("/")] + elif directories: + logger.debug(f"directories on {path} scanned") + return [f.path for f in all_files if f.path.endswith("/")] + else: + raise ValueError("At least one of files or directories must be True")