fix: ignore plugin already exists (#13888)

api/commands.py

@@ -707,12 +707,13 @@ def extract_unique_plugins(output_file: str, input_file: str):
 @click.option(
     "--output_file", prompt=True, help="The file to store the installed plugins.", default="installed_plugins.jsonl"
 )
-def install_plugins(input_file: str, output_file: str):
+@click.option("--workers", prompt=True, help="The number of workers to install plugins.", default=100)
+def install_plugins(input_file: str, output_file: str, workers: int):
     """
     Install plugins.
     """
     click.echo(click.style("Starting install plugins.", fg="white"))
 
-    PluginMigration.install_plugins(input_file, output_file)
+    PluginMigration.install_plugins(input_file, output_file, workers)
 
     click.echo(click.style("Install plugins completed.", fg="green"))

api/services/plugin/plugin_migration.py

@@ -356,7 +356,7 @@ class PluginMigration:
         return {"plugins": plugins, "plugin_not_exist": plugin_not_exist}
 
     @classmethod
-    def install_plugins(cls, extracted_plugins: str, output_file: str) -> None:
+    def install_plugins(cls, extracted_plugins: str, output_file: str, workers: int = 100) -> None:
         """
         Install plugins.
         """
@@ -370,7 +370,7 @@ class PluginMigration:
         fake_tenant_id = uuid4().hex
         logger.info(f"Installing {len(plugins['plugins'])} plugin instances for fake tenant {fake_tenant_id}")
 
-        thread_pool = ThreadPoolExecutor(max_workers=40)
+        thread_pool = ThreadPoolExecutor(max_workers=workers)
 
         response = cls.handle_plugin_instance_install(fake_tenant_id, plugins["plugins"])
         if response.get("failed"):
@@ -378,10 +378,17 @@ class PluginMigration:
 
         def install(tenant_id: str, plugin_ids: list[str]) -> None:
             logger.info(f"Installing {len(plugin_ids)} plugins for tenant {tenant_id}")
+            # fetch plugin already installed
+            installed_plugins = manager.list_plugins(tenant_id)
+            installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins]
             # at most 64 plugins one batch
             for i in range(0, len(plugin_ids), 64):
                 batch_plugin_ids = plugin_ids[i : i + 64]
-                batch_plugin_identifiers = [plugins["plugins"][plugin_id] for plugin_id in batch_plugin_ids]
+                batch_plugin_identifiers = [
+                    plugins["plugins"][plugin_id]
+                    for plugin_id in batch_plugin_ids
+                    if plugin_id not in installed_plugins_ids
+                ]
                 manager.install_from_identifiers(
                     tenant_id,
                     batch_plugin_identifiers,