Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 67 additions & 13 deletions bkflow/plugin/services/open_plugin_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

from bkflow.exceptions import APIResponseError, ValidationError
from bkflow.pipeline_plugins.query.uniform_api.utils import UniformAPIClient
from bkflow.plugin_catalog.services.source_sync_state_service import SourceSyncStateService
from bkflow.plugin.models import (
OPEN_PLUGIN_WRAPPER_VERSION,
OpenPluginCatalogIndex,
Expand Down Expand Up @@ -83,9 +84,18 @@ def sync_space_plugins_with_meta(cls, space_id, source_key=None, username="admin
元信息直接来自来源响应,不再回查只收录 V4 的缓存表——否则 V2/V3 协议插件的
名称与描述会永远取不到(响应 schema 里 `name` 是所有版本都有的必填字段)。

:return: {source_key: {plugin_id: (name, description)}}
按**来源**隔离失败:一个来源采集失败不影响其它来源,失败原因与采集代次
一起返回,调用方据此决定"跳过"还是"按覆盖范围失效",以及结果是否已过期。

:return: {
"sources": {source_key: {plugin_id: (name, description)}},
"failures": {source_key: (error_type, error_message)},
"generations": {source_key: collect_generation},
}
"""
source_plugins = cls._collect_source_plugins(space_id=space_id, source_key=source_key, username=username)
source_plugins, failures, generations = cls._collect_source_plugins(
space_id=space_id, source_key=source_key, username=username, collect_errors=True
)

for current_source_key, plugins_by_id in source_plugins.items():
cls._refresh_catalog_index(
Expand All @@ -95,32 +105,76 @@ def sync_space_plugins_with_meta(cls, space_id, source_key=None, username="admin
)

return {
src: {
plugin_id: (api_item.get("name") or "", api_item.get("description") or "")
for plugin_id, api_item in plugins_by_id.items()
}
for src, plugins_by_id in source_plugins.items()
"sources": {
src: {
plugin_id: (api_item.get("name") or "", api_item.get("description") or "")
for plugin_id, api_item in plugins_by_id.items()
}
for src, plugins_by_id in source_plugins.items()
},
"failures": failures,
"generations": generations,
}

@classmethod
def _collect_source_plugins(cls, space_id, source_key=None, username="admin"):
def _collect_source_plugins(cls, space_id, source_key=None, username="admin", collect_errors=False):
"""实际发请求拉取每个已配置来源的完整插件清单,按"执行来源"聚合去重。

`sync_space_plugins`(只服务 V4 缓存)与 `sync_space_plugins_with_meta`(服务统一插件目录,
全版本不过滤并直接带回元信息)共用这一个方法,避免同一个来源在一次同步周期里被重复请求两次。

采集期间登记**采集代次**与**配置代次**(见 `SourceSyncStateService`):
采集发起时 +1 一次采集代次并记下当时的配置代次,提交方拿这个代次来判定
"结果是否已过期"——配置在采集期间被改/删时,旧结果会被丢弃而不是当成当前结果
使用(避免误判插件失效)。

`collect_errors=True` 时按**来源**隔离失败:一个来源挂了不影响其它来源,
失败信息与采集代次一起返回给调用方(统一插件目录要按来源决定
"是否按覆盖范围失效")。
"""
source_plugins = {}
failures = {}
generations = {}
for api_key, api_entry in cls._get_sources(space_id=space_id, source_key=source_key).items():
current_source_key = cls._effective_source_key(api_key=api_key, api_entry=api_entry)
api_list = cls._fetch_api_list(
space_id=space_id,
api_entry=api_entry,
username=username,
)
generation = cls._begin_source_collect(space_id, current_source_key)
generations[current_source_key] = generation
try:
api_list = cls._fetch_api_list(
space_id=space_id,
api_entry=api_entry,
username=username,
)
except Exception as exc: # noqa: BLE001 按来源隔离,一个来源失败不影响其它来源
if not collect_errors:
raise
error_type = SourceSyncStateService.classify_error(exc)
SourceSyncStateService.mark_failure(
space_id, current_source_key, str(exc), error_type=error_type
)
logger.exception(
"[开放插件目录] 来源采集失败: space_id=%s, source_key=%s, error_type=%s",
space_id,
current_source_key,
error_type,
)
failures[current_source_key] = (error_type, str(exc))
continue
plugins_by_id = source_plugins.setdefault(current_source_key, {})
plugins_by_id.update({api_item["id"]: api_item for api_item in api_list})
if collect_errors:
return source_plugins, failures, generations
return source_plugins

@staticmethod
def _begin_source_collect(space_id, source_key):
"""登记一次采集(采集代次 +1、记下当前配置代次)。

`SourceSyncStateService` 只依赖 `plugin_catalog.models` 与 `space.*`,
不反向依赖 `plugin.services`,因此可以安全地模块级导入(无循环依赖)。
"""
return SourceSyncStateService.begin_collect(space_id, source_key)

@classmethod
def list_space_plugins(cls, space_id, source_key=None):
catalog_qs = OpenPluginCatalogIndex.objects.filter(space_id=space_id).order_by("source_key", "plugin_name")
Expand Down
8 changes: 6 additions & 2 deletions bkflow/plugin/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,17 @@ def dispatch_open_plugin_catalog_sync():

def _dispatch_plugin_catalog_sync():
"""为已完成插件目录迁移的空间投递统一插件目录同步任务。延迟 import 避免
bkflow.plugin 对 bkflow.plugin_catalog 产生模块级依赖/循环导入。"""
bkflow.plugin 对 bkflow.plugin_catalog 产生模块级依赖/循环导入。

这是**周期调度**入口(Beat 30 分钟一轮,`trigger=schedule`);失败后的快速重试
由任务自身投递自己(同一个任务,`trigger=retry`),不新增 Beat 入口。
"""
from bkflow.plugin_catalog.services.sync_service import PluginCatalogSyncService
from bkflow.plugin_catalog.tasks import sync_space_plugin_catalog_task

space_ids = sorted(set(PluginCatalogSyncService.iter_ready_spaces()))
for space_id in space_ids:
sync_space_plugin_catalog_task.delay(space_id=space_id)
sync_space_plugin_catalog_task.delay(space_id=space_id, trigger="schedule")

logger.info("空间插件目录同步任务已投递: space_count=%s", len(space_ids))
return len(space_ids)
36 changes: 36 additions & 0 deletions bkflow/plugin_catalog/migrations/0002_sourcesyncstate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Generated by Django 4.2.30 on 2026-11-17 07:39

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('plugin_catalog', '0001_initial'),
]

operations = [
migrations.CreateModel(
name='SourceSyncState',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('space_id', models.IntegerField(db_index=True, verbose_name='空间ID')),
('source_key', models.CharField(blank=True, default='', max_length=64, verbose_name='来源标识')),
('config_generation', models.CharField(blank=True, default='', max_length=64, verbose_name='配置代次')),
('collect_generation', models.BigIntegerField(default=0, verbose_name='采集代次')),
('failure_round', models.IntegerField(default=0, verbose_name='故障轮次')),
('retry_count', models.IntegerField(default=0, verbose_name='已重试次数')),
('next_retry_at', models.DateTimeField(blank=True, null=True, verbose_name='下次快速重试时间')),
('exhausted', models.BooleanField(default=False, verbose_name='快速重试是否已耗尽')),
('last_error', models.TextField(blank=True, default='', verbose_name='最近一次错误')),
('last_error_type', models.CharField(blank=True, default='', max_length=32, verbose_name='最近一次错误类型')),
('last_synced_at', models.DateTimeField(blank=True, null=True, verbose_name='最近一次成功同步时间')),
],
options={
'verbose_name': '来源同步状态',
'verbose_name_plural': '来源同步状态',
'indexes': [models.Index(fields=['space_id', 'source_key'], name='plugin_cata_space_i_080f9e_idx')],
'unique_together': {('space_id', 'source_key')},
},
),
]
47 changes: 47 additions & 0 deletions bkflow/plugin_catalog/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -376,3 +376,50 @@ class Meta:

def __str__(self):
return f"{self.space_id}:{self.operate_type}:{self.instance_id}:{self.operator}"


class SourceSyncState(models.Model):
"""
来源同步状态:承载**有限重试**与**代次判断**两个机制。

- **粒度**:一行 = 一个 `(space_id, source_key)`。`builtin` / `bk_plugin` 的
`source_key` 恒为空串(这两类没有来源细分),API 插件按「执行来源」分行。
- **代次**:`config_generation` 是来源配置的代次标识(配置内容哈希),
`collect_generation` 是采集代次(每次发起采集 +1)。采集期间配置被改/删时,
旧结果晚到会因代次不一致被丢弃,不会被当成当前结果使用(避免误判插件失效)。
- **有限重试**:`retry_count` / `next_retry_at` / `exhausted` 记录"还能不能快速重试"。
耗尽后同一故障不重新开启快速重试,回到低频检查,只有新的成功证据才能恢复。
- **按覆盖范围失效**:`last_error_type` 区分「超时(覆盖范围不明,不判定失效)」
与「鉴权失败(该来源整体挂了,按覆盖范围失效)」。

与缓存表 / 队列消息的区别:这些状态**持久化在库里**,进程重启不丢,
也不依赖某条队列消息还在不在(设计文档明确"不以缓存锁或队列消息代替")。
"""

space_id = models.IntegerField(_("空间ID"), db_index=True)
source_key = models.CharField(_("来源标识"), max_length=64, blank=True, default="")

# ------------------------------------------------------------------ 代次 ----
config_generation = models.CharField(_("配置代次"), max_length=64, blank=True, default="")
collect_generation = models.BigIntegerField(_("采集代次"), default=0)

# -------------------------------------------------------------- 故障与重试 ----
failure_round = models.IntegerField(_("故障轮次"), default=0)
retry_count = models.IntegerField(_("已重试次数"), default=0)
next_retry_at = models.DateTimeField(_("下次快速重试时间"), null=True, blank=True)
exhausted = models.BooleanField(_("快速重试是否已耗尽"), default=False)
last_error = models.TextField(_("最近一次错误"), blank=True, default="")
last_error_type = models.CharField(_("最近一次错误类型"), max_length=32, blank=True, default="")
last_synced_at = models.DateTimeField(_("最近一次成功同步时间"), null=True, blank=True)

class Meta:
verbose_name = _("来源同步状态")
verbose_name_plural = _("来源同步状态")
app_label = "plugin_catalog"
unique_together = ("space_id", "source_key")
indexes = [
models.Index(fields=["space_id", "source_key"]),
]

def __str__(self):
return f"{self.space_id}:{self.source_key or '(builtin)'}"
4 changes: 0 additions & 4 deletions bkflow/plugin_catalog/serializers/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,10 +151,6 @@ def validate(self, attrs):
return attrs


class SuggestDirectoryQuerySerializer(serializers.Serializer):
space_id = serializers.IntegerField(help_text=_("空间ID"))


class PluginDetailQuerySerializer(serializers.Serializer):
space_id = serializers.IntegerField(help_text=_("空间ID"))

Expand Down
14 changes: 3 additions & 11 deletions bkflow/plugin_catalog/services/catalog_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,9 @@ def _resolve_replace_plugin(space_id, entry_id, replace_plugin_id):
if replace_entry.id == entry_id:
raise ValidationError("建议替换插件不能是插件自身")
if replace_entry.status != PluginCatalogStatus.ENABLE:
raise ValidationError(
f"建议替换插件 {replace_plugin_id} 当前不是可用状态,不能作为替换建议"
)
# 用插件名称而不是 id,前端可直接展示给用户
plugin_name = replace_entry.plugin_name
raise ValidationError(f"建议替换插件「{plugin_name}」当前不是可用状态,不能作为替换建议")
return replace_entry

# ----------------------------------------------------------- 编辑单条(01) ----
Expand Down Expand Up @@ -207,14 +207,6 @@ def transition_status(
)
return entry

@classmethod
def suggest_restore_directory(cls, space_id, entry_id):
"""03 · 启用二次确认弹窗的默认目录建议:原目录仍存在则沿用,否则回「未分类」。"""
entry = cls._get_entry_or_404(space_id, entry_id)
if entry.directory_id and PluginDirectory.objects.filter(space_id=space_id, pk=entry.directory_id).exists():
return {"directory_id": entry.directory_id}
return {"directory_id": None}

@classmethod
def update_reason(
cls,
Expand Down
Loading
Loading