You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
348 lines
8.5 KiB
Go
348 lines
8.5 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/1Panel-dev/1Panel/agent/app/dto"
|
|
"github.com/1Panel-dev/1Panel/agent/app/model"
|
|
"github.com/1Panel-dev/1Panel/agent/app/task"
|
|
"github.com/1Panel-dev/1Panel/agent/constant"
|
|
"github.com/1Panel-dev/1Panel/agent/global"
|
|
"github.com/1Panel-dev/1Panel/agent/i18n"
|
|
"github.com/1Panel-dev/1Panel/agent/utils/common"
|
|
"github.com/1Panel-dev/1Panel/agent/utils/req_helper"
|
|
"github.com/1Panel-dev/1Panel/agent/utils/xpack"
|
|
)
|
|
|
|
type appSyncContext struct {
|
|
task *task.Task
|
|
httpClient http.Client
|
|
baseRemoteUrl string
|
|
systemVersion string
|
|
appsMap map[string]model.App
|
|
settingService ISettingService
|
|
list *dto.AppList
|
|
oldAppIds []uint
|
|
appTags []*model.AppTag
|
|
}
|
|
|
|
func (a AppService) syncAppStoreTask(t *task.Task) (err error) {
|
|
updateRes, err := a.GetAppUpdate()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !updateRes.CanUpdate {
|
|
if updateRes.IsSyncing {
|
|
t.Log(i18n.GetMsgByKey("AppStoreIsSyncing"))
|
|
return nil
|
|
}
|
|
global.LOG.Infof("[AppStore] Appstore is up to date")
|
|
t.Log(i18n.GetMsgByKey("AppStoreIsUpToDate"))
|
|
return nil
|
|
}
|
|
|
|
list := &dto.AppList{}
|
|
if updateRes.AppList == nil {
|
|
list, err = getAppList()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
list = updateRes.AppList
|
|
}
|
|
|
|
settingService := NewISettingService()
|
|
_ = settingService.Update("AppStoreSyncStatus", constant.StatusSyncing)
|
|
|
|
setting, err := settingService.GetSettingInfo()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ctx := &appSyncContext{
|
|
task: t,
|
|
httpClient: http.Client{Timeout: time.Duration(constant.TimeOut20s) * time.Second, Transport: xpack.LoadRequestTransport()},
|
|
baseRemoteUrl: fmt.Sprintf("%s/%s/1panel", global.CONF.RemoteURL.AppRepo, global.CONF.Base.Mode),
|
|
systemVersion: setting.SystemVersion,
|
|
settingService: settingService,
|
|
list: list,
|
|
appTags: make([]*model.AppTag, 0),
|
|
}
|
|
|
|
if err = SyncTags(list.Extra); err != nil {
|
|
return err
|
|
}
|
|
deleteCustomApp()
|
|
|
|
oldApps, err := appRepo.GetBy(appRepo.WithNotLocal())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ctx.oldAppIds = make([]uint, 0, len(oldApps))
|
|
for _, old := range oldApps {
|
|
ctx.oldAppIds = append(ctx.oldAppIds, old.ID)
|
|
}
|
|
|
|
ctx.appsMap = getApps(oldApps, list.Apps, setting.SystemVersion, t)
|
|
|
|
if err = ctx.syncAppIconsAndDetails(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err = ctx.classifyAndPersistApps(); err != nil {
|
|
return err
|
|
}
|
|
|
|
_ = settingService.Update("AppStoreSyncStatus", constant.StatusSyncSuccess)
|
|
_ = settingService.Update("AppStoreLastModified", strconv.Itoa(list.LastModified))
|
|
global.LOG.Infof("[AppStore] Appstore sync completed")
|
|
return nil
|
|
}
|
|
|
|
func (c *appSyncContext) syncAppIconsAndDetails() error {
|
|
c.task.LogStart(i18n.GetMsgByKey("SyncAppDetail"))
|
|
global.LOG.Infof("[AppStore] sync app detail start, total: %d", len(c.list.Apps))
|
|
|
|
downloadIconNum := 0
|
|
total := len(c.list.Apps)
|
|
|
|
for _, l := range c.list.Apps {
|
|
downloadIconNum++
|
|
if downloadIconNum%10 == 0 {
|
|
c.task.LogWithProgress(i18n.GetMsgByKey("SyncAppDetail"), downloadIconNum, total)
|
|
}
|
|
|
|
app, ok := c.appsMap[l.AppProperty.Key]
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
iconStr := c.downloadAppIcon(l.Icon)
|
|
if iconStr == "" {
|
|
global.LOG.Infof("[AppStore] save failed url=%s", l.Icon)
|
|
}
|
|
app.Icon = iconStr
|
|
|
|
app.TagsKey = l.AppProperty.Tags
|
|
if l.AppProperty.Recommend > 0 {
|
|
app.Recommend = l.AppProperty.Recommend
|
|
} else {
|
|
app.Recommend = 9999
|
|
}
|
|
app.ReadMe = l.ReadMe
|
|
app.LastModified = l.LastModified
|
|
|
|
versions := l.Versions
|
|
detailsMap := getAppDetails(app.Details, versions)
|
|
for _, v := range versions {
|
|
version := v.Name
|
|
detail := detailsMap[version]
|
|
versionUrl := fmt.Sprintf("%s/%s/%s", c.baseRemoteUrl, app.Key, version)
|
|
|
|
paramByte, _ := json.Marshal(v.AppForm)
|
|
var appForm dto.AppForm
|
|
_ = json.Unmarshal(paramByte, &appForm)
|
|
|
|
if appForm.SupportVersion > 0 && common.CompareVersion(strconv.FormatFloat(appForm.SupportVersion, 'f', -1, 64), c.systemVersion) {
|
|
delete(detailsMap, version)
|
|
continue
|
|
}
|
|
|
|
if _, ok := InitTypes[app.Type]; ok {
|
|
dockerComposeUrl := fmt.Sprintf("%s/%s", versionUrl, "docker-compose.yml")
|
|
_, composeRes, err := req_helper.HandleRequestWithClient(&c.httpClient, dockerComposeUrl, http.MethodGet, constant.TimeOut20s)
|
|
if err == nil {
|
|
detail.DockerCompose = string(composeRes)
|
|
}
|
|
} else {
|
|
detail.DockerCompose = ""
|
|
}
|
|
|
|
detail.Params = string(paramByte)
|
|
detail.DownloadUrl = fmt.Sprintf("%s/%s", versionUrl, app.Key+"-"+version+".tar.gz")
|
|
detail.DownloadCallBackUrl = v.DownloadCallBackUrl
|
|
detail.Update = true
|
|
detail.LastModified = v.LastModified
|
|
detailsMap[version] = detail
|
|
}
|
|
|
|
var newDetails []model.AppDetail
|
|
for _, detail := range detailsMap {
|
|
newDetails = append(newDetails, detail)
|
|
}
|
|
app.Details = newDetails
|
|
c.appsMap[l.AppProperty.Key] = app
|
|
}
|
|
|
|
global.LOG.Infof("[AppStore] download icon success: %d, total: %d",
|
|
downloadIconNum, total)
|
|
|
|
c.task.LogSuccess(i18n.GetMsgByKey("SyncAppDetail"))
|
|
return nil
|
|
}
|
|
|
|
func (c *appSyncContext) downloadAppIcon(iconUrl string) string {
|
|
iconStr := ""
|
|
|
|
code, iconRes, err := req_helper.HandleRequestWithClient(&c.httpClient, iconUrl, http.MethodGet, constant.TimeOut20s)
|
|
if err == nil {
|
|
if code == http.StatusOK {
|
|
if len(iconRes) > 0 {
|
|
if iconRes[0] != '<' {
|
|
iconStr = base64.StdEncoding.EncodeToString(iconRes)
|
|
}
|
|
}
|
|
} else {
|
|
global.LOG.Infof("[AppStore] download failed status=%d", code)
|
|
}
|
|
}
|
|
return iconStr
|
|
}
|
|
|
|
func (c *appSyncContext) classifyAndPersistApps() (err error) {
|
|
tags, _ := tagRepo.All()
|
|
var (
|
|
addAppArray []model.App
|
|
updateAppArray []model.App
|
|
deleteAppArray []model.App
|
|
deleteIds []uint
|
|
tagMap = make(map[string]uint, len(tags))
|
|
)
|
|
|
|
for _, v := range c.appsMap {
|
|
if v.ID == 0 {
|
|
addAppArray = append(addAppArray, v)
|
|
} else {
|
|
if v.Status == constant.AppTakeDown {
|
|
installs, _ := appInstallRepo.ListBy(context.Background(), appInstallRepo.WithAppId(v.ID))
|
|
if len(installs) > 0 {
|
|
updateAppArray = append(updateAppArray, v)
|
|
continue
|
|
}
|
|
deleteAppArray = append(deleteAppArray, v)
|
|
deleteIds = append(deleteIds, v.ID)
|
|
} else {
|
|
updateAppArray = append(updateAppArray, v)
|
|
}
|
|
}
|
|
}
|
|
|
|
tx, ctx := getTxAndContext()
|
|
defer func() {
|
|
if err != nil {
|
|
tx.Rollback()
|
|
return
|
|
}
|
|
}()
|
|
|
|
if len(addAppArray) > 0 {
|
|
if err = appRepo.BatchCreate(ctx, addAppArray); err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
if len(deleteAppArray) > 0 {
|
|
if err = appRepo.BatchDelete(ctx, deleteAppArray); err != nil {
|
|
return
|
|
}
|
|
if err = appDetailRepo.DeleteByAppIds(ctx, deleteIds); err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
for _, tag := range tags {
|
|
tagMap[tag.Key] = tag.ID
|
|
}
|
|
|
|
for _, update := range updateAppArray {
|
|
if err = appRepo.Save(ctx, &update); err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
apps := append(addAppArray, updateAppArray...)
|
|
|
|
var (
|
|
addDetails []model.AppDetail
|
|
updateDetails []model.AppDetail
|
|
deleteDetails []model.AppDetail
|
|
)
|
|
|
|
for _, app := range apps {
|
|
for _, tag := range app.TagsKey {
|
|
tagId, ok := tagMap[tag]
|
|
if ok {
|
|
exist, _ := appTagRepo.GetFirst(ctx, appTagRepo.WithByTagID(tagId), appTagRepo.WithByAppID(app.ID))
|
|
if exist == nil {
|
|
c.appTags = append(c.appTags, &model.AppTag{
|
|
AppId: app.ID,
|
|
TagId: tagId,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, d := range app.Details {
|
|
d.AppId = app.ID
|
|
if d.ID == 0 {
|
|
addDetails = append(addDetails, d)
|
|
} else {
|
|
if d.Status == constant.AppTakeDown {
|
|
runtime, _ := runtimeRepo.GetFirst(ctx, runtimeRepo.WithDetailId(d.ID))
|
|
if runtime != nil {
|
|
updateDetails = append(updateDetails, d)
|
|
continue
|
|
}
|
|
installs, _ := appInstallRepo.ListBy(ctx, appInstallRepo.WithDetailIdsIn([]uint{d.ID}))
|
|
if len(installs) > 0 {
|
|
updateDetails = append(updateDetails, d)
|
|
continue
|
|
}
|
|
deleteDetails = append(deleteDetails, d)
|
|
} else {
|
|
updateDetails = append(updateDetails, d)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(addDetails) > 0 {
|
|
if err = appDetailRepo.BatchCreate(ctx, addDetails); err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
if len(deleteDetails) > 0 {
|
|
if err = appDetailRepo.BatchDelete(ctx, deleteDetails); err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
for _, u := range updateDetails {
|
|
if err = appDetailRepo.Update(ctx, u); err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
if len(c.oldAppIds) > 0 {
|
|
if err = appTagRepo.DeleteByAppIds(ctx, deleteIds); err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
if len(c.appTags) > 0 {
|
|
if err = appTagRepo.BatchCreate(ctx, c.appTags); err != nil {
|
|
return
|
|
}
|
|
}
|
|
|
|
tx.Commit()
|
|
return nil
|
|
}
|