im-group/scheduler-backend
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
vet
2026-05-28 00:16:19 +08:00
commit 52446ccf3f
54 changed files with 4617 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.env Normal file
View File

@@ -0,0 +1,11 @@
HTTP_PORT=16811
WORKER_HTTP_PORT=16812
MONGO_HOST=47.237.103.4
MONGO_PORT=27017
MONGO_USERNAME=minio_pC5wMB
MONGO_PASSWORD=rI57PJsJhnz_qlRkfnTa0RPT
MONGO_AUTHSOURCE=admin
MONGO_DATABASE=openim_v3
SCHEDULER_MONGO_DATABASE=scheduler_center
REDIS_ADDR=127.0.0.1:6379
REDIS_PASSWORD=

11
.env.example Normal file
View File

@@ -0,0 +1,11 @@
HTTP_PORT=16811
WORKER_HTTP_PORT=16812
MONGO_HOST=47.237.103.4
MONGO_PORT=27017
MONGO_USERNAME=
MONGO_PASSWORD=
MONGO_AUTHSOURCE=admin
MONGO_DATABASE=openim_v3
SCHEDULER_MONGO_DATABASE=scheduler_center
REDIS_ADDR=127.0.0.1:6379
REDIS_PASSWORD=

9
.env.local Normal file
View File

@@ -0,0 +1,9 @@
HTTP_PORT=16811
WORKER_HTTP_PORT=16812
MONGO_HOST=47.237.103.4
MONGO_PORT=27017
MONGO_USERNAME=minio_pC5wMB
MONGO_PASSWORD=rI57PJsJhnz_qlRkfnTa0RPT
MONGO_AUTHSOURCE=admin
MONGO_DATABASE=openim_v3
SCHEDULER_MONGO_DATABASE=scheduler_center_local

9
.env.prod Normal file
View File

@@ -0,0 +1,9 @@
HTTP_PORT=16811
WORKER_HTTP_PORT=16812
MONGO_HOST=47.237.103.4
MONGO_PORT=27017
MONGO_USERNAME=minio_pC5wMB
MONGO_PASSWORD=rI57PJsJhnz_qlRkfnTa0RPT
MONGO_AUTHSOURCE=admin
MONGO_DATABASE=openim_v3
SCHEDULER_MONGO_DATABASE=scheduler_center

6
.gitignore vendored Normal file
View File

@@ -0,0 +1,6 @@
_output/
*.exe
*.test
*.out
logs/
run/

170
README.md Normal file
View File

@@ -0,0 +1,170 @@
# scheduler-backend
独立的调度中心后端工程，负责：
- 管理 API
- 本地 Go handler 注册
- 手动执行
- `gocron` 定时执行
- MongoDB 中的任务配置与执行记录存储
## 当前能力
- `GET /admin/scheduler/handlers`
- `GET /admin/scheduler/jobs`
- `GET /admin/scheduler/jobs/:id`
- `POST /admin/scheduler/jobs`
- `PUT /admin/scheduler/jobs/:id`
- `POST /admin/scheduler/jobs/:id/toggle`
- `POST /admin/scheduler/jobs/:id/run`
- `GET /admin/scheduler/executions`
## 目录结构
```text
cmd/api 管理 API 入口
cmd/worker 调度执行入口
internal/api Gin 路由
internal/executor 本地 handler 执行
internal/jobdef handler 定义与注册
internal/scheduler gocron 调度服务
internal/store/model Mongo 文档模型
internal/store/mongo Mongo 连接与存储实现
pkg/config 环境配置加载
pkg/log 日志封装
```
## 环境变量
默认会按下面顺序读取：
1. `.env`
2. `.env.<APP_ENV|GO_ENV|ENV>`
3. `.env.local`
外部进程环境变量优先级更高，不会被文件覆盖。
关键变量：
```env
HTTP_PORT=16811
WORKER_HTTP_PORT=16812
MONGO_HOST=47.237.103.4
MONGO_PORT=27017
MONGO_USERNAME=
MONGO_PASSWORD=
MONGO_AUTHSOURCE=admin
MONGO_DATABASE=openim_v3
SCHEDULER_MONGO_DATABASE=scheduler_center
```
## 启动方式
### 一条命令启动后端
```bash
cd /Users/vet/all/workspace414/scheduler-backend
./start.sh
```
这个脚本会同时启动：
- `api`
- `worker`
并把运行信息写到：
- `logs/api.log`
- `logs/worker.log`
PID 文件会写到：
- `run/api.pid`
- `run/worker.pid`
停止方式：
```bash
cd /Users/vet/all/workspace414/scheduler-backend
./stop.sh
```
### 启动 API
```bash
cd /Users/vet/all/workspace414/scheduler-backend
GOWORK=off go run ./cmd/api
```
或直接运行：
```bash
./scripts/start-api.sh
```
默认监听：
```text
http://127.0.0.1:16811
```
### 启动 worker
```bash
cd /Users/vet/all/workspace414/scheduler-backend
GOWORK=off go run ./cmd/worker
```
或直接运行：
```bash
./scripts/start-worker.sh
```
worker 会：
- 连接 Mongo
- 暴露健康检查端口 `http://127.0.0.1:16812/healthz`
- 加载启用任务
- 跳过 `manual`
- 注册 `cron` 和 `interval`
- 到点写执行记录并执行本地 Go handler
## 初始化示例任务
会向 `scheduler_center.job_configs` 写入一条示例任务：
- `name=sample-job`
- `handlerKey=sample-handler`
- `enabled=false`
- `scheduleType=manual`
- `defaultParams={"limit":10,"dryRun":true}`
运行方式：
```bash
cd /Users/vet/all/workspace414/scheduler-backend
GOWORK=off go run ./cmd/tools/seed-sample-job
```
或直接运行：
```bash
./scripts/seed-sample-job.sh
```
## 测试与构建
```bash
cd /Users/vet/all/workspace414/scheduler-backend
GOWORK=off go test ./pkg/config ./internal/jobdef ./internal/store/model ./internal/executor -v
GOWORK=off go build ./cmd/api ./cmd/worker ./cmd/tools/seed-sample-job
```
## 当前限制
- 只支持本地 Go handler
- `interval` 依赖 `time.ParseDuration`，例如 `30s`、`5m`、`1h`
- 没有审批流
- 没有分布式多 worker 协调
- 执行记录详情页接口还没单独拆出

BIN
api Executable file
View File

Binary file not shown.

72
cmd/api/main.go Normal file
View File

@@ -0,0 +1,72 @@
package main
import (
"context"
"fmt"
"os"
"os/signal"
"syscall"
"time"
"scheduler-backend/internal/api"
"scheduler-backend/internal/executor"
"scheduler-backend/internal/jobdef"
storemongo "scheduler-backend/internal/store/mongo"
"scheduler-backend/pkg/config"
"scheduler-backend/pkg/log"
)
func main() {
cfg := config.Load()
logger := log.New()
addr := fmt.Sprintf(":%d", cfg.HTTPPort)
registry := jobdef.NewRegistry(
jobdef.SampleHandler{},
jobdef.URLRewriteHandler{},
jobdef.S3MigrateHandler{},
)
rootCtx := context.Background()
databases, err := storemongo.Connect(rootCtx, cfg)
if err != nil {
logger.Error("scheduler api mongo connect failed", "error", err)
os.Exit(1)
}
jobConfigStore := storemongo.NewJobConfigStore(databases.MetaDB)
if err := jobdef.SyncJobConfigs(rootCtx, "job-config-list", jobConfigStore, logger); err != nil {
logger.Error("sync job configs failed", "error", err)
os.Exit(1)
}
execSvc := executor.NewService(registry, jobdef.Runtime{
Config: cfg,
Logger: logger,
MetaDB: databases.MetaDB,
BusinessDB: databases.BusinessDB,
})
router := api.NewRouter(api.RouterDeps{
Registry: registry,
JobConfigStore: jobConfigStore,
ExecutionStore: storemongo.NewJobExecutionStore(databases.MetaDB),
ProfileStore: storemongo.NewAdminProfileStore(databases.MetaDB),
AdminUserStore: storemongo.NewAdminUserStore(databases.MetaDB),
ConfigStore: storemongo.NewSystemConfigStore(databases.MetaDB),
Executor: execSvc,
})
logger.Info("scheduler api starting", "addr", addr)
go func() {
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
<-stop
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = databases.Client.Disconnect(ctx)
os.Exit(0)
}()
if err := router.Run(addr); err != nil {
logger.Error("scheduler api stopped", "error", err)
}
}

65
cmd/tools/seed-sample-job/main.go Normal file
View File

@@ -0,0 +1,65 @@
package main
import (
"context"
"fmt"
"os"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
storemongo "scheduler-backend/internal/store/mongo"
"scheduler-backend/pkg/config"
"scheduler-backend/pkg/log"
)
func main() {
cfg := config.Load()
logger := log.New()
databases, err := storemongo.Connect(context.Background(), cfg)
if err != nil {
logger.Error("seed sample job mongo connect failed", "error", err)
os.Exit(1)
}
defer func() {
_ = databases.Client.Disconnect(context.Background())
}()
now := time.Now()
doc := bson.M{
"name": "sample-job",
"handlerKey": "sample-handler",
"enabled": false,
"scheduleType": "manual",
"scheduleValue": "",
"defaultParams": `{"limit":10,"dryRun":true}`,
"lastStatus": "idle",
"updatedAt": now,
}
result, err := databases.MetaDB.Collection("job_configs").UpdateOne(
context.Background(),
bson.M{"name": "sample-job"},
bson.M{
"$set": doc,
"$setOnInsert": bson.M{"createdAt": now},
},
options.Update().SetUpsert(true),
)
if err != nil {
logger.Error("seed sample job failed", "error", err)
os.Exit(1)
}
logger.Info(
"seed sample job completed",
"matched",
result.MatchedCount,
"modified",
result.ModifiedCount,
"upserted",
fmt.Sprintf("%v", result.UpsertedID),
)
}

105
cmd/worker/main.go Normal file
View File

@@ -0,0 +1,105 @@
package main
import (
"context"
"fmt"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"scheduler-backend/internal/executor"
"scheduler-backend/internal/jobdef"
"scheduler-backend/internal/scheduler"
storemongo "scheduler-backend/internal/store/mongo"
"scheduler-backend/pkg/config"
"scheduler-backend/pkg/log"
"github.com/go-co-op/gocron/v2"
)
func main() {
cfg := config.Load()
logger := log.New()
registry := jobdef.NewRegistry(
jobdef.SampleHandler{},
jobdef.URLRewriteHandler{},
jobdef.S3MigrateHandler{},
)
rootCtx := context.Background()
databases, err := storemongo.Connect(rootCtx, cfg)
if err != nil {
logger.Error("scheduler worker mongo connect failed", "error", err)
os.Exit(1)
}
jobConfigStore := storemongo.NewJobConfigStore(databases.MetaDB)
if err := jobdef.SyncJobConfigs(rootCtx, "job-config-list", jobConfigStore, logger); err != nil {
logger.Error("sync job configs failed", "error", err)
os.Exit(1)
}
runtime := jobdef.Runtime{
Config: cfg,
Logger: logger,
MetaDB: databases.MetaDB,
BusinessDB: databases.BusinessDB,
}
execSvc := executor.NewService(registry, runtime)
g, err := gocron.NewScheduler()
if err != nil {
logger.Error("scheduler worker init scheduler failed", "error", err)
os.Exit(1)
}
svc := scheduler.NewService(
g,
jobConfigStore,
storemongo.NewJobExecutionStore(databases.MetaDB),
execSvc,
runtime,
)
healthServer := &http.Server{
Addr: fmt.Sprintf(":%d", cfg.WorkerHTTPPort),
ReadHeaderTimeout: 5 * time.Second,
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/healthz", "/readyz":
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
default:
w.WriteHeader(http.StatusNotFound)
}
}),
}
go func() {
logger.Info("scheduler worker health server starting", "addr", healthServer.Addr)
if err := healthServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
logger.Error("scheduler worker health server stopped", "error", err)
}
}()
logger.Info("scheduler worker starting", "db", cfg.SchedulerMongoDatabase, "healthAddr", healthServer.Addr)
if err := svc.RegisterEnabledJobs(rootCtx); err != nil {
logger.Error("scheduler worker register jobs failed", "error", err)
os.Exit(1)
}
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
<-stop
if err := svc.Shutdown(); err != nil {
logger.Error("scheduler worker shutdown failed", "error", err)
}
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := healthServer.Shutdown(shutdownCtx); err != nil {
logger.Error("scheduler worker health server shutdown failed", "error", err)
}
if err := databases.Client.Disconnect(context.Background()); err != nil {
logger.Error("scheduler worker mongo disconnect failed", "error", err)
}
}

66
go.mod Normal file
View File

@@ -0,0 +1,66 @@
module scheduler-backend
go 1.24
require (
github.com/aws/aws-sdk-go-v2 v1.41.7
github.com/aws/aws-sdk-go-v2/credentials v1.19.17
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
github.com/gin-gonic/gin v1.10.0
github.com/go-co-op/gocron/v2 v2.14.2
github.com/redis/go-redis/v9 v9.19.0
go.mongodb.org/mongo-driver v1.17.6
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
github.com/aws/smithy-go v1.25.1 // indirect
github.com/bytedance/sonic v1.11.6 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.20.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jonboulle/clockwork v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/text v0.17.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

186
go.sum Normal file
View File

@@ -0,0 +1,186 @@
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
github.com/aws/aws-sdk-go-v2/credentials v1.19.17 h1:gP2nkGsS+KMvF/jfFz2Vv2qiiOqWKyPACSzPsqHgoW8=
github.com/aws/aws-sdk-go-v2/credentials v1.19.17/go.mod h1:Bsew3S/moG5iT77giPj1q8wb/s0RE5/QfH+ASjYtuQc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
github.com/go-co-op/gocron/v2 v2.14.2 h1:S6CbI7MVfD3S/aPJNLoSg2YcGyEqzEMwUopDejuT4Oc=
github.com/go-co-op/gocron/v2 v2.14.2/go.mod h1:ZF70ZwEqz0OO4RBXE1sNxnANy/zvwLcattWEFsqpKig=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k=
github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY=
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

1031
internal/api/router.go Normal file
View File

File diff suppressed because it is too large Load Diff

33
internal/executor/service.go Normal file
View File

@@ -0,0 +1,33 @@
package executor
import (
"context"
"fmt"
"scheduler-backend/internal/jobdef"
)
type Service struct {
registry *jobdef.Registry
runtime jobdef.Runtime
}
func NewService(registry *jobdef.Registry, runtime jobdef.Runtime) *Service {
return &Service{
registry: registry,
runtime: runtime,
}
}
func (s *Service) Execute(ctx context.Context, handlerKey string, req jobdef.ExecuteRequest) error {
handler, ok := s.registry.Get(handlerKey)
if !ok {
return fmt.Errorf("handler %s not found", handlerKey)
}
if req.LogCollector == nil {
req.LogCollector = jobdef.NewLogCollector()
}
return handler.Run(ctx, s.runtime, req)
}

28
internal/executor/service_test.go Normal file
View File

@@ -0,0 +1,28 @@
package executor
import (
"context"
"testing"
"scheduler-backend/internal/jobdef"
"scheduler-backend/pkg/log"
)
func TestExecuteRunsRegisteredHandler(t *testing.T) {
registry := jobdef.NewRegistry(jobdef.SampleHandler{})
service := NewService(
registry,
jobdef.Runtime{
Logger: log.New(),
},
)
err := service.Execute(context.Background(), "sample-handler", jobdef.ExecuteRequest{
ExecutionID: "exec-1",
JobID: "job-1",
TriggerType: "manual",
})
if err != nil {
t.Fatalf("Execute() error = %v", err)
}
}

103
internal/jobdef/handler.go Normal file
View File

@@ -0,0 +1,103 @@
package jobdef
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"go.mongodb.org/mongo-driver/mongo"
"scheduler-backend/pkg/config"
"scheduler-backend/pkg/log"
)
const logTailLines = 2000
type LogCollector struct {
mu sync.Mutex
lines []string
}
func NewLogCollector() *LogCollector {
return &LogCollector{}
}
func (c *LogCollector) Appendf(format string, args ...any) {
if c == nil {
return
}
c.mu.Lock()
c.lines = append(c.lines, fmt.Sprintf(format, args...))
c.mu.Unlock()
}
func (c *LogCollector) String() string {
if c == nil {
return ""
}
c.mu.Lock()
defer c.mu.Unlock()
return strings.Join(c.lines, "\n")
}
type FlushResult struct {
LogText string
LogFile string
}
func (c *LogCollector) Flush(executionID string) FlushResult {
if c == nil {
return FlushResult{}
}
c.mu.Lock()
lines := c.lines
c.mu.Unlock()
if len(lines) == 0 {
return FlushResult{}
}
if len(lines) <= logTailLines {
return FlushResult{LogText: strings.Join(lines, "\n")}
}
filename := executionID + ".log"
dir := filepath.Join("logs", "joblog")
_ = os.MkdirAll(dir, 0o755)
fullPath := filepath.Join(dir, filename)
_ = os.WriteFile(fullPath, []byte(strings.Join(lines, "\n")+"\n"), 0o644)
tail := lines[len(lines)-logTailLines:]
header := fmt.Sprintf("[完整日志: %s (%d行)]", filename, len(lines))
return FlushResult{
LogText: header + "\n" + strings.Join(tail, "\n"),
LogFile: filename,
}
}
type ExecuteRequest struct {
ExecutionID string `json:"executionID"`
JobID string `json:"jobID"`
TriggerType string `json:"triggerType"`
Params json.RawMessage `json:"params"`
LogCollector *LogCollector `json:"-"`
}
type Runtime struct {
Config config.Config
Logger log.Logger
MetaDB *mongo.Database
BusinessDB *mongo.Database
}
type Handler interface {
Key() string
Name() string
Description() string
Run(ctx context.Context, runtime Runtime, req ExecuteRequest) error
}

62
internal/jobdef/loader.go Normal file
View File

@@ -0,0 +1,62 @@
package jobdef
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"scheduler-backend/internal/store/model"
"scheduler-backend/pkg/log"
)
type JobConfigUpserter interface {
UpsertByHandlerKey(ctx context.Context, item *model.JobConfig) error
}
type configFile struct {
Name string `json:"name"`
HandlerKey string `json:"handlerKey"`
ScheduleType string `json:"scheduleType"`
ScheduleValue string `json:"scheduleValue"`
DefaultParams string `json:"defaultParams"`
}
func SyncJobConfigs(ctx context.Context, dir string, store JobConfigUpserter, logger log.Logger) error {
matches, err := filepath.Glob(filepath.Join(dir, "*.json"))
if err != nil {
return fmt.Errorf("glob job config files: %w", err)
}
if len(matches) == 0 {
logger.Info("no job config files found", "dir", dir)
return nil
}
for _, path := range matches {
data, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("read %s: %w", path, err)
}
var cfg configFile
if err := json.Unmarshal(data, &cfg); err != nil {
return fmt.Errorf("parse %s: %w", path, err)
}
if cfg.HandlerKey == "" {
return fmt.Errorf("%s: handlerKey is required", path)
}
item := &model.JobConfig{
Name: cfg.Name,
HandlerKey: cfg.HandlerKey,
ScheduleType: cfg.ScheduleType,
ScheduleValue: cfg.ScheduleValue,
DefaultParams: cfg.DefaultParams,
}
if err := store.UpsertByHandlerKey(ctx, item); err != nil {
return fmt.Errorf("upsert %s: %w", cfg.HandlerKey, err)
}
logger.Info("synced job config", "handlerKey", cfg.HandlerKey, "name", cfg.Name, "file", filepath.Base(path))
}
return nil
}

28
internal/jobdef/registry.go Normal file
View File

@@ -0,0 +1,28 @@
package jobdef
type Registry struct {
items map[string]Handler
}
func NewRegistry(handlers ...Handler) *Registry {
items := make(map[string]Handler, len(handlers))
for _, handler := range handlers {
items[handler.Key()] = handler
}
return &Registry{items: items}
}
func (r *Registry) Get(key string) (Handler, bool) {
handler, ok := r.items[key]
return handler, ok
}
func (r *Registry) List() []Handler {
result := make([]Handler, 0, len(r.items))
for _, handler := range r.items {
result = append(result, handler)
}
return result
}

16
internal/jobdef/registry_test.go Normal file
View File

@@ -0,0 +1,16 @@
package jobdef
import "testing"
func TestRegistryGetReturnsRegisteredHandler(t *testing.T) {
registry := NewRegistry(SampleHandler{})
handler, ok := registry.Get("sample-handler")
if !ok {
t.Fatal("expected registered handler")
}
if handler.Key() != "sample-handler" {
t.Fatalf("handler key = %q, want sample-handler", handler.Key())
}
}

44
internal/jobdef/s3_migrate_handler.go Normal file
View File

@@ -0,0 +1,44 @@
package jobdef
import (
"context"
"encoding/json"
"fmt"
"scheduler-backend/internal/s3migrate"
)
type S3MigrateHandler struct{}
func (S3MigrateHandler) Key() string {
return "s3-migrate"
}
func (S3MigrateHandler) Name() string {
return "S3 Migrate"
}
func (S3MigrateHandler) Description() string {
return "Migrate objects between S3-compatible storage engines (MinIO, AWS S3, etc.). Streams files via presigned URLs and updates MongoDB engine records."
}
func (S3MigrateHandler) Run(ctx context.Context, runtime Runtime, req ExecuteRequest) error {
var params s3migrate.Params
if err := json.Unmarshal(req.Params, &params); err != nil {
return fmt.Errorf("parse params: %w", err)
}
logf := func(msg string, args ...any) {
line := fmt.Sprintf(msg, args...)
req.LogCollector.Appendf("%s", line)
runtime.Logger.Info(line, "jobID", req.JobID, "executionID", req.ExecutionID)
}
if err := s3migrate.Run(ctx, runtime.BusinessDB, params, logf); err != nil {
logf("s3-migrate failed: %v", err)
return fmt.Errorf("s3-migrate failed: %w", err)
}
logf("s3-migrate finished")
return nil
}

31
internal/jobdef/sample_handler.go Normal file
View File

@@ -0,0 +1,31 @@
package jobdef
import "context"
type SampleHandler struct{}
func (SampleHandler) Key() string {
return "sample-handler"
}
func (SampleHandler) Name() string {
return "Sample Handler"
}
func (SampleHandler) Description() string {
return "Writes a sample startup log for scheduler plumbing."
}
func (SampleHandler) Run(ctx context.Context, runtime Runtime, req ExecuteRequest) error {
req.LogCollector.Appendf("sample handler executed: jobID=%s executionID=%s triggerType=%s", req.JobID, req.ExecutionID, req.TriggerType)
runtime.Logger.Info(
"sample handler executed",
"jobID",
req.JobID,
"executionID",
req.ExecutionID,
"triggerType",
req.TriggerType,
)
return nil
}

49
internal/jobdef/url_rewrite_handler.go Normal file
View File

@@ -0,0 +1,49 @@
package jobdef
import (
"context"
"encoding/json"
"fmt"
"scheduler-backend/internal/urlrewrite"
)
type URLRewriteHandler struct{}
func (URLRewriteHandler) Key() string {
return "url-rewrite"
}
func (URLRewriteHandler) Name() string {
return "URL Rewrite"
}
func (URLRewriteHandler) Description() string {
return "Batch-rewrite URL prefixes in MongoDB (user avatars, message media, favorites). Supports dry-run, apply, rollback, and cache invalidation."
}
func (URLRewriteHandler) Run(ctx context.Context, runtime Runtime, req ExecuteRequest) error {
var params urlrewrite.Params
if err := json.Unmarshal(req.Params, &params); err != nil {
return fmt.Errorf("parse params: %w", err)
}
logf := func(msg string, args ...any) {
line := fmt.Sprintf(msg, args...)
req.LogCollector.Appendf("%s", line)
runtime.Logger.Info(line, "jobID", req.JobID, "executionID", req.ExecutionID)
}
redisCfg := urlrewrite.RedisConfig{
Addr: runtime.Config.RedisAddr,
Password: runtime.Config.RedisPassword,
}
batchID, err := urlrewrite.Run(ctx, runtime.BusinessDB, params, redisCfg, logf)
if err != nil {
logf("url-rewrite failed: %v", err)
return fmt.Errorf("url-rewrite %s failed (batch_id=%s): %w", params.Mode, batchID, err)
}
logf("url-rewrite finished, mode=%s batch_id=%s", params.Mode, batchID)
return nil
}

220
internal/s3migrate/runner.go Normal file
View File

@@ -0,0 +1,220 @@
package s3migrate
import (
"context"
"fmt"
"io"
"net/http"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
collectionName = "s3"
presignExpiry = time.Hour
)
type Params struct {
SourceEngine string `json:"sourceEngine"`
SourceEndpoint string `json:"sourceEndpoint"`
SourceBucket string `json:"sourceBucket"`
SourceAccessKey string `json:"sourceAccessKey"`
SourceSecretKey string `json:"sourceSecretKey"`
SourceRegion string `json:"sourceRegion"`
DestEngine string `json:"destEngine"`
DestEndpoint string `json:"destEndpoint"`
DestBucket string `json:"destBucket"`
DestAccessKey string `json:"destAccessKey"`
DestSecretKey string `json:"destSecretKey"`
DestRegion string `json:"destRegion"`
}
type objectRecord struct {
Name string `bson:"name"`
Engine string `bson:"engine"`
Key string `bson:"key"`
Size int64 `bson:"size"`
ContentType string `bson:"content_type"`
}
type LogFunc func(msg string, args ...any)
func Run(ctx context.Context, db *mongo.Database, params Params, logf LogFunc) error {
if err := validateParams(params); err != nil {
return err
}
if params.SourceRegion == "" {
params.SourceRegion = "us-east-1"
}
if params.DestRegion == "" {
params.DestRegion = "us-east-1"
}
srcPresign := newPresignClient(params.SourceEndpoint, params.SourceAccessKey, params.SourceSecretKey, params.SourceRegion)
dstPresign := newPresignClient(params.DestEndpoint, params.DestAccessKey, params.DestSecretKey, params.DestRegion)
coll := db.Collection(collectionName)
count, err := coll.CountDocuments(ctx, bson.M{"engine": params.SourceEngine})
if err != nil {
return fmt.Errorf("count source objects: %w", err)
}
logf("source engine=%s count=%d", params.SourceEngine, count)
if count == 0 {
logf("no objects to migrate")
return nil
}
cursor, err := coll.Find(ctx, bson.M{"engine": params.SourceEngine}, options.Find().SetBatchSize(100))
if err != nil {
return fmt.Errorf("find source objects: %w", err)
}
defer cursor.Close(ctx)
var migrated, skipped, failed int64
for cursor.Next(ctx) {
var obj objectRecord
if err := cursor.Decode(&obj); err != nil {
logf("decode error: %v", err)
failed++
continue
}
exists, err := objectExistsInDest(ctx, coll, params.DestEngine, obj.Name)
if err != nil {
logf("check dest error name=%s: %v", obj.Name, err)
failed++
continue
}
if exists {
skipped++
continue
}
if err := copyObject(ctx, srcPresign, params.SourceBucket, dstPresign, params.DestBucket, obj); err != nil {
logf("copy error name=%s: %v", obj.Name, err)
failed++
continue
}
if _, err := coll.UpdateOne(ctx,
bson.M{"engine": params.SourceEngine, "name": obj.Name},
bson.M{"$set": bson.M{"engine": params.DestEngine}},
); err != nil {
logf("update engine error name=%s: %v", obj.Name, err)
failed++
continue
}
migrated++
if migrated%100 == 0 {
logf("progress: migrated=%d skipped=%d failed=%d", migrated, skipped, failed)
}
}
logf("complete: migrated=%d skipped=%d failed=%d total=%d", migrated, skipped, failed, count)
return nil
}
func validateParams(p Params) error {
if p.SourceEngine == "" {
return fmt.Errorf("sourceEngine is required")
}
if p.DestEngine == "" {
return fmt.Errorf("destEngine is required")
}
if p.SourceEngine == p.DestEngine {
return fmt.Errorf("sourceEngine and destEngine must be different")
}
if p.SourceEndpoint == "" || p.SourceBucket == "" || p.SourceAccessKey == "" || p.SourceSecretKey == "" {
return fmt.Errorf("source S3 config (endpoint, bucket, accessKey, secretKey) is required")
}
if p.DestEndpoint == "" || p.DestBucket == "" || p.DestAccessKey == "" || p.DestSecretKey == "" {
return fmt.Errorf("dest S3 config (endpoint, bucket, accessKey, secretKey) is required")
}
return nil
}
func newPresignClient(endpoint, accessKey, secretKey, region string) *s3.PresignClient {
cfg := aws.Config{
Region: region,
Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
BaseEndpoint: &endpoint,
}
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = true
})
return s3.NewPresignClient(client)
}
func objectExistsInDest(ctx context.Context, coll *mongo.Collection, destEngine, name string) (bool, error) {
count, err := coll.CountDocuments(ctx, bson.M{"engine": destEngine, "name": name}, options.Count().SetLimit(1))
if err != nil {
return false, err
}
return count > 0, nil
}
func copyObject(ctx context.Context, srcPresign *s3.PresignClient, srcBucket string, dstPresign *s3.PresignClient, dstBucket string, obj objectRecord) error {
getReq, err := srcPresign.PresignGetObject(ctx, &s3.GetObjectInput{
Bucket: &srcBucket,
Key: &obj.Key,
}, s3.WithPresignExpires(presignExpiry))
if err != nil {
return fmt.Errorf("presign get: %w", err)
}
contentType := obj.ContentType
putReq, err := dstPresign.PresignPutObject(ctx, &s3.PutObjectInput{
Bucket: &dstBucket,
Key: &obj.Key,
ContentType: &contentType,
}, s3.WithPresignExpires(presignExpiry))
if err != nil {
return fmt.Errorf("presign put: %w", err)
}
downloadResp, err := http.Get(getReq.URL)
if err != nil {
return fmt.Errorf("download: %w", err)
}
defer downloadResp.Body.Close()
if downloadResp.StatusCode == http.StatusNotFound {
return fmt.Errorf("source object not found: %s", obj.Key)
}
if downloadResp.StatusCode != http.StatusOK {
return fmt.Errorf("download status: %s", downloadResp.Status)
}
uploadReq, err := http.NewRequestWithContext(ctx, http.MethodPut, putReq.URL, downloadResp.Body)
if err != nil {
return fmt.Errorf("create upload request: %w", err)
}
if downloadResp.ContentLength > 0 {
uploadReq.ContentLength = downloadResp.ContentLength
} else {
uploadReq.ContentLength = obj.Size
}
if contentType != "" {
uploadReq.Header.Set("Content-Type", contentType)
}
uploadResp, err := http.DefaultClient.Do(uploadReq)
if err != nil {
return fmt.Errorf("upload: %w", err)
}
defer func() { _, _ = io.Copy(io.Discard, uploadResp.Body); uploadResp.Body.Close() }()
if uploadResp.StatusCode != http.StatusOK {
return fmt.Errorf("upload status: %s", uploadResp.Status)
}
return nil
}

171
internal/scheduler/service.go Normal file
View File

@@ -0,0 +1,171 @@
package scheduler
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/go-co-op/gocron/v2"
"go.mongodb.org/mongo-driver/bson/primitive"
"scheduler-backend/internal/executor"
"scheduler-backend/internal/jobdef"
"scheduler-backend/internal/store/model"
)
type JobConfigStore interface {
ListEnabled(ctx context.Context) ([]model.JobConfig, error)
UpdateRunState(ctx context.Context, id primitive.ObjectID, status string, lastRunAt *time.Time) error
UpdateNextRunAt(ctx context.Context, id primitive.ObjectID, nextRunAt *time.Time) error
}
type JobExecutionStore interface {
Create(ctx context.Context, item *model.JobExecution) error
UpdateResult(ctx context.Context, item *model.JobExecution) error
}
type Service struct {
scheduler gocron.Scheduler
jobConfigStore JobConfigStore
executionStore JobExecutionStore
executor *executor.Service
runtime jobdef.Runtime
}
func NewService(
scheduler gocron.Scheduler,
jobConfigStore JobConfigStore,
executionStore JobExecutionStore,
executor *executor.Service,
runtime jobdef.Runtime,
) *Service {
return &Service{
scheduler: scheduler,
jobConfigStore: jobConfigStore,
executionStore: executionStore,
executor: executor,
runtime: runtime,
}
}
func (s *Service) RegisterEnabledJobs(ctx context.Context) error {
items, err := s.jobConfigStore.ListEnabled(ctx)
if err != nil {
return err
}
for _, item := range items {
if err := s.registerJob(ctx, item); err != nil {
return err
}
}
s.scheduler.Start()
return nil
}
func (s *Service) Shutdown() error {
return s.scheduler.Shutdown()
}
func (s *Service) registerJob(ctx context.Context, item model.JobConfig) error {
switch item.ScheduleType {
case "manual":
return s.jobConfigStore.UpdateNextRunAt(ctx, item.ID, nil)
case "cron":
job, err := s.scheduler.NewJob(
gocron.CronJob(item.ScheduleValue, false),
gocron.NewTask(func() {
s.runScheduledJob(context.Background(), item)
}),
)
if err != nil {
return fmt.Errorf("register cron job %s: %w", item.Name, err)
}
next, err := job.NextRun()
if err != nil {
return err
}
return s.jobConfigStore.UpdateNextRunAt(ctx, item.ID, &next)
case "interval":
duration, err := time.ParseDuration(item.ScheduleValue)
if err != nil {
return fmt.Errorf("parse interval for %s: %w", item.Name, err)
}
job, err := s.scheduler.NewJob(
gocron.DurationJob(duration),
gocron.NewTask(func() {
s.runScheduledJob(context.Background(), item)
}),
)
if err != nil {
return fmt.Errorf("register interval job %s: %w", item.Name, err)
}
next, err := job.NextRun()
if err != nil {
return err
}
return s.jobConfigStore.UpdateNextRunAt(ctx, item.ID, &next)
default:
return fmt.Errorf("unsupported schedule type %s", item.ScheduleType)
}
}
func (s *Service) runScheduledJob(ctx context.Context, job model.JobConfig) {
now := time.Now()
params := json.RawMessage(job.DefaultParams)
record := model.JobExecution{
JobConfigID: job.ID,
TriggerType: "schedule",
Status: "running",
ParamsSnapshot: string(params),
StartedAt: &now,
CreatedAt: now,
}
if err := s.executionStore.Create(ctx, &record); err != nil {
s.runtime.Logger.Error("create scheduled execution failed", "jobID", job.ID.Hex(), "error", err)
return
}
execReq := jobdef.ExecuteRequest{
ExecutionID: record.ID.Hex(),
JobID: job.ID.Hex(),
TriggerType: "schedule",
Params: params,
LogCollector: jobdef.NewLogCollector(),
}
runErr := s.executor.Execute(ctx, job.HandlerKey, execReq)
finishedAt := time.Now()
record.FinishedAt = &finishedAt
record.DurationMs = finishedAt.Sub(now).Milliseconds()
flushed := execReq.LogCollector.Flush(record.ID.Hex())
if runErr != nil {
record.Status = "failed"
record.ErrorMessage = runErr.Error()
record.ResultSummary = "scheduled execution failed"
if flushed.LogText != "" {
record.LogText = flushed.LogText
} else {
record.LogText = runErr.Error()
}
record.LogFile = flushed.LogFile
_ = s.jobConfigStore.UpdateRunState(ctx, job.ID, "failed", &finishedAt)
} else {
record.Status = "success"
record.ResultSummary = "scheduled execution succeeded"
if flushed.LogText != "" {
record.LogText = flushed.LogText
} else {
record.LogText = "handler executed successfully"
}
record.LogFile = flushed.LogFile
_ = s.jobConfigStore.UpdateRunState(ctx, job.ID, "success", &finishedAt)
}
if err := s.executionStore.UpdateResult(ctx, &record); err != nil {
s.runtime.Logger.Error("update scheduled execution failed", "executionID", record.ID.Hex(), "error", err)
}
}

18
internal/store/model/admin_profile.go Normal file
View File

@@ -0,0 +1,18 @@
package model
import (
"time"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type AdminProfile struct {
ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
Account string `bson:"account" json:"account"`
Nickname string `bson:"nickname" json:"nickname"`
Avatar string `bson:"avatar" json:"avatar"`
Remark string `bson:"remark" json:"remark"`
PasswordHash string `bson:"passwordHash" json:"-"`
CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"`
}

19
internal/store/model/admin_user.go Normal file
View File

@@ -0,0 +1,19 @@
package model
import (
"time"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type AdminUser struct {
ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
Account string `bson:"account" json:"account"`
Nickname string `bson:"nickname" json:"nickname"`
Status string `bson:"status" json:"status"`
Role string `bson:"role" json:"role"`
Remark string `bson:"remark" json:"remark"`
PasswordHash string `bson:"passwordHash" json:"-"`
CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"`
}

33
internal/store/model/job_config.go Normal file
View File

@@ -0,0 +1,33 @@
package model
import (
"time"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type JobConfig struct {
ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
Name string `bson:"name" json:"name"`
HandlerKey string `bson:"handlerKey" json:"handlerKey"`
Enabled bool `bson:"enabled" json:"enabled"`
ScheduleType string `bson:"scheduleType" json:"scheduleType"`
ScheduleValue string `bson:"scheduleValue" json:"scheduleValue"`
DefaultParams string `bson:"defaultParams" json:"defaultParams"`
LastStatus string `bson:"lastStatus" json:"lastStatus"`
LastRunAt *time.Time `bson:"lastRunAt,omitempty" json:"lastRunAt,omitempty"`
NextRunAt *time.Time `bson:"nextRunAt,omitempty" json:"nextRunAt,omitempty"`
CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"`
}
func NewJobConfig(name, handlerKey string) JobConfig {
now := time.Now()
return JobConfig{
Name: name,
HandlerKey: handlerKey,
LastStatus: "idle",
CreatedAt: now,
UpdatedAt: now,
}
}

10
internal/store/model/job_config_test.go Normal file
View File

@@ -0,0 +1,10 @@
package model
import "testing"
func TestJobConfigDefaultsToIdleStatus(t *testing.T) {
job := NewJobConfig("daily-check", "daily-check")
if job.LastStatus != "idle" {
t.Fatalf("LastStatus = %q, want idle", job.LastStatus)
}
}

23
internal/store/model/job_execution.go Normal file
View File

@@ -0,0 +1,23 @@
package model
import (
"time"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type JobExecution struct {
ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
JobConfigID primitive.ObjectID `bson:"jobConfigId" json:"jobConfigId"`
TriggerType string `bson:"triggerType" json:"triggerType"`
Status string `bson:"status" json:"status"`
ParamsSnapshot string `bson:"paramsSnapshot" json:"paramsSnapshot"`
StartedAt *time.Time `bson:"startedAt,omitempty" json:"startedAt,omitempty"`
FinishedAt *time.Time `bson:"finishedAt,omitempty" json:"finishedAt,omitempty"`
DurationMs int64 `bson:"durationMs" json:"durationMs"`
ResultSummary string `bson:"resultSummary" json:"resultSummary"`
ErrorMessage string `bson:"errorMessage" json:"errorMessage"`
LogText string `bson:"logText" json:"logText"`
LogFile string `bson:"logFile,omitempty" json:"logFile,omitempty"`
CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
}

19
internal/store/model/system_config.go Normal file
View File

@@ -0,0 +1,19 @@
package model
import (
"time"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type SystemConfig struct {
ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
Key string `bson:"key" json:"key"`
Title string `bson:"title" json:"title"`
Value string `bson:"value" json:"value"`
ValueType string `bson:"valueType" json:"valueType"`
Description string `bson:"description" json:"description"`
Enabled bool `bson:"enabled" json:"enabled"`
CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"`
}

125
internal/store/mongo/admin_profile_store.go Normal file
View File

@@ -0,0 +1,125 @@
package mongo
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
mongodriver "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"scheduler-backend/internal/store/model"
)
const defaultAdminPassword = "admin123"
type AdminProfileStore struct {
collection *mongodriver.Collection
}
func NewAdminProfileStore(db *mongodriver.Database) *AdminProfileStore {
return &AdminProfileStore{
collection: db.Collection("admin_profiles"),
}
}
func (s *AdminProfileStore) Get(ctx context.Context) (*model.AdminProfile, error) {
findCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
var item model.AdminProfile
err := s.collection.FindOne(findCtx, bson.D{}, options.FindOne().SetSort(bson.D{{Key: "updatedAt", Value: -1}})).Decode(&item)
if err == nil {
return &item, nil
}
if !errors.Is(err, mongodriver.ErrNoDocuments) {
return nil, err
}
now := time.Now()
item = model.AdminProfile{
ID: primitive.NewObjectID(),
Account: "admin",
Nickname: "调度管理员",
Avatar: "",
Remark: "default admin profile",
PasswordHash: hashPassword(defaultAdminPassword),
CreatedAt: now,
UpdatedAt: now,
}
if _, err := s.collection.InsertOne(findCtx, item); err != nil {
return nil, err
}
return &item, nil
}
func (s *AdminProfileStore) Upsert(ctx context.Context, profile *model.AdminProfile) error {
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
now := time.Now()
profile.UpdatedAt = now
if profile.CreatedAt.IsZero() {
profile.CreatedAt = now
}
if profile.ID.IsZero() {
current, err := s.Get(ctx)
if err != nil {
return err
}
profile.ID = current.ID
if profile.PasswordHash == "" {
profile.PasswordHash = current.PasswordHash
}
if profile.CreatedAt.IsZero() {
profile.CreatedAt = current.CreatedAt
}
}
_, err := s.collection.UpdateByID(updateCtx, profile.ID, bson.M{
"$set": bson.M{
"account": profile.Account,
"nickname": profile.Nickname,
"avatar": profile.Avatar,
"remark": profile.Remark,
"passwordHash": profile.PasswordHash,
"createdAt": profile.CreatedAt,
"updatedAt": profile.UpdatedAt,
},
}, options.Update().SetUpsert(true))
return err
}
func (s *AdminProfileStore) ChangePassword(ctx context.Context, oldPassword, newPassword string) error {
current, err := s.Get(ctx)
if err != nil {
return err
}
if current.PasswordHash != hashPassword(oldPassword) {
return ErrInvalidPassword
}
current.PasswordHash = hashPassword(newPassword)
return s.Upsert(ctx, current)
}
func (s *AdminProfileStore) Authenticate(ctx context.Context, account, password string) (*model.AdminProfile, error) {
current, err := s.Get(ctx)
if err != nil {
return nil, err
}
if current.Account != account || current.PasswordHash != hashPassword(password) {
return nil, ErrInvalidPassword
}
return current, nil
}
func hashPassword(password string) string {
sum := sha256.Sum256([]byte(password))
return hex.EncodeToString(sum[:])
}

152
internal/store/mongo/admin_user_store.go Normal file
View File

@@ -0,0 +1,152 @@
package mongo
import (
"context"
"errors"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
mongodriver "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"scheduler-backend/internal/store/model"
)
type AdminUserStore struct {
collection *mongodriver.Collection
}
func NewAdminUserStore(db *mongodriver.Database) *AdminUserStore {
return &AdminUserStore{
collection: db.Collection("admin_users"),
}
}
func (s *AdminUserStore) List(ctx context.Context) ([]model.AdminUser, error) {
findCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
cursor, err := s.collection.Find(findCtx, bson.D{}, options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}))
if err != nil {
return nil, err
}
defer cursor.Close(findCtx)
items := make([]model.AdminUser, 0)
for cursor.Next(findCtx) {
var item model.AdminUser
if err := cursor.Decode(&item); err != nil {
return nil, err
}
items = append(items, item)
}
if err := cursor.Err(); err != nil {
return nil, err
}
if len(items) == 0 {
defaultAdmin := model.AdminUser{
Account: "admin",
Nickname: "调度管理员",
Status: "active",
Role: "super_admin",
Remark: "default admin user",
PasswordHash: hashPassword(defaultAdminPassword),
}
if err := s.Create(ctx, &defaultAdmin); err != nil {
return nil, err
}
items = append(items, defaultAdmin)
}
return items, nil
}
func (s *AdminUserStore) GetByID(ctx context.Context, id primitive.ObjectID) (*model.AdminUser, error) {
findCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
var item model.AdminUser
err := s.collection.FindOne(findCtx, bson.M{"_id": id}).Decode(&item)
if err != nil {
if errors.Is(err, mongodriver.ErrNoDocuments) {
return nil, nil
}
return nil, err
}
return &item, nil
}
func (s *AdminUserStore) Create(ctx context.Context, item *model.AdminUser) error {
insertCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
now := time.Now()
item.CreatedAt = now
item.UpdatedAt = now
if item.Status == "" {
item.Status = "active"
}
if item.Role == "" {
item.Role = "super_admin"
}
result, err := s.collection.InsertOne(insertCtx, item)
if err != nil {
return err
}
if oid, ok := result.InsertedID.(primitive.ObjectID); ok {
item.ID = oid
}
return nil
}
func (s *AdminUserStore) Update(ctx context.Context, item *model.AdminUser) error {
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
item.UpdatedAt = time.Now()
_, err := s.collection.UpdateByID(updateCtx, item.ID, bson.M{
"$set": bson.M{
"account": item.Account,
"nickname": item.Nickname,
"status": item.Status,
"role": item.Role,
"remark": item.Remark,
"updatedAt": item.UpdatedAt,
},
})
return err
}
func (s *AdminUserStore) ChangePassword(ctx context.Context, id primitive.ObjectID, newPassword string) error {
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
_, err := s.collection.UpdateByID(updateCtx, id, bson.M{
"$set": bson.M{
"passwordHash": hashPassword(newPassword),
"updatedAt": time.Now(),
},
})
return err
}
func (s *AdminUserStore) Authenticate(ctx context.Context, account, password string) (*model.AdminUser, error) {
findCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
var item model.AdminUser
err := s.collection.FindOne(findCtx, bson.M{"account": account}).Decode(&item)
if err != nil {
if errors.Is(err, mongodriver.ErrNoDocuments) {
return nil, ErrInvalidPassword
}
return nil, err
}
if item.Status == "disabled" || item.PasswordHash != hashPassword(password) {
return nil, ErrInvalidPassword
}
return &item, nil
}

37
internal/store/mongo/client.go Normal file
View File

@@ -0,0 +1,37 @@
package mongo
import (
"context"
"time"
mongodriver "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"scheduler-backend/pkg/config"
)
type Databases struct {
Client *mongodriver.Client
MetaDB *mongodriver.Database
BusinessDB *mongodriver.Database
}
func Connect(ctx context.Context, cfg config.Config) (*Databases, error) {
client, err := mongodriver.Connect(ctx, options.Client().ApplyURI(cfg.MongoURI()))
if err != nil {
return nil, err
}
pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if err := client.Ping(pingCtx, nil); err != nil {
return nil, err
}
return &Databases{
Client: client,
MetaDB: client.Database(cfg.SchedulerMongoDatabase),
BusinessDB: client.Database(cfg.BusinessMongoDatabase),
}, nil
}

5
internal/store/mongo/errors.go Normal file
View File

@@ -0,0 +1,5 @@
package mongo
import "errors"
var ErrInvalidPassword = errors.New("invalid password")

222
internal/store/mongo/job_config_store.go Normal file
View File

@@ -0,0 +1,222 @@
package mongo
import (
"context"
"errors"
"time"
"go.mongodb.org/mongo-driver/bson"
mongodriver "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/bson/primitive"
"scheduler-backend/internal/store/model"
)
type JobConfigStore struct {
collection *mongodriver.Collection
}
func NewJobConfigStore(db *mongodriver.Database) *JobConfigStore {
return &JobConfigStore{
collection: db.Collection("job_configs"),
}
}
func (s *JobConfigStore) List(ctx context.Context) ([]model.JobConfig, error) {
findCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
cursor, err := s.collection.Find(
findCtx,
bson.D{},
options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}),
)
if err != nil {
return nil, err
}
defer cursor.Close(findCtx)
items := make([]model.JobConfig, 0)
for cursor.Next(findCtx) {
var item model.JobConfig
if err := cursor.Decode(&item); err != nil {
return nil, err
}
items = append(items, item)
}
if err := cursor.Err(); err != nil {
return nil, err
}
return items, nil
}
func (s *JobConfigStore) ListEnabled(ctx context.Context) ([]model.JobConfig, error) {
findCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
cursor, err := s.collection.Find(
findCtx,
bson.M{"enabled": true},
options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}),
)
if err != nil {
return nil, err
}
defer cursor.Close(findCtx)
items := make([]model.JobConfig, 0)
for cursor.Next(findCtx) {
var item model.JobConfig
if err := cursor.Decode(&item); err != nil {
return nil, err
}
items = append(items, item)
}
if err := cursor.Err(); err != nil {
return nil, err
}
return items, nil
}
func (s *JobConfigStore) GetByID(ctx context.Context, id primitive.ObjectID) (*model.JobConfig, error) {
findCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
var item model.JobConfig
err := s.collection.FindOne(findCtx, bson.M{"_id": id}).Decode(&item)
if err != nil {
if errors.Is(err, mongodriver.ErrNoDocuments) {
return nil, nil
}
return nil, err
}
return &item, nil
}
func (s *JobConfigStore) Create(ctx context.Context, item *model.JobConfig) error {
insertCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
now := time.Now()
item.CreatedAt = now
item.UpdatedAt = now
if item.LastStatus == "" {
item.LastStatus = "idle"
}
result, err := s.collection.InsertOne(insertCtx, item)
if err != nil {
return err
}
if oid, ok := result.InsertedID.(primitive.ObjectID); ok {
item.ID = oid
}
return nil
}
func (s *JobConfigStore) Update(ctx context.Context, item *model.JobConfig) error {
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
item.UpdatedAt = time.Now()
update := bson.M{
"$set": bson.M{
"name": item.Name,
"handlerKey": item.HandlerKey,
"enabled": item.Enabled,
"scheduleType": item.ScheduleType,
"scheduleValue": item.ScheduleValue,
"defaultParams": item.DefaultParams,
"updatedAt": item.UpdatedAt,
},
}
_, err := s.collection.UpdateByID(updateCtx, item.ID, update)
return err
}
func (s *JobConfigStore) ToggleEnabled(ctx context.Context, id primitive.ObjectID, enabled bool) error {
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
_, err := s.collection.UpdateByID(updateCtx, id, bson.M{
"$set": bson.M{
"enabled": enabled,
"updatedAt": time.Now(),
},
})
return err
}
func (s *JobConfigStore) UpdateRunState(ctx context.Context, id primitive.ObjectID, status string, lastRunAt *time.Time) error {
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
set := bson.M{
"lastStatus": status,
"updatedAt": time.Now(),
}
if lastRunAt != nil {
set["lastRunAt"] = *lastRunAt
}
_, err := s.collection.UpdateByID(updateCtx, id, bson.M{
"$set": set,
})
return err
}
func (s *JobConfigStore) UpsertByHandlerKey(ctx context.Context, item *model.JobConfig) error {
upsertCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
now := time.Now()
_, err := s.collection.UpdateOne(
upsertCtx,
bson.M{"handlerKey": item.HandlerKey},
bson.M{
"$set": bson.M{
"name": item.Name,
"handlerKey": item.HandlerKey,
"scheduleType": item.ScheduleType,
"scheduleValue": item.ScheduleValue,
"defaultParams": item.DefaultParams,
"updatedAt": now,
},
"$setOnInsert": bson.M{
"enabled": false,
"lastStatus": "idle",
"createdAt": now,
},
},
options.Update().SetUpsert(true),
)
return err
}
func (s *JobConfigStore) UpdateNextRunAt(ctx context.Context, id primitive.ObjectID, nextRunAt *time.Time) error {
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
set := bson.M{
"updatedAt": time.Now(),
}
if nextRunAt != nil {
set["nextRunAt"] = *nextRunAt
} else {
set["nextRunAt"] = nil
}
_, err := s.collection.UpdateByID(updateCtx, id, bson.M{
"$set": set,
})
return err
}

109
internal/store/mongo/job_execution_store.go Normal file
View File

@@ -0,0 +1,109 @@
package mongo
import (
"context"
"errors"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
mongodriver "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"scheduler-backend/internal/store/model"
)
type JobExecutionStore struct {
collection *mongodriver.Collection
}
func NewJobExecutionStore(db *mongodriver.Database) *JobExecutionStore {
return &JobExecutionStore{
collection: db.Collection("job_executions"),
}
}
func (s *JobExecutionStore) List(ctx context.Context) ([]model.JobExecution, error) {
findCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
cursor, err := s.collection.Find(
findCtx,
bson.D{},
options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}),
)
if err != nil {
return nil, err
}
defer cursor.Close(findCtx)
items := make([]model.JobExecution, 0)
for cursor.Next(findCtx) {
var item model.JobExecution
if err := cursor.Decode(&item); err != nil {
return nil, err
}
items = append(items, item)
}
if err := cursor.Err(); err != nil {
return nil, err
}
return items, nil
}
func (s *JobExecutionStore) GetByID(ctx context.Context, id primitive.ObjectID) (*model.JobExecution, error) {
findCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
var item model.JobExecution
err := s.collection.FindOne(findCtx, bson.M{"_id": id}).Decode(&item)
if err != nil {
if errors.Is(err, mongodriver.ErrNoDocuments) {
return nil, nil
}
return nil, err
}
return &item, nil
}
func (s *JobExecutionStore) Create(ctx context.Context, item *model.JobExecution) error {
insertCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if item.CreatedAt.IsZero() {
item.CreatedAt = time.Now()
}
result, err := s.collection.InsertOne(insertCtx, item)
if err != nil {
return err
}
if oid, ok := result.InsertedID.(primitive.ObjectID); ok {
item.ID = oid
}
return nil
}
func (s *JobExecutionStore) UpdateResult(ctx context.Context, item *model.JobExecution) error {
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
_, err := s.collection.UpdateByID(updateCtx, item.ID, bson.M{
"$set": bson.M{
"status": item.Status,
"startedAt": item.StartedAt,
"finishedAt": item.FinishedAt,
"durationMs": item.DurationMs,
"resultSummary": item.ResultSummary,
"errorMessage": item.ErrorMessage,
"logText": item.LogText,
"logFile": item.LogFile,
},
})
return err
}

113
internal/store/mongo/system_config_store.go Normal file
View File

@@ -0,0 +1,113 @@
package mongo
import (
"context"
"errors"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
mongodriver "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"scheduler-backend/internal/store/model"
)
type SystemConfigStore struct {
collection *mongodriver.Collection
}
func NewSystemConfigStore(db *mongodriver.Database) *SystemConfigStore {
return &SystemConfigStore{
collection: db.Collection("system_configs"),
}
}
func (s *SystemConfigStore) List(ctx context.Context) ([]model.SystemConfig, error) {
findCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
cursor, err := s.collection.Find(findCtx, bson.D{}, options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}))
if err != nil {
return nil, err
}
defer cursor.Close(findCtx)
items := make([]model.SystemConfig, 0)
for cursor.Next(findCtx) {
var item model.SystemConfig
if err := cursor.Decode(&item); err != nil {
return nil, err
}
items = append(items, item)
}
if err := cursor.Err(); err != nil {
return nil, err
}
return items, nil
}
func (s *SystemConfigStore) GetByID(ctx context.Context, id primitive.ObjectID) (*model.SystemConfig, error) {
findCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
var item model.SystemConfig
err := s.collection.FindOne(findCtx, bson.M{"_id": id}).Decode(&item)
if err != nil {
if errors.Is(err, mongodriver.ErrNoDocuments) {
return nil, nil
}
return nil, err
}
return &item, nil
}
func (s *SystemConfigStore) Create(ctx context.Context, item *model.SystemConfig) error {
insertCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
now := time.Now()
item.CreatedAt = now
item.UpdatedAt = now
result, err := s.collection.InsertOne(insertCtx, item)
if err != nil {
return err
}
if oid, ok := result.InsertedID.(primitive.ObjectID); ok {
item.ID = oid
}
return nil
}
func (s *SystemConfigStore) Update(ctx context.Context, item *model.SystemConfig) error {
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
item.UpdatedAt = time.Now()
_, err := s.collection.UpdateByID(updateCtx, item.ID, bson.M{
"$set": bson.M{
"key": item.Key,
"title": item.Title,
"value": item.Value,
"valueType": item.ValueType,
"description": item.Description,
"enabled": item.Enabled,
"updatedAt": item.UpdatedAt,
},
})
return err
}
func (s *SystemConfigStore) ToggleEnabled(ctx context.Context, id primitive.ObjectID, enabled bool) error {
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
_, err := s.collection.UpdateByID(updateCtx, id, bson.M{
"$set": bson.M{
"enabled": enabled,
"updatedAt": time.Now(),
},
})
return err
}

18
internal/urlrewrite/redis.go Normal file
View File

@@ -0,0 +1,18 @@
package urlrewrite
import (
"context"
"github.com/redis/go-redis/v9"
)
func newRedisClient(ctx context.Context, addr, password string) (*redis.Client, error) {
rdb := redis.NewClient(&redis.Options{
Addr: addr,
Password: password,
})
if err := rdb.Ping(ctx).Err(); err != nil {
return nil, err
}
return rdb, nil
}

65
internal/urlrewrite/rewrite.go Normal file
View File

@@ -0,0 +1,65 @@
package urlrewrite
import "encoding/json"
func rewriteStringValue(input, oldPrefix, newPrefix string) (string, bool) {
if oldPrefix == "" || newPrefix == "" {
return input, false
}
if len(input) >= len(oldPrefix) && input[:len(oldPrefix)] == oldPrefix {
return newPrefix + input[len(oldPrefix):], true
}
return input, false
}
func rewriteJSONContent(input, oldPrefix, newPrefix string) (string, bool, error) {
var data any
if err := json.Unmarshal([]byte(input), &data); err != nil {
rewritten, changed := rewriteStringValue(input, oldPrefix, newPrefix)
if changed {
return rewritten, true, nil
}
return "", false, err
}
rewritten, changed := rewriteAny(data, oldPrefix, newPrefix)
if !changed {
return input, false, nil
}
out, err := json.Marshal(rewritten)
if err != nil {
return "", false, err
}
return string(out), true, nil
}
func rewriteAny(value any, oldPrefix, newPrefix string) (any, bool) {
switch v := value.(type) {
case map[string]any:
changed := false
for key, child := range v {
newChild, childChanged := rewriteAny(child, oldPrefix, newPrefix)
if childChanged {
v[key] = newChild
changed = true
}
}
return v, changed
case []any:
changed := false
for i, child := range v {
newChild, childChanged := rewriteAny(child, oldPrefix, newPrefix)
if childChanged {
v[i] = newChild
changed = true
}
}
return v, changed
case string:
if rewritten, changed, err := rewriteJSONContent(v, oldPrefix, newPrefix); err == nil && changed {
return rewritten, true
}
return rewriteStringValue(v, oldPrefix, newPrefix)
default:
return value, false
}
}

170
internal/urlrewrite/rewrite_test.go Normal file
View File

@@ -0,0 +1,170 @@
package urlrewrite
import (
"encoding/json"
"reflect"
"testing"
)
func TestRewriteStringValue(t *testing.T) {
oldPrefix := "https://s3.jizhying.com/images/"
newPrefix := "https://dp9pkdckmd09t.cloudfront.net/"
tests := []struct {
name string
input string
want string
}{
{
name: "replace matching prefix",
input: "https://s3.jizhying.com/images/openim/data/hash/a.jpg",
want: "https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/a.jpg",
},
{
name: "keep non matching url",
input: "https://example.com/a.jpg",
want: "https://example.com/a.jpg",
},
{
name: "keep plain text",
input: "hello",
want: "hello",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, changed := rewriteStringValue(tt.input, oldPrefix, newPrefix)
if got != tt.want {
t.Fatalf("rewriteStringValue() = %q, want %q", got, tt.want)
}
if (tt.input != tt.want) != changed {
t.Fatalf("rewriteStringValue() changed = %v, want %v", changed, tt.input != tt.want)
}
})
}
}
func TestRewriteJSONContent(t *testing.T) {
oldPrefix := "https://s3.jizhying.com/images/"
newPrefix := "https://dp9pkdckmd09t.cloudfront.net/"
input := `{"sourcePicture":{"url":"https://s3.jizhying.com/images/openim/data/hash/src.jpg"},"snapshotPicture":{"url":"https://s3.jizhying.com/images/openim/data/hash/snap.jpg"},"nested":{"fileElem":{"sourceUrl":"https://s3.jizhying.com/images/openim/data/hash/file.zip"}}}`
want := `{"nested":{"fileElem":{"sourceUrl":"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/file.zip"}},"snapshotPicture":{"url":"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/snap.jpg"},"sourcePicture":{"url":"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/src.jpg"}}`
got, changed, err := rewriteJSONContent(input, oldPrefix, newPrefix)
if err != nil {
t.Fatalf("rewriteJSONContent() error = %v", err)
}
if !changed {
t.Fatalf("rewriteJSONContent() changed = false, want true")
}
if got != want {
t.Fatalf("rewriteJSONContent() = %s, want %s", got, want)
}
}
func TestRewriteJSONContentNestedJSONString(t *testing.T) {
oldPrefix := "https://s3.jizhying.com/images/"
newPrefix := "https://dp9pkdckmd09t.cloudfront.net/"
input := `{"detail":"{\"group\":{\"faceURL\":\"https://s3.jizhying.com/images/openim/data/hash/group.jpg\"},\"entrantUser\":{\"faceURL\":\"https://s3.jizhying.com/images/openim/data/hash/user.jpg\"}}"}`
want := `{"detail":"{\"group\":{\"faceURL\":\"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/group.jpg\"},\"entrantUser\":{\"faceURL\":\"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/user.jpg\"}}"}`
got, changed, err := rewriteJSONContent(input, oldPrefix, newPrefix)
if err != nil {
t.Fatalf("rewriteJSONContent() error = %v", err)
}
if !changed {
t.Fatalf("rewriteJSONContent() changed = false, want true")
}
var gotValue map[string]any
var wantValue map[string]any
if err := json.Unmarshal([]byte(got), &gotValue); err != nil {
t.Fatalf("unmarshal got error = %v", err)
}
if err := json.Unmarshal([]byte(want), &wantValue); err != nil {
t.Fatalf("unmarshal want error = %v", err)
}
var gotDetail any
var wantDetail any
if err := json.Unmarshal([]byte(gotValue["detail"].(string)), &gotDetail); err != nil {
t.Fatalf("unmarshal got detail error = %v", err)
}
if err := json.Unmarshal([]byte(wantValue["detail"].(string)), &wantDetail); err != nil {
t.Fatalf("unmarshal want detail error = %v", err)
}
gotValue["detail"] = gotDetail
wantValue["detail"] = wantDetail
if !reflect.DeepEqual(gotValue, wantValue) {
t.Fatalf("rewriteJSONContent() = %#v, want %#v", gotValue, wantValue)
}
}
func TestRewriteJSONContentInvalidJSON(t *testing.T) {
_, changed, err := rewriteJSONContent("not-json", "https://old/", "https://new/")
if err == nil {
t.Fatalf("rewriteJSONContent() error = nil, want non-nil")
}
if changed {
t.Fatalf("rewriteJSONContent() changed = true, want false")
}
}
func TestCollectMsgFieldUpdates(t *testing.T) {
oldPrefix := "https://s3.jizhying.com/images/"
newPrefix := "https://dp9pkdckmd09t.cloudfront.net/"
doc := msgDoc{
Doc: "si_a_b:0",
Msgs: []msgEntryModel{
{
Msg: &msgPayload{
Content: `{"sourcePicture":{"url":"https://s3.jizhying.com/images/openim/data/hash/1.jpg"}}`,
SenderFaceURL: "https://s3.jizhying.com/images/openim/data/hash/avatar.jpg",
},
},
{
Msg: &msgPayload{
Content: `{"text":"hello"}`,
SenderFaceURL: "",
},
},
},
}
updates, changed, samples := collectMsgFieldUpdates(doc, oldPrefix, newPrefix, 5)
if !changed {
t.Fatalf("collectMsgFieldUpdates() changed = false, want true")
}
if len(samples) != 2 {
t.Fatalf("collectMsgFieldUpdates() samples len = %d, want 2", len(samples))
}
if len(updates) != 2 {
t.Fatalf("collectMsgFieldUpdates() updates len = %d, want 2", len(updates))
}
if updates[0].Path != "msgs.0.msg.content" {
t.Fatalf("first update path = %s", updates[0].Path)
}
if updates[0].NewValue != `{"sourcePicture":{"url":"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/1.jpg"}}` {
t.Fatalf("content update = %s", updates[0].NewValue)
}
if updates[1].Path != "msgs.0.msg.sender_face_url" {
t.Fatalf("second update path = %s", updates[1].Path)
}
if updates[1].NewValue != "https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/avatar.jpg" {
t.Fatalf("sender_face_url update = %s", updates[1].NewValue)
}
}
func TestBuildBackupDocs(t *testing.T) {
updates := []fieldUpdate{
{Path: "face_url", OldValue: "https://old/a.jpg", NewValue: "https://new/a.jpg"},
{Path: "thumbnail", OldValue: "https://old/b.jpg", NewValue: "https://new/b.jpg"},
}
docs := buildBackupDocs("batch-1", "attributes", "doc-1", updates)
if len(docs) != 2 {
t.Fatalf("buildBackupDocs() len = %d, want 2", len(docs))
}
}

500
internal/urlrewrite/runner.go Normal file
View File

@@ -0,0 +1,500 @@
package urlrewrite
import (
"context"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
backupCollectionName = "url_rewrite_backups"
msgCachePrefix = "MSG_CACHE:"
ModeDryRun = "dry-run"
ModeApply = "apply"
ModeRollback = "rollback"
ModeInvalidateCache = "invalidate-cache"
)
type Params struct {
OldPrefix string `json:"oldPrefix"`
NewPrefix string `json:"newPrefix"`
Mode string `json:"mode"`
BatchID string `json:"batchId"`
SampleSize int `json:"sampleSize"`
}
type RedisConfig struct {
Addr string
Password string
}
type Summary struct {
Collection string
Scanned int64
Updated int64
Samples []string
}
type fieldUpdate struct {
Path string
OldValue string
NewValue string
}
type msgDoc struct {
ID any `bson:"_id"`
Doc string `bson:"doc_id"`
Msgs []msgEntryModel `bson:"msgs"`
}
type msgEntryModel struct {
Msg *msgPayload `bson:"msg"`
}
type msgPayload struct {
Content string `bson:"content"`
SenderFaceURL string `bson:"sender_face_url"`
Seq int64 `bson:"seq"`
}
type LogFunc func(msg string, args ...any)
func Run(ctx context.Context, db *mongo.Database, params Params, redisCfg RedisConfig, logf LogFunc) (string, error) {
if err := validateParams(params); err != nil {
return "", err
}
if params.SampleSize <= 0 {
params.SampleSize = 5
}
if params.Mode == "" {
params.Mode = ModeDryRun
}
if params.Mode == ModeApply && params.BatchID == "" {
params.BatchID = "urlrewrite-" + time.Now().Format("20060102150405")
}
backupColl := db.Collection(backupCollectionName)
if params.Mode == ModeRollback {
return params.BatchID, rollbackBatch(ctx, backupColl, db, params, logf)
}
if params.Mode == ModeInvalidateCache {
return params.BatchID, invalidateBatchCache(ctx, db, redisCfg, params, logf)
}
summaries := []Summary{
rewriteSimpleFieldCollection(ctx, db.Collection("wallets"), backupColl, "real_name_auth.id_card_photo_front", params),
rewriteSimpleFieldCollection(ctx, db.Collection("wallets"), backupColl, "real_name_auth.id_card_photo_back", params),
rewriteSimpleFieldCollection(ctx, db.Collection("favorites"), backupColl, "content", params),
rewriteSimpleFieldCollection(ctx, db.Collection("favorites"), backupColl, "thumbnail", params),
rewriteSimpleFieldCollection(ctx, db.Collection("favorites"), backupColl, "link_url", params),
rewriteSimpleFieldCollection(ctx, db.Collection("attributes"), backupColl, "face_url", params),
rewriteSimpleFieldCollection(ctx, db.Collection("attribute"), backupColl, "face_url", params),
rewriteSimpleFieldCollection(ctx, db.Collection("user"), backupColl, "face_url", params),
rewriteSimpleFieldCollection(ctx, db.Collection("group"), backupColl, "face_url", params),
rewriteSimpleFieldCollection(ctx, db.Collection("group_member"), backupColl, "face_url", params),
}
msgSummary, err := rewriteMsgCollection(ctx, db.Collection("msg"), backupColl, params)
if err != nil {
return "", fmt.Errorf("rewrite msg collection: %w", err)
}
summaries = append(summaries, msgSummary)
for _, s := range summaries {
logf("[%s] scanned=%d updated=%d", s.Collection, s.Scanned, s.Updated)
for _, sample := range s.Samples {
logf("[%s] sample: %s", s.Collection, sample)
}
}
switch params.Mode {
case ModeDryRun:
logf("dry-run complete, rerun with mode=apply to write changes")
case ModeApply:
logf("apply complete, batch_id=%s", params.BatchID)
}
return params.BatchID, nil
}
func validateParams(p Params) error {
switch p.Mode {
case ModeDryRun, ModeApply, ModeRollback, ModeInvalidateCache:
case "":
default:
return fmt.Errorf("invalid mode: %s", p.Mode)
}
if p.Mode == ModeRollback || p.Mode == ModeInvalidateCache {
if p.BatchID == "" {
return fmt.Errorf("batchId is required for %s", p.Mode)
}
} else {
if p.OldPrefix == "" || p.NewPrefix == "" {
return fmt.Errorf("oldPrefix and newPrefix are required")
}
}
return nil
}
func invalidateBatchCache(ctx context.Context, db *mongo.Database, redisCfg RedisConfig, params Params, logf LogFunc) error {
if redisCfg.Addr == "" {
return fmt.Errorf("REDIS_ADDR is not configured")
}
backupColl := db.Collection(backupCollectionName)
cursor, err := backupColl.Find(ctx, bson.M{
"batch_id": params.BatchID,
"collection": "msg",
"field": bson.M{"$regex": primitive.Regex{Pattern: `^msgs\.\d+\.msg\.`}},
})
if err != nil {
return fmt.Errorf("find backup docs: %w", err)
}
defer cursor.Close(ctx)
type backupDoc struct {
DocumentID any `bson:"document_id"`
Field string `bson:"field"`
}
docIndexes := make(map[string]map[int]struct{})
for cursor.Next(ctx) {
var item backupDoc
if err := cursor.Decode(&item); err != nil {
return fmt.Errorf("decode backup doc: %w", err)
}
docID, ok := item.DocumentID.(primitive.ObjectID)
if !ok {
continue
}
msgIndex, ok := parseMsgFieldIndex(item.Field)
if !ok {
continue
}
id := docID.Hex()
if _, exists := docIndexes[id]; !exists {
docIndexes[id] = make(map[int]struct{})
}
docIndexes[id][msgIndex] = struct{}{}
}
msgColl := db.Collection("msg")
var keys []string
for hexID, indexes := range docIndexes {
docObjectID, err := primitive.ObjectIDFromHex(hexID)
if err != nil {
return fmt.Errorf("parse object id %s: %w", hexID, err)
}
var doc msgDoc
if err := msgColl.FindOne(ctx, bson.M{"_id": docObjectID}).Decode(&doc); err != nil {
return fmt.Errorf("find msg doc %s: %w", hexID, err)
}
conversationID := trimDocIDSuffix(doc.Doc)
if conversationID == "" {
continue
}
for idx := range indexes {
if idx < 0 || idx >= len(doc.Msgs) || doc.Msgs[idx].Msg == nil || doc.Msgs[idx].Msg.Seq <= 0 {
continue
}
keys = append(keys, msgCachePrefix+conversationID+":"+strconv.Itoa(int(doc.Msgs[idx].Msg.Seq)))
}
}
keys = deduplicateStrings(keys)
if len(keys) == 0 {
logf("invalidate-cache complete, batch_id=%s keys=0", params.BatchID)
return nil
}
rdb, err := newRedisClient(ctx, redisCfg.Addr, redisCfg.Password)
if err != nil {
return fmt.Errorf("connect redis: %w", err)
}
defer rdb.Close()
if err := rdb.Del(ctx, keys...).Err(); err != nil {
return fmt.Errorf("redis del: %w", err)
}
logf("invalidate-cache complete, batch_id=%s keys=%d", params.BatchID, len(keys))
return nil
}
func rewriteSimpleFieldCollection(ctx context.Context, coll, backupColl *mongo.Collection, field string, params Params) Summary {
filter := bson.M{field: bson.M{"$regex": primitive.Regex{Pattern: "^" + regexp.QuoteMeta(params.OldPrefix)}}}
cursor, err := coll.Find(ctx, filter)
if err != nil {
return Summary{Collection: coll.Name() + "." + field, Samples: []string{fmt.Sprintf("find error: %v", err)}}
}
defer cursor.Close(ctx)
summary := Summary{Collection: coll.Name() + "." + field}
for cursor.Next(ctx) {
summary.Scanned++
var doc bson.M
if err := cursor.Decode(&doc); err != nil {
if len(summary.Samples) < params.SampleSize {
summary.Samples = append(summary.Samples, fmt.Sprintf("decode error: %v", err))
}
continue
}
original, ok := nestedString(doc, field)
if !ok {
continue
}
rewritten, changed := rewriteStringValue(original, params.OldPrefix, params.NewPrefix)
if !changed {
continue
}
updates := []fieldUpdate{{
Path: field,
OldValue: original,
NewValue: rewritten,
}}
summary.Updated++
if len(summary.Samples) < params.SampleSize {
summary.Samples = append(summary.Samples, fmt.Sprintf("%s -> %s", original, rewritten))
}
if params.Mode != ModeApply {
continue
}
if err := insertBackupDocs(ctx, backupColl, params.BatchID, coll.Name(), doc["_id"], updates); err != nil {
if len(summary.Samples) < params.SampleSize {
summary.Samples = append(summary.Samples, fmt.Sprintf("backup error: %v", err))
}
continue
}
if _, err := coll.UpdateByID(ctx, doc["_id"], bson.M{"$set": bson.M{field: rewritten}}); err != nil {
if len(summary.Samples) < params.SampleSize {
summary.Samples = append(summary.Samples, fmt.Sprintf("update error: %v", err))
}
}
}
return summary
}
func rewriteMsgCollection(ctx context.Context, coll, backupColl *mongo.Collection, params Params) (Summary, error) {
cursor, err := coll.Find(ctx, bson.M{
"$or": bson.A{
bson.M{"msgs.msg.content": bson.M{"$regex": primitive.Regex{Pattern: regexp.QuoteMeta(params.OldPrefix)}}},
bson.M{"msgs.msg.sender_face_url": bson.M{"$regex": primitive.Regex{Pattern: "^" + regexp.QuoteMeta(params.OldPrefix)}}},
},
})
if err != nil {
return Summary{}, fmt.Errorf("find msg docs: %w", err)
}
defer cursor.Close(ctx)
summary := Summary{Collection: coll.Name()}
for cursor.Next(ctx) {
summary.Scanned++
var doc msgDoc
if err := cursor.Decode(&doc); err != nil {
if len(summary.Samples) < params.SampleSize {
summary.Samples = append(summary.Samples, fmt.Sprintf("decode error: %v", err))
}
continue
}
updates, changed, samples := collectMsgFieldUpdates(doc, params.OldPrefix, params.NewPrefix, params.SampleSize-len(summary.Samples))
if !changed {
continue
}
summary.Updated++
summary.Samples = append(summary.Samples, samples...)
if params.Mode != ModeApply {
continue
}
if err := insertBackupDocs(ctx, backupColl, params.BatchID, coll.Name(), doc.ID, updates); err != nil {
if len(summary.Samples) < params.SampleSize {
summary.Samples = append(summary.Samples, fmt.Sprintf("backup error: %v", err))
}
continue
}
if _, err := coll.UpdateByID(ctx, doc.ID, bson.M{"$set": toUpdateMap(updates)}); err != nil {
return summary, fmt.Errorf("update msg doc: %w", err)
}
}
return summary, nil
}
func collectMsgFieldUpdates(doc msgDoc, oldPrefix, newPrefix string, sampleBudget int) ([]fieldUpdate, bool, []string) {
var updates []fieldUpdate
var samples []string
for i := range doc.Msgs {
if doc.Msgs[i].Msg == nil {
continue
}
if doc.Msgs[i].Msg.Content != "" {
rewritten, contentChanged, err := rewriteJSONContent(doc.Msgs[i].Msg.Content, oldPrefix, newPrefix)
if err == nil && contentChanged {
updates = append(updates, fieldUpdate{
Path: fmt.Sprintf("msgs.%d.msg.content", i),
OldValue: doc.Msgs[i].Msg.Content,
NewValue: rewritten,
})
if len(samples) < sampleBudget {
samples = append(samples, fmt.Sprintf("doc=%s msgs.%d.msg.content updated", doc.Doc, i))
}
}
}
if doc.Msgs[i].Msg.SenderFaceURL != "" {
rewritten, faceChanged := rewriteStringValue(doc.Msgs[i].Msg.SenderFaceURL, oldPrefix, newPrefix)
if faceChanged {
updates = append(updates, fieldUpdate{
Path: fmt.Sprintf("msgs.%d.msg.sender_face_url", i),
OldValue: doc.Msgs[i].Msg.SenderFaceURL,
NewValue: rewritten,
})
if len(samples) < sampleBudget {
samples = append(samples, fmt.Sprintf("doc=%s msgs.%d.msg.sender_face_url updated", doc.Doc, i))
}
}
}
}
return updates, len(updates) > 0, samples
}
func toUpdateMap(updates []fieldUpdate) bson.M {
sets := bson.M{}
for _, u := range updates {
sets[u.Path] = u.NewValue
}
return sets
}
func buildBackupDocs(batchID, collection string, documentID any, updates []fieldUpdate) []any {
now := time.Now()
docs := make([]any, 0, len(updates))
for _, u := range updates {
docs = append(docs, bson.M{
"batch_id": batchID,
"collection": collection,
"document_id": documentID,
"field": u.Path,
"old_value": u.OldValue,
"new_value": u.NewValue,
"created_at": now,
})
}
return docs
}
func insertBackupDocs(ctx context.Context, backupColl *mongo.Collection, batchID, collection string, documentID any, updates []fieldUpdate) error {
if backupColl == nil || len(updates) == 0 {
return nil
}
_, err := backupColl.InsertMany(ctx, buildBackupDocs(batchID, collection, documentID, updates))
return err
}
func rollbackBatch(ctx context.Context, backupColl *mongo.Collection, db *mongo.Database, params Params, logf LogFunc) error {
cursor, err := backupColl.Find(ctx, bson.M{"batch_id": params.BatchID}, options.Find().SetSort(bson.D{
{Key: "collection", Value: 1},
{Key: "document_id", Value: 1},
{Key: "field", Value: 1},
}))
if err != nil {
return fmt.Errorf("find backup docs: %w", err)
}
defer cursor.Close(ctx)
type rollbackDoc struct {
Collection string `bson:"collection"`
Field string `bson:"field"`
OldValue string `bson:"old_value"`
DocumentID any `bson:"document_id"`
}
grouped := map[string]map[any][]fieldUpdate{}
for cursor.Next(ctx) {
var doc rollbackDoc
if err := cursor.Decode(&doc); err != nil {
return fmt.Errorf("decode backup doc: %w", err)
}
if _, ok := grouped[doc.Collection]; !ok {
grouped[doc.Collection] = map[any][]fieldUpdate{}
}
grouped[doc.Collection][doc.DocumentID] = append(grouped[doc.Collection][doc.DocumentID], fieldUpdate{
Path: doc.Field,
NewValue: doc.OldValue,
})
}
if len(grouped) == 0 {
return fmt.Errorf("no backup records found for batch_id %s", params.BatchID)
}
var collectionNames []string
for name := range grouped {
collectionNames = append(collectionNames, name)
}
sort.Strings(collectionNames)
for _, name := range collectionNames {
coll := db.Collection(name)
for documentID, updates := range grouped[name] {
if _, err := coll.UpdateByID(ctx, documentID, bson.M{"$set": toUpdateMap(updates)}); err != nil {
return fmt.Errorf("rollback %s: %w", name, err)
}
}
}
logf("rollback complete, batch_id=%s", params.BatchID)
return nil
}
func parseMsgFieldIndex(path string) (int, bool) {
parts := strings.Split(path, ".")
if len(parts) < 4 || parts[0] != "msgs" {
return 0, false
}
idx, err := strconv.Atoi(parts[1])
if err != nil {
return 0, false
}
return idx, true
}
func trimDocIDSuffix(docID string) string {
pos := strings.LastIndex(docID, ":")
if pos <= 0 {
return docID
}
return docID[:pos]
}
func deduplicateStrings(values []string) []string {
seen := make(map[string]struct{}, len(values))
result := make([]string, 0, len(values))
for _, v := range values {
if _, ok := seen[v]; ok {
continue
}
seen[v] = struct{}{}
result = append(result, v)
}
return result
}
func nestedString(doc bson.M, path string) (string, bool) {
current := any(doc)
for _, segment := range strings.Split(path, ".") {
switch m := current.(type) {
case bson.M:
current = m[segment]
case map[string]any:
current = m[segment]
default:
return "", false
}
}
val, ok := current.(string)
return val, ok
}

7
job-config-list/s3-migrate.json Normal file
View File

@@ -0,0 +1,7 @@
{
"name": "S3对象迁移",
"handlerKey": "s3-migrate",
"scheduleType": "manual",
"scheduleValue": "",
"defaultParams": "{\"sourceEngine\":\"minio\",\"sourceEndpoint\":\"http://minio:9000\",\"sourceBucket\":\"openim\",\"sourceAccessKey\":\"\",\"sourceSecretKey\":\"\",\"destEngine\":\"aws\",\"destEndpoint\":\"https://s3.ap-southeast-1.amazonaws.com\",\"destBucket\":\"openim\",\"destAccessKey\":\"\",\"destSecretKey\":\"\",\"destRegion\":\"ap-southeast-1\"}"
}

7
job-config-list/sample-handler.json Normal file
View File

@@ -0,0 +1,7 @@
{
"name": "示例任务",
"handlerKey": "sample-handler",
"scheduleType": "manual",
"scheduleValue": "",
"defaultParams": "{\"limit\":10,\"dryRun\":true}"
}

7
job-config-list/url-rewrite.json Normal file
View File

@@ -0,0 +1,7 @@
{
"name": "URL前缀替换",
"handlerKey": "url-rewrite",
"scheduleType": "manual",
"scheduleValue": "",
"defaultParams": "{\"oldPrefix\":\"https://s3.jizhying.com/images/\",\"newPrefix\":\"https://dp9pkdckmd09t.cloudfront.net/\",\"mode\":\"dry-run\",\"sampleSize\":10}"
}

127
pkg/config/config.go Normal file
View File

@@ -0,0 +1,127 @@
package config
import (
"bufio"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
)
type Config struct {
HTTPPort int
WorkerHTTPPort int
MongoHost string
MongoPort int
MongoUsername string
MongoPassword string
MongoAuthSource string
BusinessMongoDatabase string
SchedulerMongoDatabase string
RedisAddr string
RedisPassword string
}
func Load() Config {
loadEnvFiles()
return Config{
HTTPPort: getenvInt("HTTP_PORT", 10090),
WorkerHTTPPort: getenvInt("WORKER_HTTP_PORT", 10091),
MongoHost: getenv("MONGO_HOST", "127.0.0.1"),
MongoPort: getenvInt("MONGO_PORT", 27017),
MongoUsername: getenv("MONGO_USERNAME", ""),
MongoPassword: getenv("MONGO_PASSWORD", ""),
MongoAuthSource: getenv("MONGO_AUTHSOURCE", "admin"),
BusinessMongoDatabase: getenv("MONGO_DATABASE", "openim_v3"),
SchedulerMongoDatabase: getenv("SCHEDULER_MONGO_DATABASE", "scheduler_center"),
RedisAddr: getenv("REDIS_ADDR", ""),
RedisPassword: getenv("REDIS_PASSWORD", ""),
}
}
func loadEnvFiles() {
loadEnvFile(".env")
if envName := currentEnvName(); envName != "" {
loadEnvFile(".env." + envName)
}
loadEnvFile(".env.local")
}
func currentEnvName() string {
for _, key := range []string{"APP_ENV", "GO_ENV", "ENV"} {
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
return value
}
}
return ""
}
func loadEnvFile(name string) {
file, err := os.Open(filepath.Clean(name))
if err != nil {
return
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
key, value, ok := strings.Cut(line, "=")
if !ok {
continue
}
key = strings.TrimSpace(key)
value = strings.TrimSpace(value)
if key == "" {
continue
}
if _, exists := os.LookupEnv(key); exists {
continue
}
_ = os.Setenv(key, value)
}
}
func (c Config) MongoURI() string {
return fmt.Sprintf(
"mongodb://%s:%s@%s:%d/?authSource=%s",
c.MongoUsername,
c.MongoPassword,
c.MongoHost,
c.MongoPort,
c.MongoAuthSource,
)
}
func getenv(key, fallback string) string {
if value := os.Getenv(key); value != "" {
return value
}
return fallback
}
func getenvInt(key string, fallback int) int {
raw := getenv(key, "")
if raw == "" {
return fallback
}
value, err := strconv.Atoi(raw)
if err != nil {
return fallback
}
return value
}

21
pkg/config/config_test.go Normal file
View File

@@ -0,0 +1,21 @@
package config
import "testing"
func TestMongoURI(t *testing.T) {
cfg := Config{
MongoHost: "127.0.0.1",
MongoPort: 27017,
MongoUsername: "user",
MongoPassword: "pass",
MongoAuthSource: "admin",
BusinessMongoDatabase: "openim_v3",
SchedulerMongoDatabase: "scheduler_center",
}
got := cfg.MongoURI()
want := "mongodb://user:pass@127.0.0.1:27017/?authSource=admin"
if got != want {
t.Fatalf("MongoURI() = %q, want %q", got, want)
}
}

12
pkg/log/log.go Normal file
View File

@@ -0,0 +1,12 @@
package log
import (
"log/slog"
"os"
)
type Logger = *slog.Logger
func New() Logger {
return slog.New(slog.NewTextHandler(os.Stdout, nil))
}

5
scripts/seed-sample-job.sh Executable file
View File

@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
GOWORK=off go run ./cmd/tools/seed-sample-job

5
scripts/start-api.sh Executable file
View File

@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
GOWORK=off go run ./cmd/api

5
scripts/start-worker.sh Executable file
View File

@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
GOWORK=off go run ./cmd/worker

BIN
seed-sample-job Executable file
View File

Binary file not shown.

169
start.sh Executable file
View File

@@ -0,0 +1,169 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "$0")" && pwd)"
RUN_DIR="$ROOT_DIR/run"
LOG_DIR="$ROOT_DIR/logs"
API_PID_FILE="$RUN_DIR/api.pid"
WORKER_PID_FILE="$RUN_DIR/worker.pid"
API_LOG_FILE="$LOG_DIR/api.log"
WORKER_LOG_FILE="$LOG_DIR/worker.log"
API_BIN="$RUN_DIR/api"
WORKER_BIN="$RUN_DIR/worker"
HTTP_PORT=16811
WORKER_HTTP_PORT=16812
mkdir -p "$RUN_DIR" "$LOG_DIR"
load_env_config() {
local env_file
for env_file in ".env"; do
if [[ -f "$ROOT_DIR/$env_file" ]]; then
# shellcheck disable=SC1090
source "$ROOT_DIR/$env_file"
fi
done
if [[ -n "${APP_ENV:-}" && -f "$ROOT_DIR/.env.$APP_ENV" ]]; then
# shellcheck disable=SC1090
source "$ROOT_DIR/.env.$APP_ENV"
elif [[ -n "${GO_ENV:-}" && -f "$ROOT_DIR/.env.$GO_ENV" ]]; then
# shellcheck disable=SC1090
source "$ROOT_DIR/.env.$GO_ENV"
elif [[ -n "${ENV:-}" && -f "$ROOT_DIR/.env.$ENV" ]]; then
# shellcheck disable=SC1090
source "$ROOT_DIR/.env.$ENV"
fi
if [[ -f "$ROOT_DIR/.env.local" ]]; then
# shellcheck disable=SC1090
source "$ROOT_DIR/.env.local"
fi
}
port_pid() {
local port="$1"
lsof -tiTCP:"$port" -sTCP:LISTEN 2>/dev/null | head -n 1 || true
}
is_scheduler_process() {
local pid="$1"
local cmd
cmd="$(ps -p "$pid" -o command= 2>/dev/null || true)"
[[ "$cmd" == *"/scheduler-backend/run/api"* || "$cmd" == *"/scheduler-backend/run/worker"* || "$cmd" == *"go run ./cmd/api"* || "$cmd" == *"go run ./cmd/worker"* || "$cmd" == *"/scheduler-backend/api"* || "$cmd" == *"/scheduler-backend/worker"* || "$cmd" == *"/exe/api"* || "$cmd" == *"/exe/worker"* ]]
}
is_running() {
local pid_file="$1"
if [[ ! -f "$pid_file" ]]; then
return 1
fi
local pid
pid="$(cat "$pid_file")"
if [[ -z "$pid" ]]; then
return 1
fi
kill -0 "$pid" 2>/dev/null
}
ensure_port_free() {
local name="$1"
local port="$2"
local pid
pid="$(port_pid "$port")"
if [[ -z "$pid" ]]; then
return 0
fi
if is_scheduler_process "$pid"; then
kill "$pid" 2>/dev/null || true
sleep 1
pid="$(port_pid "$port")"
if [[ -z "$pid" ]]; then
echo "$name previous process stopped, port=$port"
return 0
fi
fi
echo "$name port $port is already occupied by pid=$pid" >&2
ps -p "$pid" -o pid,ppid,command >&2 || true
return 1
}
start_process() {
local name="$1"
local pid_file="$2"
local log_file="$3"
shift 3
if is_running "$pid_file"; then
echo "$name is already running, restarting pid=$(cat "$pid_file")"
kill "$(cat "$pid_file")" 2>/dev/null || true
sleep 1
fi
rm -f "$pid_file"
(
cd "$ROOT_DIR"
nohup "$@" >>"$log_file" 2>&1 &
echo $! >"$pid_file"
)
sleep 1
if is_running "$pid_file"; then
echo "$name started, pid=$(cat "$pid_file"), log=$log_file"
return 0
fi
echo "failed to start $name, check $log_file" >&2
rm -f "$pid_file"
return 1
}
build_binary() {
local output="$1"
local target="$2"
(
cd "$ROOT_DIR"
GOWORK=off go build -o "$output" "$target"
)
}
stop_process() {
local name="$1"
local pid_file="$2"
if ! is_running "$pid_file"; then
rm -f "$pid_file"
return 0
fi
kill "$(cat "$pid_file")" 2>/dev/null || true
rm -f "$pid_file"
echo "$name stopped"
}
build_binary "$API_BIN" ./cmd/api
build_binary "$WORKER_BIN" ./cmd/worker
load_env_config
ensure_port_free "api" "${HTTP_PORT}"
ensure_port_free "worker" "${WORKER_HTTP_PORT}"
start_process "api" "$API_PID_FILE" "$API_LOG_FILE" "$API_BIN"
if ! start_process "worker" "$WORKER_PID_FILE" "$WORKER_LOG_FILE" "$WORKER_BIN"; then
stop_process "api" "$API_PID_FILE"
exit 1
fi
echo "scheduler-backend started"
echo "api pid: $(cat "$API_PID_FILE")"
echo "worker pid: $(cat "$WORKER_PID_FILE")"
echo "api log: $API_LOG_FILE"
echo "worker log: $WORKER_LOG_FILE"

88
stop.sh Executable file
View File

@@ -0,0 +1,88 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "$0")" && pwd)"
RUN_DIR="$ROOT_DIR/run"
API_PID_FILE="$RUN_DIR/api.pid"
WORKER_PID_FILE="$RUN_DIR/worker.pid"
HTTP_PORT=16811
WORKER_HTTP_PORT=16812
load_env_config() {
local env_file
for env_file in ".env"; do
if [[ -f "$ROOT_DIR/$env_file" ]]; then
# shellcheck disable=SC1090
source "$ROOT_DIR/$env_file"
fi
done
if [[ -n "${APP_ENV:-}" && -f "$ROOT_DIR/.env.$APP_ENV" ]]; then
# shellcheck disable=SC1090
source "$ROOT_DIR/.env.$APP_ENV"
elif [[ -n "${GO_ENV:-}" && -f "$ROOT_DIR/.env.$GO_ENV" ]]; then
# shellcheck disable=SC1090
source "$ROOT_DIR/.env.$GO_ENV"
elif [[ -n "${ENV:-}" && -f "$ROOT_DIR/.env.$ENV" ]]; then
# shellcheck disable=SC1090
source "$ROOT_DIR/.env.$ENV"
fi
if [[ -f "$ROOT_DIR/.env.local" ]]; then
# shellcheck disable=SC1090
source "$ROOT_DIR/.env.local"
fi
}
port_pid() {
local port="$1"
lsof -tiTCP:"$port" -sTCP:LISTEN 2>/dev/null | head -n 1 || true
}
stop_port_process() {
local name="$1"
local port="$2"
local pid
pid="$(port_pid "$port")"
if [[ -z "$pid" ]]; then
return 0
fi
kill "$pid" 2>/dev/null || true
echo "$name stopped by port, pid=$pid"
}
stop_process() {
local name="$1"
local pid_file="$2"
if [[ ! -f "$pid_file" ]]; then
echo "$name is not running"
return 0
fi
local pid
pid="$(cat "$pid_file")"
if [[ -z "$pid" ]]; then
rm -f "$pid_file"
echo "$name pid file is empty, cleaned"
return 0
fi
if kill -0 "$pid" 2>/dev/null; then
kill "$pid"
echo "$name stopped, pid=$pid"
else
echo "$name is not running, stale pid=$pid"
fi
rm -f "$pid_file"
}
stop_process "worker" "$WORKER_PID_FILE"
stop_process "api" "$API_PID_FILE"
load_env_config
stop_port_process "worker" "${WORKER_HTTP_PORT}"
stop_port_process "api" "${HTTP_PORT}"

BIN
worker Executable file
View File

Binary file not shown.