commit 52446ccf3f25461df1fdf8a0e49a16943d0c1c66 Author: vet Date: Thu May 28 00:16:19 2026 +0800 init diff --git a/.env b/.env new file mode 100644 index 0000000..ca5f7c6 --- /dev/null +++ b/.env @@ -0,0 +1,11 @@ +HTTP_PORT=16811 +WORKER_HTTP_PORT=16812 +MONGO_HOST=47.237.103.4 +MONGO_PORT=27017 +MONGO_USERNAME=minio_pC5wMB +MONGO_PASSWORD=rI57PJsJhnz_qlRkfnTa0RPT +MONGO_AUTHSOURCE=admin +MONGO_DATABASE=openim_v3 +SCHEDULER_MONGO_DATABASE=scheduler_center +REDIS_ADDR=127.0.0.1:6379 +REDIS_PASSWORD= diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..392e99b --- /dev/null +++ b/.env.example @@ -0,0 +1,11 @@ +HTTP_PORT=16811 +WORKER_HTTP_PORT=16812 +MONGO_HOST=47.237.103.4 +MONGO_PORT=27017 +MONGO_USERNAME= +MONGO_PASSWORD= +MONGO_AUTHSOURCE=admin +MONGO_DATABASE=openim_v3 +SCHEDULER_MONGO_DATABASE=scheduler_center +REDIS_ADDR=127.0.0.1:6379 +REDIS_PASSWORD= diff --git a/.env.local b/.env.local new file mode 100644 index 0000000..44dea80 --- /dev/null +++ b/.env.local @@ -0,0 +1,9 @@ +HTTP_PORT=16811 +WORKER_HTTP_PORT=16812 +MONGO_HOST=47.237.103.4 +MONGO_PORT=27017 +MONGO_USERNAME=minio_pC5wMB +MONGO_PASSWORD=rI57PJsJhnz_qlRkfnTa0RPT +MONGO_AUTHSOURCE=admin +MONGO_DATABASE=openim_v3 +SCHEDULER_MONGO_DATABASE=scheduler_center_local diff --git a/.env.prod b/.env.prod new file mode 100644 index 0000000..95b4b77 --- /dev/null +++ b/.env.prod @@ -0,0 +1,9 @@ +HTTP_PORT=16811 +WORKER_HTTP_PORT=16812 +MONGO_HOST=47.237.103.4 +MONGO_PORT=27017 +MONGO_USERNAME=minio_pC5wMB +MONGO_PASSWORD=rI57PJsJhnz_qlRkfnTa0RPT +MONGO_AUTHSOURCE=admin +MONGO_DATABASE=openim_v3 +SCHEDULER_MONGO_DATABASE=scheduler_center diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2cf4861 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +_output/ +*.exe +*.test +*.out +logs/ +run/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..3bb3997 --- /dev/null +++ b/README.md @@ -0,0 +1,170 @@ +# scheduler-backend + +独立的调度中心后端工程，负责： + +- 管理 API +- 本地 Go handler 注册 +- 手动执行 +- `gocron` 定时执行 +- MongoDB 中的任务配置与执行记录存储 + +## 当前能力 + +- `GET /admin/scheduler/handlers` +- `GET /admin/scheduler/jobs` +- `GET /admin/scheduler/jobs/:id` +- `POST /admin/scheduler/jobs` +- `PUT /admin/scheduler/jobs/:id` +- `POST /admin/scheduler/jobs/:id/toggle` +- `POST /admin/scheduler/jobs/:id/run` +- `GET /admin/scheduler/executions` + +## 目录结构 + +```text +cmd/api 管理 API 入口 +cmd/worker 调度执行入口 +internal/api Gin 路由 +internal/executor 本地 handler 执行 +internal/jobdef handler 定义与注册 +internal/scheduler gocron 调度服务 +internal/store/model Mongo 文档模型 +internal/store/mongo Mongo 连接与存储实现 +pkg/config 环境配置加载 +pkg/log 日志封装 +``` + +## 环境变量 + +默认会按下面顺序读取： + +1. `.env` +2. `.env.` +3. `.env.local` + +外部进程环境变量优先级更高，不会被文件覆盖。 + +关键变量： + +```env +HTTP_PORT=16811 +WORKER_HTTP_PORT=16812 +MONGO_HOST=47.237.103.4 +MONGO_PORT=27017 +MONGO_USERNAME= +MONGO_PASSWORD= +MONGO_AUTHSOURCE=admin +MONGO_DATABASE=openim_v3 +SCHEDULER_MONGO_DATABASE=scheduler_center +``` + +## 启动方式 + +### 一条命令启动后端 + +```bash +cd /Users/vet/all/workspace414/scheduler-backend +./start.sh +``` + +这个脚本会同时启动： + +- `api` +- `worker` + +并把运行信息写到： + +- `logs/api.log` +- `logs/worker.log` + +PID 文件会写到： + +- `run/api.pid` +- `run/worker.pid` + +停止方式： + +```bash +cd /Users/vet/all/workspace414/scheduler-backend +./stop.sh +``` + +### 启动 API + +```bash +cd /Users/vet/all/workspace414/scheduler-backend +GOWORK=off go run ./cmd/api +``` + +或直接运行： + +```bash +./scripts/start-api.sh +``` + +默认监听： + +```text +http://127.0.0.1:16811 +``` + +### 启动 worker + +```bash +cd /Users/vet/all/workspace414/scheduler-backend +GOWORK=off go run ./cmd/worker +``` + +或直接运行： + +```bash +./scripts/start-worker.sh +``` + +worker 会： + +- 连接 Mongo +- 暴露健康检查端口 `http://127.0.0.1:16812/healthz` +- 加载启用任务 +- 跳过 `manual` +- 注册 `cron` 和 `interval` +- 到点写执行记录并执行本地 Go handler + +## 初始化示例任务 + +会向 `scheduler_center.job_configs` 写入一条示例任务： + +- `name=sample-job` +- `handlerKey=sample-handler` +- `enabled=false` +- `scheduleType=manual` +- `defaultParams={"limit":10,"dryRun":true}` + +运行方式： + +```bash +cd /Users/vet/all/workspace414/scheduler-backend +GOWORK=off go run ./cmd/tools/seed-sample-job +``` + +或直接运行： + +```bash +./scripts/seed-sample-job.sh +``` + +## 测试与构建 + +```bash +cd /Users/vet/all/workspace414/scheduler-backend +GOWORK=off go test ./pkg/config ./internal/jobdef ./internal/store/model ./internal/executor -v +GOWORK=off go build ./cmd/api ./cmd/worker ./cmd/tools/seed-sample-job +``` + +## 当前限制 + +- 只支持本地 Go handler +- `interval` 依赖 `time.ParseDuration`，例如 `30s`、`5m`、`1h` +- 没有审批流 +- 没有分布式多 worker 协调 +- 执行记录详情页接口还没单独拆出 diff --git a/api b/api new file mode 100755 index 0000000..8c7f5d9 Binary files /dev/null and b/api differ diff --git a/cmd/api/main.go b/cmd/api/main.go new file mode 100644 index 0000000..a80f4b4 --- /dev/null +++ b/cmd/api/main.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "scheduler-backend/internal/api" + "scheduler-backend/internal/executor" + "scheduler-backend/internal/jobdef" + storemongo "scheduler-backend/internal/store/mongo" + "scheduler-backend/pkg/config" + "scheduler-backend/pkg/log" +) + +func main() { + cfg := config.Load() + logger := log.New() + addr := fmt.Sprintf(":%d", cfg.HTTPPort) + registry := jobdef.NewRegistry( + jobdef.SampleHandler{}, + jobdef.URLRewriteHandler{}, + jobdef.S3MigrateHandler{}, + ) + rootCtx := context.Background() + databases, err := storemongo.Connect(rootCtx, cfg) + if err != nil { + logger.Error("scheduler api mongo connect failed", "error", err) + os.Exit(1) + } + jobConfigStore := storemongo.NewJobConfigStore(databases.MetaDB) + if err := jobdef.SyncJobConfigs(rootCtx, "job-config-list", jobConfigStore, logger); err != nil { + logger.Error("sync job configs failed", "error", err) + os.Exit(1) + } + + execSvc := executor.NewService(registry, jobdef.Runtime{ + Config: cfg, + Logger: logger, + MetaDB: databases.MetaDB, + BusinessDB: databases.BusinessDB, + }) + + router := api.NewRouter(api.RouterDeps{ + Registry: registry, + JobConfigStore: jobConfigStore, + ExecutionStore: storemongo.NewJobExecutionStore(databases.MetaDB), + ProfileStore: storemongo.NewAdminProfileStore(databases.MetaDB), + AdminUserStore: storemongo.NewAdminUserStore(databases.MetaDB), + ConfigStore: storemongo.NewSystemConfigStore(databases.MetaDB), + Executor: execSvc, + }) + logger.Info("scheduler api starting", "addr", addr) + + go func() { + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + <-stop + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = databases.Client.Disconnect(ctx) + os.Exit(0) + }() + + if err := router.Run(addr); err != nil { + logger.Error("scheduler api stopped", "error", err) + } +} diff --git a/cmd/tools/seed-sample-job/main.go b/cmd/tools/seed-sample-job/main.go new file mode 100644 index 0000000..b8eb7c1 --- /dev/null +++ b/cmd/tools/seed-sample-job/main.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/options" + + storemongo "scheduler-backend/internal/store/mongo" + "scheduler-backend/pkg/config" + "scheduler-backend/pkg/log" +) + +func main() { + cfg := config.Load() + logger := log.New() + + databases, err := storemongo.Connect(context.Background(), cfg) + if err != nil { + logger.Error("seed sample job mongo connect failed", "error", err) + os.Exit(1) + } + defer func() { + _ = databases.Client.Disconnect(context.Background()) + }() + + now := time.Now() + doc := bson.M{ + "name": "sample-job", + "handlerKey": "sample-handler", + "enabled": false, + "scheduleType": "manual", + "scheduleValue": "", + "defaultParams": `{"limit":10,"dryRun":true}`, + "lastStatus": "idle", + "updatedAt": now, + } + + result, err := databases.MetaDB.Collection("job_configs").UpdateOne( + context.Background(), + bson.M{"name": "sample-job"}, + bson.M{ + "$set": doc, + "$setOnInsert": bson.M{"createdAt": now}, + }, + options.Update().SetUpsert(true), + ) + if err != nil { + logger.Error("seed sample job failed", "error", err) + os.Exit(1) + } + + logger.Info( + "seed sample job completed", + "matched", + result.MatchedCount, + "modified", + result.ModifiedCount, + "upserted", + fmt.Sprintf("%v", result.UpsertedID), + ) +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 0000000..c25ae69 --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "scheduler-backend/internal/executor" + "scheduler-backend/internal/jobdef" + "scheduler-backend/internal/scheduler" + storemongo "scheduler-backend/internal/store/mongo" + "scheduler-backend/pkg/config" + "scheduler-backend/pkg/log" + + "github.com/go-co-op/gocron/v2" +) + +func main() { + cfg := config.Load() + logger := log.New() + registry := jobdef.NewRegistry( + jobdef.SampleHandler{}, + jobdef.URLRewriteHandler{}, + jobdef.S3MigrateHandler{}, + ) + rootCtx := context.Background() + databases, err := storemongo.Connect(rootCtx, cfg) + if err != nil { + logger.Error("scheduler worker mongo connect failed", "error", err) + os.Exit(1) + } + + jobConfigStore := storemongo.NewJobConfigStore(databases.MetaDB) + if err := jobdef.SyncJobConfigs(rootCtx, "job-config-list", jobConfigStore, logger); err != nil { + logger.Error("sync job configs failed", "error", err) + os.Exit(1) + } + + runtime := jobdef.Runtime{ + Config: cfg, + Logger: logger, + MetaDB: databases.MetaDB, + BusinessDB: databases.BusinessDB, + } + execSvc := executor.NewService(registry, runtime) + g, err := gocron.NewScheduler() + if err != nil { + logger.Error("scheduler worker init scheduler failed", "error", err) + os.Exit(1) + } + svc := scheduler.NewService( + g, + jobConfigStore, + storemongo.NewJobExecutionStore(databases.MetaDB), + execSvc, + runtime, + ) + + healthServer := &http.Server{ + Addr: fmt.Sprintf(":%d", cfg.WorkerHTTPPort), + ReadHeaderTimeout: 5 * time.Second, + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/healthz", "/readyz": + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + default: + w.WriteHeader(http.StatusNotFound) + } + }), + } + + go func() { + logger.Info("scheduler worker health server starting", "addr", healthServer.Addr) + if err := healthServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + logger.Error("scheduler worker health server stopped", "error", err) + } + }() + + logger.Info("scheduler worker starting", "db", cfg.SchedulerMongoDatabase, "healthAddr", healthServer.Addr) + if err := svc.RegisterEnabledJobs(rootCtx); err != nil { + logger.Error("scheduler worker register jobs failed", "error", err) + os.Exit(1) + } + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + <-stop + + if err := svc.Shutdown(); err != nil { + logger.Error("scheduler worker shutdown failed", "error", err) + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := healthServer.Shutdown(shutdownCtx); err != nil { + logger.Error("scheduler worker health server shutdown failed", "error", err) + } + if err := databases.Client.Disconnect(context.Background()); err != nil { + logger.Error("scheduler worker mongo disconnect failed", "error", err) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f14ec31 --- /dev/null +++ b/go.mod @@ -0,0 +1,66 @@ +module scheduler-backend + +go 1.24 + +require ( + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/credentials v1.19.17 + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 + github.com/gin-gonic/gin v1.10.0 + github.com/go-co-op/gocron/v2 v2.14.2 + github.com/redis/go-redis/v9 v9.19.0 + go.mongodb.org/mongo-driver v1.17.6 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/smithy-go v1.25.1 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jonboulle/clockwork v0.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.17.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c960fe1 --- /dev/null +++ b/go.sum @@ -0,0 +1,186 @@ +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.17 h1:gP2nkGsS+KMvF/jfFz2Vv2qiiOqWKyPACSzPsqHgoW8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.17/go.mod h1:Bsew3S/moG5iT77giPj1q8wb/s0RE5/QfH+ASjYtuQc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-co-op/gocron/v2 v2.14.2 h1:S6CbI7MVfD3S/aPJNLoSg2YcGyEqzEMwUopDejuT4Oc= +github.com/go-co-op/gocron/v2 v2.14.2/go.mod h1:ZF70ZwEqz0OO4RBXE1sNxnANy/zvwLcattWEFsqpKig= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k= +github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= +go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/internal/api/router.go b/internal/api/router.go new file mode 100644 index 0000000..f801fa3 --- /dev/null +++ b/internal/api/router.go @@ -0,0 +1,1031 @@ +package api + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson/primitive" + "scheduler-backend/internal/executor" + "scheduler-backend/internal/jobdef" + "scheduler-backend/internal/store/model" +) + +type handlerItem struct { + Key string `json:"key"` + Name string `json:"name"` + Description string `json:"description"` +} + +type handlerListResponse struct { + List []handlerItem `json:"list"` +} + +type jobItem struct { + ID string `json:"id"` + Name string `json:"name"` + HandlerKey string `json:"handlerKey"` + Enabled bool `json:"enabled"` + ScheduleType string `json:"scheduleType"` + ScheduleValue string `json:"scheduleValue"` + DefaultParams string `json:"defaultParams"` + LastStatus string `json:"lastStatus"` + LastRunAt string `json:"lastRunAt,omitempty"` + NextRunAt string `json:"nextRunAt,omitempty"` +} + +type jobListResponse struct { + List []jobItem `json:"list"` + Total int `json:"total"` +} + +type executionItem struct { + ID string `json:"id"` + JobName string `json:"jobName"` + TriggerType string `json:"triggerType"` + Status string `json:"status"` + StartedAt string `json:"startedAt,omitempty"` + FinishedAt string `json:"finishedAt,omitempty"` + ResultSummary string `json:"resultSummary"` +} + +type executionDetailItem struct { + ID string `json:"id"` + JobConfigID string `json:"jobConfigId"` + TriggerType string `json:"triggerType"` + Status string `json:"status"` + ParamsSnapshot string `json:"paramsSnapshot"` + StartedAt string `json:"startedAt,omitempty"` + FinishedAt string `json:"finishedAt,omitempty"` + DurationMs int64 `json:"durationMs"` + ResultSummary string `json:"resultSummary"` + ErrorMessage string `json:"errorMessage"` + LogText string `json:"logText"` + LogFile string `json:"logFile,omitempty"` + CreatedAt string `json:"createdAt"` +} + +type executionListResponse struct { + List []executionItem `json:"list"` + Total int `json:"total"` +} + +type profileItem struct { + ID string `json:"id"` + Account string `json:"account"` + Nickname string `json:"nickname"` + Avatar string `json:"avatar"` + Remark string `json:"remark"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` +} + +type adminUserItem struct { + ID string `json:"id"` + Account string `json:"account"` + Nickname string `json:"nickname"` + Status string `json:"status"` + Role string `json:"role"` + Remark string `json:"remark"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` +} + +type adminUserListResponse struct { + List []adminUserItem `json:"list"` + Total int `json:"total"` +} + +type systemConfigItem struct { + ID string `json:"id"` + Key string `json:"key"` + Title string `json:"title"` + Value string `json:"value"` + ValueType string `json:"valueType"` + Description string `json:"description"` + Enabled bool `json:"enabled"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` +} + +type systemConfigListResponse struct { + List []systemConfigItem `json:"list"` + Total int `json:"total"` +} + +type dashboardOverviewResponse struct { + JobTotal int `json:"jobTotal"` + EnabledJobTotal int `json:"enabledJobTotal"` + ExecutionToday int `json:"executionToday"` + FailedToday int `json:"failedToday"` + LastExecutionAt string `json:"lastExecutionAt,omitempty"` + FailedJobTotal int `json:"failedJobTotal"` + AdminUserTotal int `json:"adminUserTotal"` + SystemConfigTotal int `json:"systemConfigTotal"` + RecentExecutions []executionItem `json:"recentExecutions"` +} + +type JobConfigLister interface { + List(ctx context.Context) ([]model.JobConfig, error) + GetByID(ctx context.Context, id primitive.ObjectID) (*model.JobConfig, error) + Create(ctx context.Context, item *model.JobConfig) error + Update(ctx context.Context, item *model.JobConfig) error + ToggleEnabled(ctx context.Context, id primitive.ObjectID, enabled bool) error + UpdateRunState(ctx context.Context, id primitive.ObjectID, status string, lastRunAt *time.Time) error +} + +type JobExecutionLister interface { + List(ctx context.Context) ([]model.JobExecution, error) + GetByID(ctx context.Context, id primitive.ObjectID) (*model.JobExecution, error) + Create(ctx context.Context, item *model.JobExecution) error + UpdateResult(ctx context.Context, item *model.JobExecution) error +} + +type AdminProfileStore interface { + Get(ctx context.Context) (*model.AdminProfile, error) + Upsert(ctx context.Context, profile *model.AdminProfile) error + ChangePassword(ctx context.Context, oldPassword, newPassword string) error + Authenticate(ctx context.Context, account, password string) (*model.AdminProfile, error) +} + +type AdminUserStore interface { + List(ctx context.Context) ([]model.AdminUser, error) + GetByID(ctx context.Context, id primitive.ObjectID) (*model.AdminUser, error) + Create(ctx context.Context, item *model.AdminUser) error + Update(ctx context.Context, item *model.AdminUser) error + ChangePassword(ctx context.Context, id primitive.ObjectID, newPassword string) error + Authenticate(ctx context.Context, account, password string) (*model.AdminUser, error) +} + +type SystemConfigStore interface { + List(ctx context.Context) ([]model.SystemConfig, error) + GetByID(ctx context.Context, id primitive.ObjectID) (*model.SystemConfig, error) + Create(ctx context.Context, item *model.SystemConfig) error + Update(ctx context.Context, item *model.SystemConfig) error + ToggleEnabled(ctx context.Context, id primitive.ObjectID, enabled bool) error +} + +type RouterDeps struct { + Registry *jobdef.Registry + JobConfigStore JobConfigLister + ExecutionStore JobExecutionLister + ProfileStore AdminProfileStore + AdminUserStore AdminUserStore + ConfigStore SystemConfigStore + Executor *executor.Service +} + +type upsertJobRequest struct { + Name string `json:"name" binding:"required"` + HandlerKey string `json:"handlerKey" binding:"required"` + Enabled bool `json:"enabled"` + ScheduleType string `json:"scheduleType" binding:"required"` + ScheduleValue string `json:"scheduleValue"` + DefaultParams string `json:"defaultParams"` +} + +type toggleJobRequest struct { + Enabled bool `json:"enabled"` +} + +type runJobRequest struct { + Params json.RawMessage `json:"params"` +} + +type updateProfileRequest struct { + Account string `json:"account" binding:"required"` + Nickname string `json:"nickname" binding:"required"` + Avatar string `json:"avatar"` + Remark string `json:"remark"` +} + +type changeProfilePasswordRequest struct { + OldPassword string `json:"oldPassword" binding:"required"` + NewPassword string `json:"newPassword" binding:"required"` +} + +type loginRequest struct { + Account string `json:"account" binding:"required"` + Password string `json:"password" binding:"required"` +} + +type upsertAdminUserRequest struct { + Account string `json:"account" binding:"required"` + Nickname string `json:"nickname" binding:"required"` + Status string `json:"status"` + Role string `json:"role"` + Remark string `json:"remark"` + Password string `json:"password"` +} + +type changeAdminPasswordRequest struct { + NewPassword string `json:"newPassword" binding:"required"` +} + +type upsertSystemConfigRequest struct { + Key string `json:"key" binding:"required"` + Title string `json:"title" binding:"required"` + Value string `json:"value"` + ValueType string `json:"valueType" binding:"required"` + Description string `json:"description"` + Enabled bool `json:"enabled"` +} + +type toggleSystemConfigRequest struct { + Enabled bool `json:"enabled"` +} + +func NewRouter(deps RouterDeps) *gin.Engine { + router := gin.Default() + router.Use(func(c *gin.Context) { + c.Header("Access-Control-Allow-Origin", "*") + c.Header("Access-Control-Allow-Headers", "Content-Type, Authorization") + c.Header("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS") + c.Header("Access-Control-Expose-Headers", "Content-Length, Content-Type") + if c.Request.Method == http.MethodOptions { + c.AbortWithStatus(http.StatusNoContent) + return + } + c.Next() + }) + router.POST("/admin/account/login", func(c *gin.Context) { + var req loginRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + profile, err := deps.ProfileStore.Authenticate(c.Request.Context(), req.Account, req.Password) + if err == nil { + c.JSON(http.StatusOK, toProfileItem(*profile)) + return + } + + adminUser, userErr := deps.AdminUserStore.Authenticate(c.Request.Context(), req.Account, req.Password) + if userErr != nil { + c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid account or password"}) + return + } + + c.JSON(http.StatusOK, adminUserItem{ + ID: objectIDToString(adminUser.ID), + Account: adminUser.Account, + Nickname: adminUser.Nickname, + Status: adminUser.Status, + Role: adminUser.Role, + Remark: adminUser.Remark, + CreatedAt: adminUser.CreatedAt.Format(time.RFC3339), + UpdatedAt: adminUser.UpdatedAt.Format(time.RFC3339), + }) + }) + router.GET("/admin/dashboard/overview", func(c *gin.Context) { + jobs, err := deps.JobConfigStore.List(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + executions, err := deps.ExecutionStore.List(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + adminUsers, err := deps.AdminUserStore.List(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + systemConfigs, err := deps.ConfigStore.List(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + enabledCount := 0 + failedJobCount := 0 + jobNameMap := make(map[string]string, len(jobs)) + for _, item := range jobs { + jobNameMap[item.ID.Hex()] = item.Name + if item.Enabled { + enabledCount++ + } + if item.LastStatus == "failed" { + failedJobCount++ + } + } + + now := time.Now() + todayStart := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + executionToday := 0 + failedToday := 0 + lastExecutionAt := "" + recent := make([]executionItem, 0, min(len(executions), 5)) + for idx, item := range executions { + if idx < 5 { + recent = append(recent, executionItem{ + ID: objectIDToString(item.ID), + JobName: jobDisplayName(jobNameMap, item.JobConfigID), + TriggerType: item.TriggerType, + Status: item.Status, + StartedAt: formatTime(item.StartedAt), + FinishedAt: formatTime(item.FinishedAt), + ResultSummary: item.ResultSummary, + }) + } + if lastExecutionAt == "" && item.StartedAt != nil { + lastExecutionAt = item.StartedAt.Format(time.RFC3339) + } + if item.CreatedAt.After(todayStart) { + executionToday++ + if item.Status == "failed" { + failedToday++ + } + } + } + + c.JSON(http.StatusOK, dashboardOverviewResponse{ + JobTotal: len(jobs), + EnabledJobTotal: enabledCount, + ExecutionToday: executionToday, + FailedToday: failedToday, + LastExecutionAt: lastExecutionAt, + FailedJobTotal: failedJobCount, + AdminUserTotal: len(adminUsers), + SystemConfigTotal: len(systemConfigs), + RecentExecutions: recent, + }) + }) + router.GET("/admin/scheduler/handlers", func(c *gin.Context) { + items := make([]handlerItem, 0, len(deps.Registry.List())) + for _, handler := range deps.Registry.List() { + items = append(items, handlerItem{ + Key: handler.Key(), + Name: handler.Name(), + Description: handler.Description(), + }) + } + + c.JSON(http.StatusOK, handlerListResponse{List: items}) + }) + router.GET("/admin/scheduler/jobs", func(c *gin.Context) { + items, err := deps.JobConfigStore.List(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + responseItems := make([]jobItem, 0, len(items)) + for _, item := range items { + responseItems = append(responseItems, toJobItem(item)) + } + + c.JSON(http.StatusOK, jobListResponse{ + List: responseItems, + Total: len(responseItems), + }) + }) + router.GET("/admin/scheduler/jobs/:id", func(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) + return + } + + item, err := deps.JobConfigStore.GetByID(c.Request.Context(), id) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if item == nil { + c.JSON(http.StatusNotFound, gin.H{"error": "job not found"}) + return + } + + c.JSON(http.StatusOK, toJobItem(*item)) + }) + router.POST("/admin/scheduler/jobs", func(c *gin.Context) { + var req upsertJobRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + item := model.NewJobConfig(req.Name, req.HandlerKey) + item.Enabled = req.Enabled + item.ScheduleType = req.ScheduleType + item.ScheduleValue = req.ScheduleValue + item.DefaultParams = req.DefaultParams + + if err := deps.JobConfigStore.Create(c.Request.Context(), &item); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusCreated, toJobItem(item)) + }) + router.PUT("/admin/scheduler/jobs/:id", func(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) + return + } + + current, err := deps.JobConfigStore.GetByID(c.Request.Context(), id) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if current == nil { + c.JSON(http.StatusNotFound, gin.H{"error": "job not found"}) + return + } + + var req upsertJobRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + current.Name = req.Name + current.HandlerKey = req.HandlerKey + current.Enabled = req.Enabled + current.ScheduleType = req.ScheduleType + current.ScheduleValue = req.ScheduleValue + current.DefaultParams = req.DefaultParams + + if err := deps.JobConfigStore.Update(c.Request.Context(), current); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, toJobItem(*current)) + }) + router.POST("/admin/scheduler/jobs/:id/toggle", func(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) + return + } + + var req toggleJobRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + if err := deps.JobConfigStore.ToggleEnabled(c.Request.Context(), id, req.Enabled); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{"success": true}) + }) + router.POST("/admin/scheduler/jobs/:id/run", func(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) + return + } + + job, err := deps.JobConfigStore.GetByID(c.Request.Context(), id) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if job == nil { + c.JSON(http.StatusNotFound, gin.H{"error": "job not found"}) + return + } + + var req runJobRequest + if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + params := req.Params + params, err = mergeExecutionParams(job.DefaultParams, req.Params) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + now := time.Now() + execRecord := model.JobExecution{ + JobConfigID: job.ID, + TriggerType: "manual", + Status: "running", + ParamsSnapshot: string(params), + StartedAt: &now, + CreatedAt: now, + } + if err := deps.ExecutionStore.Create(c.Request.Context(), &execRecord); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + execReq := jobdef.ExecuteRequest{ + ExecutionID: execRecord.ID.Hex(), + JobID: job.ID.Hex(), + TriggerType: "manual", + Params: params, + LogCollector: jobdef.NewLogCollector(), + } + runErr := deps.Executor.Execute(c.Request.Context(), job.HandlerKey, execReq) + + finishedAt := time.Now() + execRecord.FinishedAt = &finishedAt + execRecord.DurationMs = finishedAt.Sub(now).Milliseconds() + flushed := execReq.LogCollector.Flush(execRecord.ID.Hex()) + if runErr != nil { + execRecord.Status = "failed" + execRecord.ErrorMessage = runErr.Error() + execRecord.ResultSummary = "manual execution failed" + if flushed.LogText != "" { + execRecord.LogText = flushed.LogText + } else { + execRecord.LogText = runErr.Error() + } + execRecord.LogFile = flushed.LogFile + _ = deps.JobConfigStore.UpdateRunState(c.Request.Context(), job.ID, "failed", &finishedAt) + } else { + execRecord.Status = "success" + execRecord.ResultSummary = "manual execution succeeded" + if flushed.LogText != "" { + execRecord.LogText = flushed.LogText + } else { + execRecord.LogText = "handler executed successfully" + } + execRecord.LogFile = flushed.LogFile + _ = deps.JobConfigStore.UpdateRunState(c.Request.Context(), job.ID, "success", &finishedAt) + } + + if err := deps.ExecutionStore.UpdateResult(c.Request.Context(), &execRecord); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "executionID": execRecord.ID.Hex(), + "status": execRecord.Status, + "resultSummary": execRecord.ResultSummary, + "errorMessage": execRecord.ErrorMessage, + }) + }) + router.GET("/admin/scheduler/executions", func(c *gin.Context) { + jobs, err := deps.JobConfigStore.List(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + items, err := deps.ExecutionStore.List(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + jobNameMap := make(map[string]string, len(jobs)) + for _, item := range jobs { + jobNameMap[item.ID.Hex()] = item.Name + } + + responseItems := make([]executionItem, 0, len(items)) + for _, item := range items { + responseItems = append(responseItems, executionItem{ + ID: objectIDToString(item.ID), + JobName: jobDisplayName(jobNameMap, item.JobConfigID), + TriggerType: item.TriggerType, + Status: item.Status, + StartedAt: formatTime(item.StartedAt), + FinishedAt: formatTime(item.FinishedAt), + ResultSummary: item.ResultSummary, + }) + } + + c.JSON(http.StatusOK, executionListResponse{ + List: responseItems, + Total: len(responseItems), + }) + }) + router.GET("/admin/scheduler/executions/:id", func(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) + return + } + + item, err := deps.ExecutionStore.GetByID(c.Request.Context(), id) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if item == nil { + c.JSON(http.StatusNotFound, gin.H{"error": "execution not found"}) + return + } + + c.JSON(http.StatusOK, toExecutionDetailItem(*item)) + }) + router.GET("/admin/scheduler/executions/:id/logfile", func(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) + return + } + item, err := deps.ExecutionStore.GetByID(c.Request.Context(), id) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if item == nil || item.LogFile == "" { + c.JSON(http.StatusNotFound, gin.H{"error": "log file not found"}) + return + } + filePath := filepath.Join("logs", "joblog", item.LogFile) + if _, statErr := os.Stat(filePath); os.IsNotExist(statErr) { + c.JSON(http.StatusNotFound, gin.H{"error": "log file not found on disk"}) + return + } + c.FileAttachment(filePath, item.LogFile) + }) + router.GET("/admin/account/profile", func(c *gin.Context) { + profile, err := deps.ProfileStore.Get(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, toProfileItem(*profile)) + }) + router.PUT("/admin/account/profile", func(c *gin.Context) { + current, err := deps.ProfileStore.Get(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + var req updateProfileRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + current.Account = req.Account + current.Nickname = req.Nickname + current.Avatar = req.Avatar + current.Remark = req.Remark + + if err := deps.ProfileStore.Upsert(c.Request.Context(), current); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, toProfileItem(*current)) + }) + router.POST("/admin/account/change-password", func(c *gin.Context) { + var req changeProfilePasswordRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + if err := deps.ProfileStore.ChangePassword(c.Request.Context(), req.OldPassword, req.NewPassword); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"success": true}) + }) + router.GET("/admin/admin-users", func(c *gin.Context) { + items, err := deps.AdminUserStore.List(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + result := make([]adminUserItem, 0, len(items)) + for _, item := range items { + result = append(result, toAdminUserItem(item)) + } + c.JSON(http.StatusOK, adminUserListResponse{List: result, Total: len(result)}) + }) + router.POST("/admin/admin-users", func(c *gin.Context) { + var req upsertAdminUserRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + item := model.AdminUser{ + Account: req.Account, + Nickname: req.Nickname, + Status: req.Status, + Role: req.Role, + Remark: req.Remark, + PasswordHash: modelPasswordHash(req.Password), + } + if item.PasswordHash == "" { + item.PasswordHash = modelPasswordHash("admin123") + } + if err := deps.AdminUserStore.Create(c.Request.Context(), &item); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusCreated, toAdminUserItem(item)) + }) + router.PUT("/admin/admin-users/:id", func(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) + return + } + current, err := deps.AdminUserStore.GetByID(c.Request.Context(), id) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if current == nil { + c.JSON(http.StatusNotFound, gin.H{"error": "admin user not found"}) + return + } + var req upsertAdminUserRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + current.Account = req.Account + current.Nickname = req.Nickname + current.Status = req.Status + current.Role = req.Role + current.Remark = req.Remark + if err := deps.AdminUserStore.Update(c.Request.Context(), current); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, toAdminUserItem(*current)) + }) + router.POST("/admin/admin-users/:id/change-password", func(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) + return + } + var req changeAdminPasswordRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + if err := deps.AdminUserStore.ChangePassword(c.Request.Context(), id, req.NewPassword); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"success": true}) + }) + router.GET("/admin/system-configs", func(c *gin.Context) { + items, err := deps.ConfigStore.List(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + result := make([]systemConfigItem, 0, len(items)) + for _, item := range items { + result = append(result, toSystemConfigItem(item)) + } + c.JSON(http.StatusOK, systemConfigListResponse{List: result, Total: len(result)}) + }) + router.POST("/admin/system-configs", func(c *gin.Context) { + var req upsertSystemConfigRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + item := model.SystemConfig{ + Key: req.Key, + Title: req.Title, + Value: req.Value, + ValueType: req.ValueType, + Description: req.Description, + Enabled: req.Enabled, + } + if err := deps.ConfigStore.Create(c.Request.Context(), &item); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusCreated, toSystemConfigItem(item)) + }) + router.PUT("/admin/system-configs/:id", func(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) + return + } + current, err := deps.ConfigStore.GetByID(c.Request.Context(), id) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if current == nil { + c.JSON(http.StatusNotFound, gin.H{"error": "system config not found"}) + return + } + var req upsertSystemConfigRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + current.Key = req.Key + current.Title = req.Title + current.Value = req.Value + current.ValueType = req.ValueType + current.Description = req.Description + current.Enabled = req.Enabled + if err := deps.ConfigStore.Update(c.Request.Context(), current); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, toSystemConfigItem(*current)) + }) + router.POST("/admin/system-configs/:id/toggle", func(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) + return + } + var req toggleSystemConfigRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + if err := deps.ConfigStore.ToggleEnabled(c.Request.Context(), id, req.Enabled); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"success": true}) + }) + + return router +} + +func toJobItem(item model.JobConfig) jobItem { + return jobItem{ + ID: objectIDToString(item.ID), + Name: item.Name, + HandlerKey: item.HandlerKey, + Enabled: item.Enabled, + ScheduleType: item.ScheduleType, + ScheduleValue: item.ScheduleValue, + DefaultParams: item.DefaultParams, + LastStatus: item.LastStatus, + LastRunAt: formatTime(item.LastRunAt), + NextRunAt: formatTime(item.NextRunAt), + } +} + +func toExecutionDetailItem(item model.JobExecution) executionDetailItem { + return executionDetailItem{ + ID: objectIDToString(item.ID), + JobConfigID: objectIDToString(item.JobConfigID), + TriggerType: item.TriggerType, + Status: item.Status, + ParamsSnapshot: item.ParamsSnapshot, + StartedAt: formatTime(item.StartedAt), + FinishedAt: formatTime(item.FinishedAt), + DurationMs: item.DurationMs, + ResultSummary: item.ResultSummary, + ErrorMessage: item.ErrorMessage, + LogText: item.LogText, + LogFile: item.LogFile, + CreatedAt: item.CreatedAt.Format(time.RFC3339), + } +} + +func toProfileItem(item model.AdminProfile) profileItem { + return profileItem{ + ID: objectIDToString(item.ID), + Account: item.Account, + Nickname: item.Nickname, + Avatar: item.Avatar, + Remark: item.Remark, + CreatedAt: item.CreatedAt.Format(time.RFC3339), + UpdatedAt: item.UpdatedAt.Format(time.RFC3339), + } +} + +func toAdminUserItem(item model.AdminUser) adminUserItem { + return adminUserItem{ + ID: objectIDToString(item.ID), + Account: item.Account, + Nickname: item.Nickname, + Status: item.Status, + Role: item.Role, + Remark: item.Remark, + CreatedAt: item.CreatedAt.Format(time.RFC3339), + UpdatedAt: item.UpdatedAt.Format(time.RFC3339), + } +} + +func toSystemConfigItem(item model.SystemConfig) systemConfigItem { + return systemConfigItem{ + ID: objectIDToString(item.ID), + Key: item.Key, + Title: item.Title, + Value: item.Value, + ValueType: item.ValueType, + Description: item.Description, + Enabled: item.Enabled, + CreatedAt: item.CreatedAt.Format(time.RFC3339), + UpdatedAt: item.UpdatedAt.Format(time.RFC3339), + } +} + +func mergeExecutionParams(defaultParams string, customParams json.RawMessage) (json.RawMessage, error) { + if len(customParams) == 0 { + if defaultParams == "" { + return nil, nil + } + if !json.Valid([]byte(defaultParams)) { + return nil, errInvalidJSON("defaultParams") + } + return json.RawMessage(defaultParams), nil + } + + if !json.Valid(customParams) { + return nil, errInvalidJSON("params") + } + + if defaultParams == "" { + return customParams, nil + } + + if !json.Valid([]byte(defaultParams)) { + return nil, errInvalidJSON("defaultParams") + } + + var base map[string]any + var override map[string]any + + if err := json.Unmarshal([]byte(defaultParams), &base); err != nil { + return nil, errInvalidJSON("defaultParams") + } + if err := json.Unmarshal(customParams, &override); err != nil { + return nil, errInvalidJSON("params") + } + + if base == nil || override == nil { + return customParams, nil + } + + for key, value := range override { + base[key] = value + } + + merged, err := json.Marshal(base) + if err != nil { + return nil, err + } + + return merged, nil +} + +func errInvalidJSON(field string) error { + return fmt.Errorf("%s must be valid JSON", field) +} + +func formatTime(value *time.Time) string { + if value == nil { + return "" + } + + return value.Format(time.RFC3339) +} + +func objectIDToString(value primitive.ObjectID) string { + if value.IsZero() { + return "" + } + + return value.Hex() +} + +func jobDisplayName(jobNameMap map[string]string, id primitive.ObjectID) string { + if name := jobNameMap[id.Hex()]; name != "" { + return name + } + return id.Hex() +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func modelPasswordHash(password string) string { + if password == "" { + return "" + } + sum := sha256.Sum256([]byte(password)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/executor/service.go b/internal/executor/service.go new file mode 100644 index 0000000..e173dcd --- /dev/null +++ b/internal/executor/service.go @@ -0,0 +1,33 @@ +package executor + +import ( + "context" + "fmt" + + "scheduler-backend/internal/jobdef" +) + +type Service struct { + registry *jobdef.Registry + runtime jobdef.Runtime +} + +func NewService(registry *jobdef.Registry, runtime jobdef.Runtime) *Service { + return &Service{ + registry: registry, + runtime: runtime, + } +} + +func (s *Service) Execute(ctx context.Context, handlerKey string, req jobdef.ExecuteRequest) error { + handler, ok := s.registry.Get(handlerKey) + if !ok { + return fmt.Errorf("handler %s not found", handlerKey) + } + + if req.LogCollector == nil { + req.LogCollector = jobdef.NewLogCollector() + } + + return handler.Run(ctx, s.runtime, req) +} diff --git a/internal/executor/service_test.go b/internal/executor/service_test.go new file mode 100644 index 0000000..75d9b21 --- /dev/null +++ b/internal/executor/service_test.go @@ -0,0 +1,28 @@ +package executor + +import ( + "context" + "testing" + + "scheduler-backend/internal/jobdef" + "scheduler-backend/pkg/log" +) + +func TestExecuteRunsRegisteredHandler(t *testing.T) { + registry := jobdef.NewRegistry(jobdef.SampleHandler{}) + service := NewService( + registry, + jobdef.Runtime{ + Logger: log.New(), + }, + ) + + err := service.Execute(context.Background(), "sample-handler", jobdef.ExecuteRequest{ + ExecutionID: "exec-1", + JobID: "job-1", + TriggerType: "manual", + }) + if err != nil { + t.Fatalf("Execute() error = %v", err) + } +} diff --git a/internal/jobdef/handler.go b/internal/jobdef/handler.go new file mode 100644 index 0000000..bb47afa --- /dev/null +++ b/internal/jobdef/handler.go @@ -0,0 +1,103 @@ +package jobdef + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "go.mongodb.org/mongo-driver/mongo" + + "scheduler-backend/pkg/config" + "scheduler-backend/pkg/log" +) + +const logTailLines = 2000 + +type LogCollector struct { + mu sync.Mutex + lines []string +} + +func NewLogCollector() *LogCollector { + return &LogCollector{} +} + +func (c *LogCollector) Appendf(format string, args ...any) { + if c == nil { + return + } + c.mu.Lock() + c.lines = append(c.lines, fmt.Sprintf(format, args...)) + c.mu.Unlock() +} + +func (c *LogCollector) String() string { + if c == nil { + return "" + } + c.mu.Lock() + defer c.mu.Unlock() + return strings.Join(c.lines, "\n") +} + +type FlushResult struct { + LogText string + LogFile string +} + +func (c *LogCollector) Flush(executionID string) FlushResult { + if c == nil { + return FlushResult{} + } + c.mu.Lock() + lines := c.lines + c.mu.Unlock() + + if len(lines) == 0 { + return FlushResult{} + } + + if len(lines) <= logTailLines { + return FlushResult{LogText: strings.Join(lines, "\n")} + } + + filename := executionID + ".log" + dir := filepath.Join("logs", "joblog") + _ = os.MkdirAll(dir, 0o755) + fullPath := filepath.Join(dir, filename) + _ = os.WriteFile(fullPath, []byte(strings.Join(lines, "\n")+"\n"), 0o644) + + tail := lines[len(lines)-logTailLines:] + header := fmt.Sprintf("[完整日志: %s (%d行)]", filename, len(lines)) + + return FlushResult{ + LogText: header + "\n" + strings.Join(tail, "\n"), + LogFile: filename, + } +} + +type ExecuteRequest struct { + ExecutionID string `json:"executionID"` + JobID string `json:"jobID"` + TriggerType string `json:"triggerType"` + Params json.RawMessage `json:"params"` + LogCollector *LogCollector `json:"-"` +} + +type Runtime struct { + Config config.Config + Logger log.Logger + MetaDB *mongo.Database + BusinessDB *mongo.Database +} + +type Handler interface { + Key() string + Name() string + Description() string + Run(ctx context.Context, runtime Runtime, req ExecuteRequest) error +} diff --git a/internal/jobdef/loader.go b/internal/jobdef/loader.go new file mode 100644 index 0000000..98889a8 --- /dev/null +++ b/internal/jobdef/loader.go @@ -0,0 +1,62 @@ +package jobdef + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "scheduler-backend/internal/store/model" + "scheduler-backend/pkg/log" +) + +type JobConfigUpserter interface { + UpsertByHandlerKey(ctx context.Context, item *model.JobConfig) error +} + +type configFile struct { + Name string `json:"name"` + HandlerKey string `json:"handlerKey"` + ScheduleType string `json:"scheduleType"` + ScheduleValue string `json:"scheduleValue"` + DefaultParams string `json:"defaultParams"` +} + +func SyncJobConfigs(ctx context.Context, dir string, store JobConfigUpserter, logger log.Logger) error { + matches, err := filepath.Glob(filepath.Join(dir, "*.json")) + if err != nil { + return fmt.Errorf("glob job config files: %w", err) + } + if len(matches) == 0 { + logger.Info("no job config files found", "dir", dir) + return nil + } + + for _, path := range matches { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + var cfg configFile + if err := json.Unmarshal(data, &cfg); err != nil { + return fmt.Errorf("parse %s: %w", path, err) + } + if cfg.HandlerKey == "" { + return fmt.Errorf("%s: handlerKey is required", path) + } + + item := &model.JobConfig{ + Name: cfg.Name, + HandlerKey: cfg.HandlerKey, + ScheduleType: cfg.ScheduleType, + ScheduleValue: cfg.ScheduleValue, + DefaultParams: cfg.DefaultParams, + } + if err := store.UpsertByHandlerKey(ctx, item); err != nil { + return fmt.Errorf("upsert %s: %w", cfg.HandlerKey, err) + } + logger.Info("synced job config", "handlerKey", cfg.HandlerKey, "name", cfg.Name, "file", filepath.Base(path)) + } + return nil +} diff --git a/internal/jobdef/registry.go b/internal/jobdef/registry.go new file mode 100644 index 0000000..346e683 --- /dev/null +++ b/internal/jobdef/registry.go @@ -0,0 +1,28 @@ +package jobdef + +type Registry struct { + items map[string]Handler +} + +func NewRegistry(handlers ...Handler) *Registry { + items := make(map[string]Handler, len(handlers)) + for _, handler := range handlers { + items[handler.Key()] = handler + } + + return &Registry{items: items} +} + +func (r *Registry) Get(key string) (Handler, bool) { + handler, ok := r.items[key] + return handler, ok +} + +func (r *Registry) List() []Handler { + result := make([]Handler, 0, len(r.items)) + for _, handler := range r.items { + result = append(result, handler) + } + + return result +} diff --git a/internal/jobdef/registry_test.go b/internal/jobdef/registry_test.go new file mode 100644 index 0000000..bc27ce7 --- /dev/null +++ b/internal/jobdef/registry_test.go @@ -0,0 +1,16 @@ +package jobdef + +import "testing" + +func TestRegistryGetReturnsRegisteredHandler(t *testing.T) { + registry := NewRegistry(SampleHandler{}) + + handler, ok := registry.Get("sample-handler") + if !ok { + t.Fatal("expected registered handler") + } + + if handler.Key() != "sample-handler" { + t.Fatalf("handler key = %q, want sample-handler", handler.Key()) + } +} diff --git a/internal/jobdef/s3_migrate_handler.go b/internal/jobdef/s3_migrate_handler.go new file mode 100644 index 0000000..77362a3 --- /dev/null +++ b/internal/jobdef/s3_migrate_handler.go @@ -0,0 +1,44 @@ +package jobdef + +import ( + "context" + "encoding/json" + "fmt" + + "scheduler-backend/internal/s3migrate" +) + +type S3MigrateHandler struct{} + +func (S3MigrateHandler) Key() string { + return "s3-migrate" +} + +func (S3MigrateHandler) Name() string { + return "S3 Migrate" +} + +func (S3MigrateHandler) Description() string { + return "Migrate objects between S3-compatible storage engines (MinIO, AWS S3, etc.). Streams files via presigned URLs and updates MongoDB engine records." +} + +func (S3MigrateHandler) Run(ctx context.Context, runtime Runtime, req ExecuteRequest) error { + var params s3migrate.Params + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + return fmt.Errorf("parse params: %w", err) + } + + logf := func(msg string, args ...any) { + line := fmt.Sprintf(msg, args...) + req.LogCollector.Appendf("%s", line) + runtime.Logger.Info(line, "jobID", req.JobID, "executionID", req.ExecutionID) + } + + if err := s3migrate.Run(ctx, runtime.BusinessDB, params, logf); err != nil { + logf("s3-migrate failed: %v", err) + return fmt.Errorf("s3-migrate failed: %w", err) + } + + logf("s3-migrate finished") + return nil +} diff --git a/internal/jobdef/sample_handler.go b/internal/jobdef/sample_handler.go new file mode 100644 index 0000000..08aa8cc --- /dev/null +++ b/internal/jobdef/sample_handler.go @@ -0,0 +1,31 @@ +package jobdef + +import "context" + +type SampleHandler struct{} + +func (SampleHandler) Key() string { + return "sample-handler" +} + +func (SampleHandler) Name() string { + return "Sample Handler" +} + +func (SampleHandler) Description() string { + return "Writes a sample startup log for scheduler plumbing." +} + +func (SampleHandler) Run(ctx context.Context, runtime Runtime, req ExecuteRequest) error { + req.LogCollector.Appendf("sample handler executed: jobID=%s executionID=%s triggerType=%s", req.JobID, req.ExecutionID, req.TriggerType) + runtime.Logger.Info( + "sample handler executed", + "jobID", + req.JobID, + "executionID", + req.ExecutionID, + "triggerType", + req.TriggerType, + ) + return nil +} diff --git a/internal/jobdef/url_rewrite_handler.go b/internal/jobdef/url_rewrite_handler.go new file mode 100644 index 0000000..483e666 --- /dev/null +++ b/internal/jobdef/url_rewrite_handler.go @@ -0,0 +1,49 @@ +package jobdef + +import ( + "context" + "encoding/json" + "fmt" + + "scheduler-backend/internal/urlrewrite" +) + +type URLRewriteHandler struct{} + +func (URLRewriteHandler) Key() string { + return "url-rewrite" +} + +func (URLRewriteHandler) Name() string { + return "URL Rewrite" +} + +func (URLRewriteHandler) Description() string { + return "Batch-rewrite URL prefixes in MongoDB (user avatars, message media, favorites). Supports dry-run, apply, rollback, and cache invalidation." +} + +func (URLRewriteHandler) Run(ctx context.Context, runtime Runtime, req ExecuteRequest) error { + var params urlrewrite.Params + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + return fmt.Errorf("parse params: %w", err) + } + + logf := func(msg string, args ...any) { + line := fmt.Sprintf(msg, args...) + req.LogCollector.Appendf("%s", line) + runtime.Logger.Info(line, "jobID", req.JobID, "executionID", req.ExecutionID) + } + + redisCfg := urlrewrite.RedisConfig{ + Addr: runtime.Config.RedisAddr, + Password: runtime.Config.RedisPassword, + } + batchID, err := urlrewrite.Run(ctx, runtime.BusinessDB, params, redisCfg, logf) + if err != nil { + logf("url-rewrite failed: %v", err) + return fmt.Errorf("url-rewrite %s failed (batch_id=%s): %w", params.Mode, batchID, err) + } + + logf("url-rewrite finished, mode=%s batch_id=%s", params.Mode, batchID) + return nil +} diff --git a/internal/s3migrate/runner.go b/internal/s3migrate/runner.go new file mode 100644 index 0000000..5b37eec --- /dev/null +++ b/internal/s3migrate/runner.go @@ -0,0 +1,220 @@ +package s3migrate + +import ( + "context" + "fmt" + "io" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const ( + collectionName = "s3" + presignExpiry = time.Hour +) + +type Params struct { + SourceEngine string `json:"sourceEngine"` + SourceEndpoint string `json:"sourceEndpoint"` + SourceBucket string `json:"sourceBucket"` + SourceAccessKey string `json:"sourceAccessKey"` + SourceSecretKey string `json:"sourceSecretKey"` + SourceRegion string `json:"sourceRegion"` + + DestEngine string `json:"destEngine"` + DestEndpoint string `json:"destEndpoint"` + DestBucket string `json:"destBucket"` + DestAccessKey string `json:"destAccessKey"` + DestSecretKey string `json:"destSecretKey"` + DestRegion string `json:"destRegion"` +} + +type objectRecord struct { + Name string `bson:"name"` + Engine string `bson:"engine"` + Key string `bson:"key"` + Size int64 `bson:"size"` + ContentType string `bson:"content_type"` +} + +type LogFunc func(msg string, args ...any) + +func Run(ctx context.Context, db *mongo.Database, params Params, logf LogFunc) error { + if err := validateParams(params); err != nil { + return err + } + if params.SourceRegion == "" { + params.SourceRegion = "us-east-1" + } + if params.DestRegion == "" { + params.DestRegion = "us-east-1" + } + + srcPresign := newPresignClient(params.SourceEndpoint, params.SourceAccessKey, params.SourceSecretKey, params.SourceRegion) + dstPresign := newPresignClient(params.DestEndpoint, params.DestAccessKey, params.DestSecretKey, params.DestRegion) + + coll := db.Collection(collectionName) + count, err := coll.CountDocuments(ctx, bson.M{"engine": params.SourceEngine}) + if err != nil { + return fmt.Errorf("count source objects: %w", err) + } + logf("source engine=%s count=%d", params.SourceEngine, count) + + if count == 0 { + logf("no objects to migrate") + return nil + } + + cursor, err := coll.Find(ctx, bson.M{"engine": params.SourceEngine}, options.Find().SetBatchSize(100)) + if err != nil { + return fmt.Errorf("find source objects: %w", err) + } + defer cursor.Close(ctx) + + var migrated, skipped, failed int64 + for cursor.Next(ctx) { + var obj objectRecord + if err := cursor.Decode(&obj); err != nil { + logf("decode error: %v", err) + failed++ + continue + } + + exists, err := objectExistsInDest(ctx, coll, params.DestEngine, obj.Name) + if err != nil { + logf("check dest error name=%s: %v", obj.Name, err) + failed++ + continue + } + if exists { + skipped++ + continue + } + + if err := copyObject(ctx, srcPresign, params.SourceBucket, dstPresign, params.DestBucket, obj); err != nil { + logf("copy error name=%s: %v", obj.Name, err) + failed++ + continue + } + + if _, err := coll.UpdateOne(ctx, + bson.M{"engine": params.SourceEngine, "name": obj.Name}, + bson.M{"$set": bson.M{"engine": params.DestEngine}}, + ); err != nil { + logf("update engine error name=%s: %v", obj.Name, err) + failed++ + continue + } + + migrated++ + if migrated%100 == 0 { + logf("progress: migrated=%d skipped=%d failed=%d", migrated, skipped, failed) + } + } + + logf("complete: migrated=%d skipped=%d failed=%d total=%d", migrated, skipped, failed, count) + return nil +} + +func validateParams(p Params) error { + if p.SourceEngine == "" { + return fmt.Errorf("sourceEngine is required") + } + if p.DestEngine == "" { + return fmt.Errorf("destEngine is required") + } + if p.SourceEngine == p.DestEngine { + return fmt.Errorf("sourceEngine and destEngine must be different") + } + if p.SourceEndpoint == "" || p.SourceBucket == "" || p.SourceAccessKey == "" || p.SourceSecretKey == "" { + return fmt.Errorf("source S3 config (endpoint, bucket, accessKey, secretKey) is required") + } + if p.DestEndpoint == "" || p.DestBucket == "" || p.DestAccessKey == "" || p.DestSecretKey == "" { + return fmt.Errorf("dest S3 config (endpoint, bucket, accessKey, secretKey) is required") + } + return nil +} + +func newPresignClient(endpoint, accessKey, secretKey, region string) *s3.PresignClient { + cfg := aws.Config{ + Region: region, + Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""), + BaseEndpoint: &endpoint, + } + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = true + }) + return s3.NewPresignClient(client) +} + +func objectExistsInDest(ctx context.Context, coll *mongo.Collection, destEngine, name string) (bool, error) { + count, err := coll.CountDocuments(ctx, bson.M{"engine": destEngine, "name": name}, options.Count().SetLimit(1)) + if err != nil { + return false, err + } + return count > 0, nil +} + +func copyObject(ctx context.Context, srcPresign *s3.PresignClient, srcBucket string, dstPresign *s3.PresignClient, dstBucket string, obj objectRecord) error { + getReq, err := srcPresign.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: &srcBucket, + Key: &obj.Key, + }, s3.WithPresignExpires(presignExpiry)) + if err != nil { + return fmt.Errorf("presign get: %w", err) + } + + contentType := obj.ContentType + putReq, err := dstPresign.PresignPutObject(ctx, &s3.PutObjectInput{ + Bucket: &dstBucket, + Key: &obj.Key, + ContentType: &contentType, + }, s3.WithPresignExpires(presignExpiry)) + if err != nil { + return fmt.Errorf("presign put: %w", err) + } + + downloadResp, err := http.Get(getReq.URL) + if err != nil { + return fmt.Errorf("download: %w", err) + } + defer downloadResp.Body.Close() + + if downloadResp.StatusCode == http.StatusNotFound { + return fmt.Errorf("source object not found: %s", obj.Key) + } + if downloadResp.StatusCode != http.StatusOK { + return fmt.Errorf("download status: %s", downloadResp.Status) + } + + uploadReq, err := http.NewRequestWithContext(ctx, http.MethodPut, putReq.URL, downloadResp.Body) + if err != nil { + return fmt.Errorf("create upload request: %w", err) + } + if downloadResp.ContentLength > 0 { + uploadReq.ContentLength = downloadResp.ContentLength + } else { + uploadReq.ContentLength = obj.Size + } + if contentType != "" { + uploadReq.Header.Set("Content-Type", contentType) + } + + uploadResp, err := http.DefaultClient.Do(uploadReq) + if err != nil { + return fmt.Errorf("upload: %w", err) + } + defer func() { _, _ = io.Copy(io.Discard, uploadResp.Body); uploadResp.Body.Close() }() + + if uploadResp.StatusCode != http.StatusOK { + return fmt.Errorf("upload status: %s", uploadResp.Status) + } + return nil +} diff --git a/internal/scheduler/service.go b/internal/scheduler/service.go new file mode 100644 index 0000000..6c6905e --- /dev/null +++ b/internal/scheduler/service.go @@ -0,0 +1,171 @@ +package scheduler + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/go-co-op/gocron/v2" + "go.mongodb.org/mongo-driver/bson/primitive" + + "scheduler-backend/internal/executor" + "scheduler-backend/internal/jobdef" + "scheduler-backend/internal/store/model" +) + +type JobConfigStore interface { + ListEnabled(ctx context.Context) ([]model.JobConfig, error) + UpdateRunState(ctx context.Context, id primitive.ObjectID, status string, lastRunAt *time.Time) error + UpdateNextRunAt(ctx context.Context, id primitive.ObjectID, nextRunAt *time.Time) error +} + +type JobExecutionStore interface { + Create(ctx context.Context, item *model.JobExecution) error + UpdateResult(ctx context.Context, item *model.JobExecution) error +} + +type Service struct { + scheduler gocron.Scheduler + jobConfigStore JobConfigStore + executionStore JobExecutionStore + executor *executor.Service + runtime jobdef.Runtime +} + +func NewService( + scheduler gocron.Scheduler, + jobConfigStore JobConfigStore, + executionStore JobExecutionStore, + executor *executor.Service, + runtime jobdef.Runtime, +) *Service { + return &Service{ + scheduler: scheduler, + jobConfigStore: jobConfigStore, + executionStore: executionStore, + executor: executor, + runtime: runtime, + } +} + +func (s *Service) RegisterEnabledJobs(ctx context.Context) error { + items, err := s.jobConfigStore.ListEnabled(ctx) + if err != nil { + return err + } + + for _, item := range items { + if err := s.registerJob(ctx, item); err != nil { + return err + } + } + + s.scheduler.Start() + return nil +} + +func (s *Service) Shutdown() error { + return s.scheduler.Shutdown() +} + +func (s *Service) registerJob(ctx context.Context, item model.JobConfig) error { + switch item.ScheduleType { + case "manual": + return s.jobConfigStore.UpdateNextRunAt(ctx, item.ID, nil) + case "cron": + job, err := s.scheduler.NewJob( + gocron.CronJob(item.ScheduleValue, false), + gocron.NewTask(func() { + s.runScheduledJob(context.Background(), item) + }), + ) + if err != nil { + return fmt.Errorf("register cron job %s: %w", item.Name, err) + } + next, err := job.NextRun() + if err != nil { + return err + } + return s.jobConfigStore.UpdateNextRunAt(ctx, item.ID, &next) + case "interval": + duration, err := time.ParseDuration(item.ScheduleValue) + if err != nil { + return fmt.Errorf("parse interval for %s: %w", item.Name, err) + } + job, err := s.scheduler.NewJob( + gocron.DurationJob(duration), + gocron.NewTask(func() { + s.runScheduledJob(context.Background(), item) + }), + ) + if err != nil { + return fmt.Errorf("register interval job %s: %w", item.Name, err) + } + next, err := job.NextRun() + if err != nil { + return err + } + return s.jobConfigStore.UpdateNextRunAt(ctx, item.ID, &next) + default: + return fmt.Errorf("unsupported schedule type %s", item.ScheduleType) + } +} + +func (s *Service) runScheduledJob(ctx context.Context, job model.JobConfig) { + now := time.Now() + params := json.RawMessage(job.DefaultParams) + record := model.JobExecution{ + JobConfigID: job.ID, + TriggerType: "schedule", + Status: "running", + ParamsSnapshot: string(params), + StartedAt: &now, + CreatedAt: now, + } + + if err := s.executionStore.Create(ctx, &record); err != nil { + s.runtime.Logger.Error("create scheduled execution failed", "jobID", job.ID.Hex(), "error", err) + return + } + + execReq := jobdef.ExecuteRequest{ + ExecutionID: record.ID.Hex(), + JobID: job.ID.Hex(), + TriggerType: "schedule", + Params: params, + LogCollector: jobdef.NewLogCollector(), + } + runErr := s.executor.Execute(ctx, job.HandlerKey, execReq) + + finishedAt := time.Now() + record.FinishedAt = &finishedAt + record.DurationMs = finishedAt.Sub(now).Milliseconds() + flushed := execReq.LogCollector.Flush(record.ID.Hex()) + if runErr != nil { + record.Status = "failed" + record.ErrorMessage = runErr.Error() + record.ResultSummary = "scheduled execution failed" + if flushed.LogText != "" { + record.LogText = flushed.LogText + } else { + record.LogText = runErr.Error() + } + record.LogFile = flushed.LogFile + _ = s.jobConfigStore.UpdateRunState(ctx, job.ID, "failed", &finishedAt) + } else { + record.Status = "success" + record.ResultSummary = "scheduled execution succeeded" + if flushed.LogText != "" { + record.LogText = flushed.LogText + } else { + record.LogText = "handler executed successfully" + } + record.LogFile = flushed.LogFile + _ = s.jobConfigStore.UpdateRunState(ctx, job.ID, "success", &finishedAt) + } + + if err := s.executionStore.UpdateResult(ctx, &record); err != nil { + s.runtime.Logger.Error("update scheduled execution failed", "executionID", record.ID.Hex(), "error", err) + } +} diff --git a/internal/store/model/admin_profile.go b/internal/store/model/admin_profile.go new file mode 100644 index 0000000..f67f9ea --- /dev/null +++ b/internal/store/model/admin_profile.go @@ -0,0 +1,18 @@ +package model + +import ( + "time" + + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type AdminProfile struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id"` + Account string `bson:"account" json:"account"` + Nickname string `bson:"nickname" json:"nickname"` + Avatar string `bson:"avatar" json:"avatar"` + Remark string `bson:"remark" json:"remark"` + PasswordHash string `bson:"passwordHash" json:"-"` + CreatedAt time.Time `bson:"createdAt" json:"createdAt"` + UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"` +} diff --git a/internal/store/model/admin_user.go b/internal/store/model/admin_user.go new file mode 100644 index 0000000..0675e4d --- /dev/null +++ b/internal/store/model/admin_user.go @@ -0,0 +1,19 @@ +package model + +import ( + "time" + + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type AdminUser struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id"` + Account string `bson:"account" json:"account"` + Nickname string `bson:"nickname" json:"nickname"` + Status string `bson:"status" json:"status"` + Role string `bson:"role" json:"role"` + Remark string `bson:"remark" json:"remark"` + PasswordHash string `bson:"passwordHash" json:"-"` + CreatedAt time.Time `bson:"createdAt" json:"createdAt"` + UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"` +} diff --git a/internal/store/model/job_config.go b/internal/store/model/job_config.go new file mode 100644 index 0000000..0f62175 --- /dev/null +++ b/internal/store/model/job_config.go @@ -0,0 +1,33 @@ +package model + +import ( + "time" + + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type JobConfig struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id"` + Name string `bson:"name" json:"name"` + HandlerKey string `bson:"handlerKey" json:"handlerKey"` + Enabled bool `bson:"enabled" json:"enabled"` + ScheduleType string `bson:"scheduleType" json:"scheduleType"` + ScheduleValue string `bson:"scheduleValue" json:"scheduleValue"` + DefaultParams string `bson:"defaultParams" json:"defaultParams"` + LastStatus string `bson:"lastStatus" json:"lastStatus"` + LastRunAt *time.Time `bson:"lastRunAt,omitempty" json:"lastRunAt,omitempty"` + NextRunAt *time.Time `bson:"nextRunAt,omitempty" json:"nextRunAt,omitempty"` + CreatedAt time.Time `bson:"createdAt" json:"createdAt"` + UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"` +} + +func NewJobConfig(name, handlerKey string) JobConfig { + now := time.Now() + return JobConfig{ + Name: name, + HandlerKey: handlerKey, + LastStatus: "idle", + CreatedAt: now, + UpdatedAt: now, + } +} diff --git a/internal/store/model/job_config_test.go b/internal/store/model/job_config_test.go new file mode 100644 index 0000000..bb1a375 --- /dev/null +++ b/internal/store/model/job_config_test.go @@ -0,0 +1,10 @@ +package model + +import "testing" + +func TestJobConfigDefaultsToIdleStatus(t *testing.T) { + job := NewJobConfig("daily-check", "daily-check") + if job.LastStatus != "idle" { + t.Fatalf("LastStatus = %q, want idle", job.LastStatus) + } +} diff --git a/internal/store/model/job_execution.go b/internal/store/model/job_execution.go new file mode 100644 index 0000000..1cca18d --- /dev/null +++ b/internal/store/model/job_execution.go @@ -0,0 +1,23 @@ +package model + +import ( + "time" + + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type JobExecution struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id"` + JobConfigID primitive.ObjectID `bson:"jobConfigId" json:"jobConfigId"` + TriggerType string `bson:"triggerType" json:"triggerType"` + Status string `bson:"status" json:"status"` + ParamsSnapshot string `bson:"paramsSnapshot" json:"paramsSnapshot"` + StartedAt *time.Time `bson:"startedAt,omitempty" json:"startedAt,omitempty"` + FinishedAt *time.Time `bson:"finishedAt,omitempty" json:"finishedAt,omitempty"` + DurationMs int64 `bson:"durationMs" json:"durationMs"` + ResultSummary string `bson:"resultSummary" json:"resultSummary"` + ErrorMessage string `bson:"errorMessage" json:"errorMessage"` + LogText string `bson:"logText" json:"logText"` + LogFile string `bson:"logFile,omitempty" json:"logFile,omitempty"` + CreatedAt time.Time `bson:"createdAt" json:"createdAt"` +} diff --git a/internal/store/model/system_config.go b/internal/store/model/system_config.go new file mode 100644 index 0000000..01928e3 --- /dev/null +++ b/internal/store/model/system_config.go @@ -0,0 +1,19 @@ +package model + +import ( + "time" + + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type SystemConfig struct { + ID primitive.ObjectID `bson:"_id,omitempty" json:"id"` + Key string `bson:"key" json:"key"` + Title string `bson:"title" json:"title"` + Value string `bson:"value" json:"value"` + ValueType string `bson:"valueType" json:"valueType"` + Description string `bson:"description" json:"description"` + Enabled bool `bson:"enabled" json:"enabled"` + CreatedAt time.Time `bson:"createdAt" json:"createdAt"` + UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"` +} diff --git a/internal/store/mongo/admin_profile_store.go b/internal/store/mongo/admin_profile_store.go new file mode 100644 index 0000000..0c12ed7 --- /dev/null +++ b/internal/store/mongo/admin_profile_store.go @@ -0,0 +1,125 @@ +package mongo + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongodriver "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "scheduler-backend/internal/store/model" +) + +const defaultAdminPassword = "admin123" + +type AdminProfileStore struct { + collection *mongodriver.Collection +} + +func NewAdminProfileStore(db *mongodriver.Database) *AdminProfileStore { + return &AdminProfileStore{ + collection: db.Collection("admin_profiles"), + } +} + +func (s *AdminProfileStore) Get(ctx context.Context) (*model.AdminProfile, error) { + findCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var item model.AdminProfile + err := s.collection.FindOne(findCtx, bson.D{}, options.FindOne().SetSort(bson.D{{Key: "updatedAt", Value: -1}})).Decode(&item) + if err == nil { + return &item, nil + } + if !errors.Is(err, mongodriver.ErrNoDocuments) { + return nil, err + } + + now := time.Now() + item = model.AdminProfile{ + ID: primitive.NewObjectID(), + Account: "admin", + Nickname: "调度管理员", + Avatar: "", + Remark: "default admin profile", + PasswordHash: hashPassword(defaultAdminPassword), + CreatedAt: now, + UpdatedAt: now, + } + if _, err := s.collection.InsertOne(findCtx, item); err != nil { + return nil, err + } + return &item, nil +} + +func (s *AdminProfileStore) Upsert(ctx context.Context, profile *model.AdminProfile) error { + updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + now := time.Now() + profile.UpdatedAt = now + if profile.CreatedAt.IsZero() { + profile.CreatedAt = now + } + + if profile.ID.IsZero() { + current, err := s.Get(ctx) + if err != nil { + return err + } + profile.ID = current.ID + if profile.PasswordHash == "" { + profile.PasswordHash = current.PasswordHash + } + if profile.CreatedAt.IsZero() { + profile.CreatedAt = current.CreatedAt + } + } + + _, err := s.collection.UpdateByID(updateCtx, profile.ID, bson.M{ + "$set": bson.M{ + "account": profile.Account, + "nickname": profile.Nickname, + "avatar": profile.Avatar, + "remark": profile.Remark, + "passwordHash": profile.PasswordHash, + "createdAt": profile.CreatedAt, + "updatedAt": profile.UpdatedAt, + }, + }, options.Update().SetUpsert(true)) + return err +} + +func (s *AdminProfileStore) ChangePassword(ctx context.Context, oldPassword, newPassword string) error { + current, err := s.Get(ctx) + if err != nil { + return err + } + if current.PasswordHash != hashPassword(oldPassword) { + return ErrInvalidPassword + } + + current.PasswordHash = hashPassword(newPassword) + return s.Upsert(ctx, current) +} + +func (s *AdminProfileStore) Authenticate(ctx context.Context, account, password string) (*model.AdminProfile, error) { + current, err := s.Get(ctx) + if err != nil { + return nil, err + } + if current.Account != account || current.PasswordHash != hashPassword(password) { + return nil, ErrInvalidPassword + } + return current, nil +} + +func hashPassword(password string) string { + sum := sha256.Sum256([]byte(password)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/store/mongo/admin_user_store.go b/internal/store/mongo/admin_user_store.go new file mode 100644 index 0000000..a5ef6ce --- /dev/null +++ b/internal/store/mongo/admin_user_store.go @@ -0,0 +1,152 @@ +package mongo + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongodriver "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "scheduler-backend/internal/store/model" +) + +type AdminUserStore struct { + collection *mongodriver.Collection +} + +func NewAdminUserStore(db *mongodriver.Database) *AdminUserStore { + return &AdminUserStore{ + collection: db.Collection("admin_users"), + } +} + +func (s *AdminUserStore) List(ctx context.Context) ([]model.AdminUser, error) { + findCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + cursor, err := s.collection.Find(findCtx, bson.D{}, options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}})) + if err != nil { + return nil, err + } + defer cursor.Close(findCtx) + + items := make([]model.AdminUser, 0) + for cursor.Next(findCtx) { + var item model.AdminUser + if err := cursor.Decode(&item); err != nil { + return nil, err + } + items = append(items, item) + } + if err := cursor.Err(); err != nil { + return nil, err + } + + if len(items) == 0 { + defaultAdmin := model.AdminUser{ + Account: "admin", + Nickname: "调度管理员", + Status: "active", + Role: "super_admin", + Remark: "default admin user", + PasswordHash: hashPassword(defaultAdminPassword), + } + if err := s.Create(ctx, &defaultAdmin); err != nil { + return nil, err + } + items = append(items, defaultAdmin) + } + + return items, nil +} + +func (s *AdminUserStore) GetByID(ctx context.Context, id primitive.ObjectID) (*model.AdminUser, error) { + findCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var item model.AdminUser + err := s.collection.FindOne(findCtx, bson.M{"_id": id}).Decode(&item) + if err != nil { + if errors.Is(err, mongodriver.ErrNoDocuments) { + return nil, nil + } + return nil, err + } + return &item, nil +} + +func (s *AdminUserStore) Create(ctx context.Context, item *model.AdminUser) error { + insertCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + now := time.Now() + item.CreatedAt = now + item.UpdatedAt = now + if item.Status == "" { + item.Status = "active" + } + if item.Role == "" { + item.Role = "super_admin" + } + + result, err := s.collection.InsertOne(insertCtx, item) + if err != nil { + return err + } + if oid, ok := result.InsertedID.(primitive.ObjectID); ok { + item.ID = oid + } + return nil +} + +func (s *AdminUserStore) Update(ctx context.Context, item *model.AdminUser) error { + updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + item.UpdatedAt = time.Now() + _, err := s.collection.UpdateByID(updateCtx, item.ID, bson.M{ + "$set": bson.M{ + "account": item.Account, + "nickname": item.Nickname, + "status": item.Status, + "role": item.Role, + "remark": item.Remark, + "updatedAt": item.UpdatedAt, + }, + }) + return err +} + +func (s *AdminUserStore) ChangePassword(ctx context.Context, id primitive.ObjectID, newPassword string) error { + updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := s.collection.UpdateByID(updateCtx, id, bson.M{ + "$set": bson.M{ + "passwordHash": hashPassword(newPassword), + "updatedAt": time.Now(), + }, + }) + return err +} + +func (s *AdminUserStore) Authenticate(ctx context.Context, account, password string) (*model.AdminUser, error) { + findCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var item model.AdminUser + err := s.collection.FindOne(findCtx, bson.M{"account": account}).Decode(&item) + if err != nil { + if errors.Is(err, mongodriver.ErrNoDocuments) { + return nil, ErrInvalidPassword + } + return nil, err + } + if item.Status == "disabled" || item.PasswordHash != hashPassword(password) { + return nil, ErrInvalidPassword + } + return &item, nil +} diff --git a/internal/store/mongo/client.go b/internal/store/mongo/client.go new file mode 100644 index 0000000..5d0adaa --- /dev/null +++ b/internal/store/mongo/client.go @@ -0,0 +1,37 @@ +package mongo + +import ( + "context" + "time" + + mongodriver "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "scheduler-backend/pkg/config" +) + +type Databases struct { + Client *mongodriver.Client + MetaDB *mongodriver.Database + BusinessDB *mongodriver.Database +} + +func Connect(ctx context.Context, cfg config.Config) (*Databases, error) { + client, err := mongodriver.Connect(ctx, options.Client().ApplyURI(cfg.MongoURI())) + if err != nil { + return nil, err + } + + pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + if err := client.Ping(pingCtx, nil); err != nil { + return nil, err + } + + return &Databases{ + Client: client, + MetaDB: client.Database(cfg.SchedulerMongoDatabase), + BusinessDB: client.Database(cfg.BusinessMongoDatabase), + }, nil +} diff --git a/internal/store/mongo/errors.go b/internal/store/mongo/errors.go new file mode 100644 index 0000000..fa57f87 --- /dev/null +++ b/internal/store/mongo/errors.go @@ -0,0 +1,5 @@ +package mongo + +import "errors" + +var ErrInvalidPassword = errors.New("invalid password") diff --git a/internal/store/mongo/job_config_store.go b/internal/store/mongo/job_config_store.go new file mode 100644 index 0000000..f5d8002 --- /dev/null +++ b/internal/store/mongo/job_config_store.go @@ -0,0 +1,222 @@ +package mongo + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + mongodriver "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/bson/primitive" + + "scheduler-backend/internal/store/model" +) + +type JobConfigStore struct { + collection *mongodriver.Collection +} + +func NewJobConfigStore(db *mongodriver.Database) *JobConfigStore { + return &JobConfigStore{ + collection: db.Collection("job_configs"), + } +} + +func (s *JobConfigStore) List(ctx context.Context) ([]model.JobConfig, error) { + findCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + cursor, err := s.collection.Find( + findCtx, + bson.D{}, + options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}), + ) + if err != nil { + return nil, err + } + defer cursor.Close(findCtx) + + items := make([]model.JobConfig, 0) + for cursor.Next(findCtx) { + var item model.JobConfig + if err := cursor.Decode(&item); err != nil { + return nil, err + } + items = append(items, item) + } + + if err := cursor.Err(); err != nil { + return nil, err + } + + return items, nil +} + +func (s *JobConfigStore) ListEnabled(ctx context.Context) ([]model.JobConfig, error) { + findCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + cursor, err := s.collection.Find( + findCtx, + bson.M{"enabled": true}, + options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}), + ) + if err != nil { + return nil, err + } + defer cursor.Close(findCtx) + + items := make([]model.JobConfig, 0) + for cursor.Next(findCtx) { + var item model.JobConfig + if err := cursor.Decode(&item); err != nil { + return nil, err + } + items = append(items, item) + } + + if err := cursor.Err(); err != nil { + return nil, err + } + + return items, nil +} + +func (s *JobConfigStore) GetByID(ctx context.Context, id primitive.ObjectID) (*model.JobConfig, error) { + findCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var item model.JobConfig + err := s.collection.FindOne(findCtx, bson.M{"_id": id}).Decode(&item) + if err != nil { + if errors.Is(err, mongodriver.ErrNoDocuments) { + return nil, nil + } + return nil, err + } + + return &item, nil +} + +func (s *JobConfigStore) Create(ctx context.Context, item *model.JobConfig) error { + insertCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + now := time.Now() + item.CreatedAt = now + item.UpdatedAt = now + if item.LastStatus == "" { + item.LastStatus = "idle" + } + + result, err := s.collection.InsertOne(insertCtx, item) + if err != nil { + return err + } + + if oid, ok := result.InsertedID.(primitive.ObjectID); ok { + item.ID = oid + } + + return nil +} + +func (s *JobConfigStore) Update(ctx context.Context, item *model.JobConfig) error { + updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + item.UpdatedAt = time.Now() + update := bson.M{ + "$set": bson.M{ + "name": item.Name, + "handlerKey": item.HandlerKey, + "enabled": item.Enabled, + "scheduleType": item.ScheduleType, + "scheduleValue": item.ScheduleValue, + "defaultParams": item.DefaultParams, + "updatedAt": item.UpdatedAt, + }, + } + + _, err := s.collection.UpdateByID(updateCtx, item.ID, update) + return err +} + +func (s *JobConfigStore) ToggleEnabled(ctx context.Context, id primitive.ObjectID, enabled bool) error { + updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := s.collection.UpdateByID(updateCtx, id, bson.M{ + "$set": bson.M{ + "enabled": enabled, + "updatedAt": time.Now(), + }, + }) + return err +} + +func (s *JobConfigStore) UpdateRunState(ctx context.Context, id primitive.ObjectID, status string, lastRunAt *time.Time) error { + updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + set := bson.M{ + "lastStatus": status, + "updatedAt": time.Now(), + } + if lastRunAt != nil { + set["lastRunAt"] = *lastRunAt + } + + _, err := s.collection.UpdateByID(updateCtx, id, bson.M{ + "$set": set, + }) + return err +} + +func (s *JobConfigStore) UpsertByHandlerKey(ctx context.Context, item *model.JobConfig) error { + upsertCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + now := time.Now() + _, err := s.collection.UpdateOne( + upsertCtx, + bson.M{"handlerKey": item.HandlerKey}, + bson.M{ + "$set": bson.M{ + "name": item.Name, + "handlerKey": item.HandlerKey, + "scheduleType": item.ScheduleType, + "scheduleValue": item.ScheduleValue, + "defaultParams": item.DefaultParams, + "updatedAt": now, + }, + "$setOnInsert": bson.M{ + "enabled": false, + "lastStatus": "idle", + "createdAt": now, + }, + }, + options.Update().SetUpsert(true), + ) + return err +} + +func (s *JobConfigStore) UpdateNextRunAt(ctx context.Context, id primitive.ObjectID, nextRunAt *time.Time) error { + updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + set := bson.M{ + "updatedAt": time.Now(), + } + if nextRunAt != nil { + set["nextRunAt"] = *nextRunAt + } else { + set["nextRunAt"] = nil + } + + _, err := s.collection.UpdateByID(updateCtx, id, bson.M{ + "$set": set, + }) + return err +} diff --git a/internal/store/mongo/job_execution_store.go b/internal/store/mongo/job_execution_store.go new file mode 100644 index 0000000..0cdebd2 --- /dev/null +++ b/internal/store/mongo/job_execution_store.go @@ -0,0 +1,109 @@ +package mongo + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongodriver "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "scheduler-backend/internal/store/model" +) + +type JobExecutionStore struct { + collection *mongodriver.Collection +} + +func NewJobExecutionStore(db *mongodriver.Database) *JobExecutionStore { + return &JobExecutionStore{ + collection: db.Collection("job_executions"), + } +} + +func (s *JobExecutionStore) List(ctx context.Context) ([]model.JobExecution, error) { + findCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + cursor, err := s.collection.Find( + findCtx, + bson.D{}, + options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}), + ) + if err != nil { + return nil, err + } + defer cursor.Close(findCtx) + + items := make([]model.JobExecution, 0) + for cursor.Next(findCtx) { + var item model.JobExecution + if err := cursor.Decode(&item); err != nil { + return nil, err + } + items = append(items, item) + } + + if err := cursor.Err(); err != nil { + return nil, err + } + + return items, nil +} + +func (s *JobExecutionStore) GetByID(ctx context.Context, id primitive.ObjectID) (*model.JobExecution, error) { + findCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var item model.JobExecution + err := s.collection.FindOne(findCtx, bson.M{"_id": id}).Decode(&item) + if err != nil { + if errors.Is(err, mongodriver.ErrNoDocuments) { + return nil, nil + } + return nil, err + } + + return &item, nil +} + +func (s *JobExecutionStore) Create(ctx context.Context, item *model.JobExecution) error { + insertCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + if item.CreatedAt.IsZero() { + item.CreatedAt = time.Now() + } + + result, err := s.collection.InsertOne(insertCtx, item) + if err != nil { + return err + } + + if oid, ok := result.InsertedID.(primitive.ObjectID); ok { + item.ID = oid + } + + return nil +} + +func (s *JobExecutionStore) UpdateResult(ctx context.Context, item *model.JobExecution) error { + updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := s.collection.UpdateByID(updateCtx, item.ID, bson.M{ + "$set": bson.M{ + "status": item.Status, + "startedAt": item.StartedAt, + "finishedAt": item.FinishedAt, + "durationMs": item.DurationMs, + "resultSummary": item.ResultSummary, + "errorMessage": item.ErrorMessage, + "logText": item.LogText, + "logFile": item.LogFile, + }, + }) + return err +} diff --git a/internal/store/mongo/system_config_store.go b/internal/store/mongo/system_config_store.go new file mode 100644 index 0000000..3ff5be1 --- /dev/null +++ b/internal/store/mongo/system_config_store.go @@ -0,0 +1,113 @@ +package mongo + +import ( + "context" + "errors" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongodriver "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "scheduler-backend/internal/store/model" +) + +type SystemConfigStore struct { + collection *mongodriver.Collection +} + +func NewSystemConfigStore(db *mongodriver.Database) *SystemConfigStore { + return &SystemConfigStore{ + collection: db.Collection("system_configs"), + } +} + +func (s *SystemConfigStore) List(ctx context.Context) ([]model.SystemConfig, error) { + findCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + cursor, err := s.collection.Find(findCtx, bson.D{}, options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}})) + if err != nil { + return nil, err + } + defer cursor.Close(findCtx) + + items := make([]model.SystemConfig, 0) + for cursor.Next(findCtx) { + var item model.SystemConfig + if err := cursor.Decode(&item); err != nil { + return nil, err + } + items = append(items, item) + } + if err := cursor.Err(); err != nil { + return nil, err + } + return items, nil +} + +func (s *SystemConfigStore) GetByID(ctx context.Context, id primitive.ObjectID) (*model.SystemConfig, error) { + findCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var item model.SystemConfig + err := s.collection.FindOne(findCtx, bson.M{"_id": id}).Decode(&item) + if err != nil { + if errors.Is(err, mongodriver.ErrNoDocuments) { + return nil, nil + } + return nil, err + } + return &item, nil +} + +func (s *SystemConfigStore) Create(ctx context.Context, item *model.SystemConfig) error { + insertCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + now := time.Now() + item.CreatedAt = now + item.UpdatedAt = now + + result, err := s.collection.InsertOne(insertCtx, item) + if err != nil { + return err + } + if oid, ok := result.InsertedID.(primitive.ObjectID); ok { + item.ID = oid + } + return nil +} + +func (s *SystemConfigStore) Update(ctx context.Context, item *model.SystemConfig) error { + updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + item.UpdatedAt = time.Now() + _, err := s.collection.UpdateByID(updateCtx, item.ID, bson.M{ + "$set": bson.M{ + "key": item.Key, + "title": item.Title, + "value": item.Value, + "valueType": item.ValueType, + "description": item.Description, + "enabled": item.Enabled, + "updatedAt": item.UpdatedAt, + }, + }) + return err +} + +func (s *SystemConfigStore) ToggleEnabled(ctx context.Context, id primitive.ObjectID, enabled bool) error { + updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := s.collection.UpdateByID(updateCtx, id, bson.M{ + "$set": bson.M{ + "enabled": enabled, + "updatedAt": time.Now(), + }, + }) + return err +} diff --git a/internal/urlrewrite/redis.go b/internal/urlrewrite/redis.go new file mode 100644 index 0000000..344ff1c --- /dev/null +++ b/internal/urlrewrite/redis.go @@ -0,0 +1,18 @@ +package urlrewrite + +import ( + "context" + + "github.com/redis/go-redis/v9" +) + +func newRedisClient(ctx context.Context, addr, password string) (*redis.Client, error) { + rdb := redis.NewClient(&redis.Options{ + Addr: addr, + Password: password, + }) + if err := rdb.Ping(ctx).Err(); err != nil { + return nil, err + } + return rdb, nil +} diff --git a/internal/urlrewrite/rewrite.go b/internal/urlrewrite/rewrite.go new file mode 100644 index 0000000..17fd425 --- /dev/null +++ b/internal/urlrewrite/rewrite.go @@ -0,0 +1,65 @@ +package urlrewrite + +import "encoding/json" + +func rewriteStringValue(input, oldPrefix, newPrefix string) (string, bool) { + if oldPrefix == "" || newPrefix == "" { + return input, false + } + if len(input) >= len(oldPrefix) && input[:len(oldPrefix)] == oldPrefix { + return newPrefix + input[len(oldPrefix):], true + } + return input, false +} + +func rewriteJSONContent(input, oldPrefix, newPrefix string) (string, bool, error) { + var data any + if err := json.Unmarshal([]byte(input), &data); err != nil { + rewritten, changed := rewriteStringValue(input, oldPrefix, newPrefix) + if changed { + return rewritten, true, nil + } + return "", false, err + } + rewritten, changed := rewriteAny(data, oldPrefix, newPrefix) + if !changed { + return input, false, nil + } + out, err := json.Marshal(rewritten) + if err != nil { + return "", false, err + } + return string(out), true, nil +} + +func rewriteAny(value any, oldPrefix, newPrefix string) (any, bool) { + switch v := value.(type) { + case map[string]any: + changed := false + for key, child := range v { + newChild, childChanged := rewriteAny(child, oldPrefix, newPrefix) + if childChanged { + v[key] = newChild + changed = true + } + } + return v, changed + case []any: + changed := false + for i, child := range v { + newChild, childChanged := rewriteAny(child, oldPrefix, newPrefix) + if childChanged { + v[i] = newChild + changed = true + } + } + return v, changed + case string: + if rewritten, changed, err := rewriteJSONContent(v, oldPrefix, newPrefix); err == nil && changed { + return rewritten, true + } + return rewriteStringValue(v, oldPrefix, newPrefix) + default: + return value, false + } +} diff --git a/internal/urlrewrite/rewrite_test.go b/internal/urlrewrite/rewrite_test.go new file mode 100644 index 0000000..8ecd1b5 --- /dev/null +++ b/internal/urlrewrite/rewrite_test.go @@ -0,0 +1,170 @@ +package urlrewrite + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestRewriteStringValue(t *testing.T) { + oldPrefix := "https://s3.jizhying.com/images/" + newPrefix := "https://dp9pkdckmd09t.cloudfront.net/" + + tests := []struct { + name string + input string + want string + }{ + { + name: "replace matching prefix", + input: "https://s3.jizhying.com/images/openim/data/hash/a.jpg", + want: "https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/a.jpg", + }, + { + name: "keep non matching url", + input: "https://example.com/a.jpg", + want: "https://example.com/a.jpg", + }, + { + name: "keep plain text", + input: "hello", + want: "hello", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, changed := rewriteStringValue(tt.input, oldPrefix, newPrefix) + if got != tt.want { + t.Fatalf("rewriteStringValue() = %q, want %q", got, tt.want) + } + if (tt.input != tt.want) != changed { + t.Fatalf("rewriteStringValue() changed = %v, want %v", changed, tt.input != tt.want) + } + }) + } +} + +func TestRewriteJSONContent(t *testing.T) { + oldPrefix := "https://s3.jizhying.com/images/" + newPrefix := "https://dp9pkdckmd09t.cloudfront.net/" + + input := `{"sourcePicture":{"url":"https://s3.jizhying.com/images/openim/data/hash/src.jpg"},"snapshotPicture":{"url":"https://s3.jizhying.com/images/openim/data/hash/snap.jpg"},"nested":{"fileElem":{"sourceUrl":"https://s3.jizhying.com/images/openim/data/hash/file.zip"}}}` + want := `{"nested":{"fileElem":{"sourceUrl":"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/file.zip"}},"snapshotPicture":{"url":"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/snap.jpg"},"sourcePicture":{"url":"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/src.jpg"}}` + + got, changed, err := rewriteJSONContent(input, oldPrefix, newPrefix) + if err != nil { + t.Fatalf("rewriteJSONContent() error = %v", err) + } + if !changed { + t.Fatalf("rewriteJSONContent() changed = false, want true") + } + if got != want { + t.Fatalf("rewriteJSONContent() = %s, want %s", got, want) + } +} + +func TestRewriteJSONContentNestedJSONString(t *testing.T) { + oldPrefix := "https://s3.jizhying.com/images/" + newPrefix := "https://dp9pkdckmd09t.cloudfront.net/" + + input := `{"detail":"{\"group\":{\"faceURL\":\"https://s3.jizhying.com/images/openim/data/hash/group.jpg\"},\"entrantUser\":{\"faceURL\":\"https://s3.jizhying.com/images/openim/data/hash/user.jpg\"}}"}` + want := `{"detail":"{\"group\":{\"faceURL\":\"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/group.jpg\"},\"entrantUser\":{\"faceURL\":\"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/user.jpg\"}}"}` + + got, changed, err := rewriteJSONContent(input, oldPrefix, newPrefix) + if err != nil { + t.Fatalf("rewriteJSONContent() error = %v", err) + } + if !changed { + t.Fatalf("rewriteJSONContent() changed = false, want true") + } + var gotValue map[string]any + var wantValue map[string]any + if err := json.Unmarshal([]byte(got), &gotValue); err != nil { + t.Fatalf("unmarshal got error = %v", err) + } + if err := json.Unmarshal([]byte(want), &wantValue); err != nil { + t.Fatalf("unmarshal want error = %v", err) + } + var gotDetail any + var wantDetail any + if err := json.Unmarshal([]byte(gotValue["detail"].(string)), &gotDetail); err != nil { + t.Fatalf("unmarshal got detail error = %v", err) + } + if err := json.Unmarshal([]byte(wantValue["detail"].(string)), &wantDetail); err != nil { + t.Fatalf("unmarshal want detail error = %v", err) + } + gotValue["detail"] = gotDetail + wantValue["detail"] = wantDetail + if !reflect.DeepEqual(gotValue, wantValue) { + t.Fatalf("rewriteJSONContent() = %#v, want %#v", gotValue, wantValue) + } +} + +func TestRewriteJSONContentInvalidJSON(t *testing.T) { + _, changed, err := rewriteJSONContent("not-json", "https://old/", "https://new/") + if err == nil { + t.Fatalf("rewriteJSONContent() error = nil, want non-nil") + } + if changed { + t.Fatalf("rewriteJSONContent() changed = true, want false") + } +} + +func TestCollectMsgFieldUpdates(t *testing.T) { + oldPrefix := "https://s3.jizhying.com/images/" + newPrefix := "https://dp9pkdckmd09t.cloudfront.net/" + + doc := msgDoc{ + Doc: "si_a_b:0", + Msgs: []msgEntryModel{ + { + Msg: &msgPayload{ + Content: `{"sourcePicture":{"url":"https://s3.jizhying.com/images/openim/data/hash/1.jpg"}}`, + SenderFaceURL: "https://s3.jizhying.com/images/openim/data/hash/avatar.jpg", + }, + }, + { + Msg: &msgPayload{ + Content: `{"text":"hello"}`, + SenderFaceURL: "", + }, + }, + }, + } + + updates, changed, samples := collectMsgFieldUpdates(doc, oldPrefix, newPrefix, 5) + if !changed { + t.Fatalf("collectMsgFieldUpdates() changed = false, want true") + } + if len(samples) != 2 { + t.Fatalf("collectMsgFieldUpdates() samples len = %d, want 2", len(samples)) + } + if len(updates) != 2 { + t.Fatalf("collectMsgFieldUpdates() updates len = %d, want 2", len(updates)) + } + if updates[0].Path != "msgs.0.msg.content" { + t.Fatalf("first update path = %s", updates[0].Path) + } + if updates[0].NewValue != `{"sourcePicture":{"url":"https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/1.jpg"}}` { + t.Fatalf("content update = %s", updates[0].NewValue) + } + if updates[1].Path != "msgs.0.msg.sender_face_url" { + t.Fatalf("second update path = %s", updates[1].Path) + } + if updates[1].NewValue != "https://dp9pkdckmd09t.cloudfront.net/openim/data/hash/avatar.jpg" { + t.Fatalf("sender_face_url update = %s", updates[1].NewValue) + } +} + +func TestBuildBackupDocs(t *testing.T) { + updates := []fieldUpdate{ + {Path: "face_url", OldValue: "https://old/a.jpg", NewValue: "https://new/a.jpg"}, + {Path: "thumbnail", OldValue: "https://old/b.jpg", NewValue: "https://new/b.jpg"}, + } + + docs := buildBackupDocs("batch-1", "attributes", "doc-1", updates) + if len(docs) != 2 { + t.Fatalf("buildBackupDocs() len = %d, want 2", len(docs)) + } +} diff --git a/internal/urlrewrite/runner.go b/internal/urlrewrite/runner.go new file mode 100644 index 0000000..b9f164e --- /dev/null +++ b/internal/urlrewrite/runner.go @@ -0,0 +1,500 @@ +package urlrewrite + +import ( + "context" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const ( + backupCollectionName = "url_rewrite_backups" + msgCachePrefix = "MSG_CACHE:" + + ModeDryRun = "dry-run" + ModeApply = "apply" + ModeRollback = "rollback" + ModeInvalidateCache = "invalidate-cache" +) + +type Params struct { + OldPrefix string `json:"oldPrefix"` + NewPrefix string `json:"newPrefix"` + Mode string `json:"mode"` + BatchID string `json:"batchId"` + SampleSize int `json:"sampleSize"` +} + +type RedisConfig struct { + Addr string + Password string +} + +type Summary struct { + Collection string + Scanned int64 + Updated int64 + Samples []string +} + +type fieldUpdate struct { + Path string + OldValue string + NewValue string +} + +type msgDoc struct { + ID any `bson:"_id"` + Doc string `bson:"doc_id"` + Msgs []msgEntryModel `bson:"msgs"` +} + +type msgEntryModel struct { + Msg *msgPayload `bson:"msg"` +} + +type msgPayload struct { + Content string `bson:"content"` + SenderFaceURL string `bson:"sender_face_url"` + Seq int64 `bson:"seq"` +} + +type LogFunc func(msg string, args ...any) + +func Run(ctx context.Context, db *mongo.Database, params Params, redisCfg RedisConfig, logf LogFunc) (string, error) { + if err := validateParams(params); err != nil { + return "", err + } + if params.SampleSize <= 0 { + params.SampleSize = 5 + } + if params.Mode == "" { + params.Mode = ModeDryRun + } + if params.Mode == ModeApply && params.BatchID == "" { + params.BatchID = "urlrewrite-" + time.Now().Format("20060102150405") + } + + backupColl := db.Collection(backupCollectionName) + + if params.Mode == ModeRollback { + return params.BatchID, rollbackBatch(ctx, backupColl, db, params, logf) + } + if params.Mode == ModeInvalidateCache { + return params.BatchID, invalidateBatchCache(ctx, db, redisCfg, params, logf) + } + + summaries := []Summary{ + rewriteSimpleFieldCollection(ctx, db.Collection("wallets"), backupColl, "real_name_auth.id_card_photo_front", params), + rewriteSimpleFieldCollection(ctx, db.Collection("wallets"), backupColl, "real_name_auth.id_card_photo_back", params), + rewriteSimpleFieldCollection(ctx, db.Collection("favorites"), backupColl, "content", params), + rewriteSimpleFieldCollection(ctx, db.Collection("favorites"), backupColl, "thumbnail", params), + rewriteSimpleFieldCollection(ctx, db.Collection("favorites"), backupColl, "link_url", params), + rewriteSimpleFieldCollection(ctx, db.Collection("attributes"), backupColl, "face_url", params), + rewriteSimpleFieldCollection(ctx, db.Collection("attribute"), backupColl, "face_url", params), + rewriteSimpleFieldCollection(ctx, db.Collection("user"), backupColl, "face_url", params), + rewriteSimpleFieldCollection(ctx, db.Collection("group"), backupColl, "face_url", params), + rewriteSimpleFieldCollection(ctx, db.Collection("group_member"), backupColl, "face_url", params), + } + + msgSummary, err := rewriteMsgCollection(ctx, db.Collection("msg"), backupColl, params) + if err != nil { + return "", fmt.Errorf("rewrite msg collection: %w", err) + } + summaries = append(summaries, msgSummary) + + for _, s := range summaries { + logf("[%s] scanned=%d updated=%d", s.Collection, s.Scanned, s.Updated) + for _, sample := range s.Samples { + logf("[%s] sample: %s", s.Collection, sample) + } + } + + switch params.Mode { + case ModeDryRun: + logf("dry-run complete, rerun with mode=apply to write changes") + case ModeApply: + logf("apply complete, batch_id=%s", params.BatchID) + } + return params.BatchID, nil +} + +func validateParams(p Params) error { + switch p.Mode { + case ModeDryRun, ModeApply, ModeRollback, ModeInvalidateCache: + case "": + default: + return fmt.Errorf("invalid mode: %s", p.Mode) + } + if p.Mode == ModeRollback || p.Mode == ModeInvalidateCache { + if p.BatchID == "" { + return fmt.Errorf("batchId is required for %s", p.Mode) + } + } else { + if p.OldPrefix == "" || p.NewPrefix == "" { + return fmt.Errorf("oldPrefix and newPrefix are required") + } + } + return nil +} + +func invalidateBatchCache(ctx context.Context, db *mongo.Database, redisCfg RedisConfig, params Params, logf LogFunc) error { + if redisCfg.Addr == "" { + return fmt.Errorf("REDIS_ADDR is not configured") + } + backupColl := db.Collection(backupCollectionName) + cursor, err := backupColl.Find(ctx, bson.M{ + "batch_id": params.BatchID, + "collection": "msg", + "field": bson.M{"$regex": primitive.Regex{Pattern: `^msgs\.\d+\.msg\.`}}, + }) + if err != nil { + return fmt.Errorf("find backup docs: %w", err) + } + defer cursor.Close(ctx) + + type backupDoc struct { + DocumentID any `bson:"document_id"` + Field string `bson:"field"` + } + + docIndexes := make(map[string]map[int]struct{}) + for cursor.Next(ctx) { + var item backupDoc + if err := cursor.Decode(&item); err != nil { + return fmt.Errorf("decode backup doc: %w", err) + } + docID, ok := item.DocumentID.(primitive.ObjectID) + if !ok { + continue + } + msgIndex, ok := parseMsgFieldIndex(item.Field) + if !ok { + continue + } + id := docID.Hex() + if _, exists := docIndexes[id]; !exists { + docIndexes[id] = make(map[int]struct{}) + } + docIndexes[id][msgIndex] = struct{}{} + } + + msgColl := db.Collection("msg") + var keys []string + for hexID, indexes := range docIndexes { + docObjectID, err := primitive.ObjectIDFromHex(hexID) + if err != nil { + return fmt.Errorf("parse object id %s: %w", hexID, err) + } + var doc msgDoc + if err := msgColl.FindOne(ctx, bson.M{"_id": docObjectID}).Decode(&doc); err != nil { + return fmt.Errorf("find msg doc %s: %w", hexID, err) + } + conversationID := trimDocIDSuffix(doc.Doc) + if conversationID == "" { + continue + } + for idx := range indexes { + if idx < 0 || idx >= len(doc.Msgs) || doc.Msgs[idx].Msg == nil || doc.Msgs[idx].Msg.Seq <= 0 { + continue + } + keys = append(keys, msgCachePrefix+conversationID+":"+strconv.Itoa(int(doc.Msgs[idx].Msg.Seq))) + } + } + + keys = deduplicateStrings(keys) + if len(keys) == 0 { + logf("invalidate-cache complete, batch_id=%s keys=0", params.BatchID) + return nil + } + + rdb, err := newRedisClient(ctx, redisCfg.Addr, redisCfg.Password) + if err != nil { + return fmt.Errorf("connect redis: %w", err) + } + defer rdb.Close() + + if err := rdb.Del(ctx, keys...).Err(); err != nil { + return fmt.Errorf("redis del: %w", err) + } + logf("invalidate-cache complete, batch_id=%s keys=%d", params.BatchID, len(keys)) + return nil +} + +func rewriteSimpleFieldCollection(ctx context.Context, coll, backupColl *mongo.Collection, field string, params Params) Summary { + filter := bson.M{field: bson.M{"$regex": primitive.Regex{Pattern: "^" + regexp.QuoteMeta(params.OldPrefix)}}} + cursor, err := coll.Find(ctx, filter) + if err != nil { + return Summary{Collection: coll.Name() + "." + field, Samples: []string{fmt.Sprintf("find error: %v", err)}} + } + defer cursor.Close(ctx) + + summary := Summary{Collection: coll.Name() + "." + field} + for cursor.Next(ctx) { + summary.Scanned++ + var doc bson.M + if err := cursor.Decode(&doc); err != nil { + if len(summary.Samples) < params.SampleSize { + summary.Samples = append(summary.Samples, fmt.Sprintf("decode error: %v", err)) + } + continue + } + original, ok := nestedString(doc, field) + if !ok { + continue + } + rewritten, changed := rewriteStringValue(original, params.OldPrefix, params.NewPrefix) + if !changed { + continue + } + updates := []fieldUpdate{{ + Path: field, + OldValue: original, + NewValue: rewritten, + }} + summary.Updated++ + if len(summary.Samples) < params.SampleSize { + summary.Samples = append(summary.Samples, fmt.Sprintf("%s -> %s", original, rewritten)) + } + if params.Mode != ModeApply { + continue + } + if err := insertBackupDocs(ctx, backupColl, params.BatchID, coll.Name(), doc["_id"], updates); err != nil { + if len(summary.Samples) < params.SampleSize { + summary.Samples = append(summary.Samples, fmt.Sprintf("backup error: %v", err)) + } + continue + } + if _, err := coll.UpdateByID(ctx, doc["_id"], bson.M{"$set": bson.M{field: rewritten}}); err != nil { + if len(summary.Samples) < params.SampleSize { + summary.Samples = append(summary.Samples, fmt.Sprintf("update error: %v", err)) + } + } + } + return summary +} + +func rewriteMsgCollection(ctx context.Context, coll, backupColl *mongo.Collection, params Params) (Summary, error) { + cursor, err := coll.Find(ctx, bson.M{ + "$or": bson.A{ + bson.M{"msgs.msg.content": bson.M{"$regex": primitive.Regex{Pattern: regexp.QuoteMeta(params.OldPrefix)}}}, + bson.M{"msgs.msg.sender_face_url": bson.M{"$regex": primitive.Regex{Pattern: "^" + regexp.QuoteMeta(params.OldPrefix)}}}, + }, + }) + if err != nil { + return Summary{}, fmt.Errorf("find msg docs: %w", err) + } + defer cursor.Close(ctx) + + summary := Summary{Collection: coll.Name()} + for cursor.Next(ctx) { + summary.Scanned++ + var doc msgDoc + if err := cursor.Decode(&doc); err != nil { + if len(summary.Samples) < params.SampleSize { + summary.Samples = append(summary.Samples, fmt.Sprintf("decode error: %v", err)) + } + continue + } + updates, changed, samples := collectMsgFieldUpdates(doc, params.OldPrefix, params.NewPrefix, params.SampleSize-len(summary.Samples)) + if !changed { + continue + } + summary.Updated++ + summary.Samples = append(summary.Samples, samples...) + if params.Mode != ModeApply { + continue + } + if err := insertBackupDocs(ctx, backupColl, params.BatchID, coll.Name(), doc.ID, updates); err != nil { + if len(summary.Samples) < params.SampleSize { + summary.Samples = append(summary.Samples, fmt.Sprintf("backup error: %v", err)) + } + continue + } + if _, err := coll.UpdateByID(ctx, doc.ID, bson.M{"$set": toUpdateMap(updates)}); err != nil { + return summary, fmt.Errorf("update msg doc: %w", err) + } + } + return summary, nil +} + +func collectMsgFieldUpdates(doc msgDoc, oldPrefix, newPrefix string, sampleBudget int) ([]fieldUpdate, bool, []string) { + var updates []fieldUpdate + var samples []string + for i := range doc.Msgs { + if doc.Msgs[i].Msg == nil { + continue + } + if doc.Msgs[i].Msg.Content != "" { + rewritten, contentChanged, err := rewriteJSONContent(doc.Msgs[i].Msg.Content, oldPrefix, newPrefix) + if err == nil && contentChanged { + updates = append(updates, fieldUpdate{ + Path: fmt.Sprintf("msgs.%d.msg.content", i), + OldValue: doc.Msgs[i].Msg.Content, + NewValue: rewritten, + }) + if len(samples) < sampleBudget { + samples = append(samples, fmt.Sprintf("doc=%s msgs.%d.msg.content updated", doc.Doc, i)) + } + } + } + if doc.Msgs[i].Msg.SenderFaceURL != "" { + rewritten, faceChanged := rewriteStringValue(doc.Msgs[i].Msg.SenderFaceURL, oldPrefix, newPrefix) + if faceChanged { + updates = append(updates, fieldUpdate{ + Path: fmt.Sprintf("msgs.%d.msg.sender_face_url", i), + OldValue: doc.Msgs[i].Msg.SenderFaceURL, + NewValue: rewritten, + }) + if len(samples) < sampleBudget { + samples = append(samples, fmt.Sprintf("doc=%s msgs.%d.msg.sender_face_url updated", doc.Doc, i)) + } + } + } + } + return updates, len(updates) > 0, samples +} + +func toUpdateMap(updates []fieldUpdate) bson.M { + sets := bson.M{} + for _, u := range updates { + sets[u.Path] = u.NewValue + } + return sets +} + +func buildBackupDocs(batchID, collection string, documentID any, updates []fieldUpdate) []any { + now := time.Now() + docs := make([]any, 0, len(updates)) + for _, u := range updates { + docs = append(docs, bson.M{ + "batch_id": batchID, + "collection": collection, + "document_id": documentID, + "field": u.Path, + "old_value": u.OldValue, + "new_value": u.NewValue, + "created_at": now, + }) + } + return docs +} + +func insertBackupDocs(ctx context.Context, backupColl *mongo.Collection, batchID, collection string, documentID any, updates []fieldUpdate) error { + if backupColl == nil || len(updates) == 0 { + return nil + } + _, err := backupColl.InsertMany(ctx, buildBackupDocs(batchID, collection, documentID, updates)) + return err +} + +func rollbackBatch(ctx context.Context, backupColl *mongo.Collection, db *mongo.Database, params Params, logf LogFunc) error { + cursor, err := backupColl.Find(ctx, bson.M{"batch_id": params.BatchID}, options.Find().SetSort(bson.D{ + {Key: "collection", Value: 1}, + {Key: "document_id", Value: 1}, + {Key: "field", Value: 1}, + })) + if err != nil { + return fmt.Errorf("find backup docs: %w", err) + } + defer cursor.Close(ctx) + + type rollbackDoc struct { + Collection string `bson:"collection"` + Field string `bson:"field"` + OldValue string `bson:"old_value"` + DocumentID any `bson:"document_id"` + } + + grouped := map[string]map[any][]fieldUpdate{} + for cursor.Next(ctx) { + var doc rollbackDoc + if err := cursor.Decode(&doc); err != nil { + return fmt.Errorf("decode backup doc: %w", err) + } + if _, ok := grouped[doc.Collection]; !ok { + grouped[doc.Collection] = map[any][]fieldUpdate{} + } + grouped[doc.Collection][doc.DocumentID] = append(grouped[doc.Collection][doc.DocumentID], fieldUpdate{ + Path: doc.Field, + NewValue: doc.OldValue, + }) + } + if len(grouped) == 0 { + return fmt.Errorf("no backup records found for batch_id %s", params.BatchID) + } + + var collectionNames []string + for name := range grouped { + collectionNames = append(collectionNames, name) + } + sort.Strings(collectionNames) + + for _, name := range collectionNames { + coll := db.Collection(name) + for documentID, updates := range grouped[name] { + if _, err := coll.UpdateByID(ctx, documentID, bson.M{"$set": toUpdateMap(updates)}); err != nil { + return fmt.Errorf("rollback %s: %w", name, err) + } + } + } + logf("rollback complete, batch_id=%s", params.BatchID) + return nil +} + +func parseMsgFieldIndex(path string) (int, bool) { + parts := strings.Split(path, ".") + if len(parts) < 4 || parts[0] != "msgs" { + return 0, false + } + idx, err := strconv.Atoi(parts[1]) + if err != nil { + return 0, false + } + return idx, true +} + +func trimDocIDSuffix(docID string) string { + pos := strings.LastIndex(docID, ":") + if pos <= 0 { + return docID + } + return docID[:pos] +} + +func deduplicateStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + result := make([]string, 0, len(values)) + for _, v := range values { + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + result = append(result, v) + } + return result +} + +func nestedString(doc bson.M, path string) (string, bool) { + current := any(doc) + for _, segment := range strings.Split(path, ".") { + switch m := current.(type) { + case bson.M: + current = m[segment] + case map[string]any: + current = m[segment] + default: + return "", false + } + } + val, ok := current.(string) + return val, ok +} diff --git a/job-config-list/s3-migrate.json b/job-config-list/s3-migrate.json new file mode 100644 index 0000000..9edc478 --- /dev/null +++ b/job-config-list/s3-migrate.json @@ -0,0 +1,7 @@ +{ + "name": "S3对象迁移", + "handlerKey": "s3-migrate", + "scheduleType": "manual", + "scheduleValue": "", + "defaultParams": "{\"sourceEngine\":\"minio\",\"sourceEndpoint\":\"http://minio:9000\",\"sourceBucket\":\"openim\",\"sourceAccessKey\":\"\",\"sourceSecretKey\":\"\",\"destEngine\":\"aws\",\"destEndpoint\":\"https://s3.ap-southeast-1.amazonaws.com\",\"destBucket\":\"openim\",\"destAccessKey\":\"\",\"destSecretKey\":\"\",\"destRegion\":\"ap-southeast-1\"}" +} diff --git a/job-config-list/sample-handler.json b/job-config-list/sample-handler.json new file mode 100644 index 0000000..ec0b24d --- /dev/null +++ b/job-config-list/sample-handler.json @@ -0,0 +1,7 @@ +{ + "name": "示例任务", + "handlerKey": "sample-handler", + "scheduleType": "manual", + "scheduleValue": "", + "defaultParams": "{\"limit\":10,\"dryRun\":true}" +} diff --git a/job-config-list/url-rewrite.json b/job-config-list/url-rewrite.json new file mode 100644 index 0000000..02c0d51 --- /dev/null +++ b/job-config-list/url-rewrite.json @@ -0,0 +1,7 @@ +{ + "name": "URL前缀替换", + "handlerKey": "url-rewrite", + "scheduleType": "manual", + "scheduleValue": "", + "defaultParams": "{\"oldPrefix\":\"https://s3.jizhying.com/images/\",\"newPrefix\":\"https://dp9pkdckmd09t.cloudfront.net/\",\"mode\":\"dry-run\",\"sampleSize\":10}" +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..828e96d --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,127 @@ +package config + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +type Config struct { + HTTPPort int + WorkerHTTPPort int + MongoHost string + MongoPort int + MongoUsername string + MongoPassword string + MongoAuthSource string + BusinessMongoDatabase string + SchedulerMongoDatabase string + RedisAddr string + RedisPassword string +} + +func Load() Config { + loadEnvFiles() + + return Config{ + HTTPPort: getenvInt("HTTP_PORT", 10090), + WorkerHTTPPort: getenvInt("WORKER_HTTP_PORT", 10091), + MongoHost: getenv("MONGO_HOST", "127.0.0.1"), + MongoPort: getenvInt("MONGO_PORT", 27017), + MongoUsername: getenv("MONGO_USERNAME", ""), + MongoPassword: getenv("MONGO_PASSWORD", ""), + MongoAuthSource: getenv("MONGO_AUTHSOURCE", "admin"), + BusinessMongoDatabase: getenv("MONGO_DATABASE", "openim_v3"), + SchedulerMongoDatabase: getenv("SCHEDULER_MONGO_DATABASE", "scheduler_center"), + RedisAddr: getenv("REDIS_ADDR", ""), + RedisPassword: getenv("REDIS_PASSWORD", ""), + } +} + +func loadEnvFiles() { + loadEnvFile(".env") + + if envName := currentEnvName(); envName != "" { + loadEnvFile(".env." + envName) + } + + loadEnvFile(".env.local") +} + +func currentEnvName() string { + for _, key := range []string{"APP_ENV", "GO_ENV", "ENV"} { + if value := strings.TrimSpace(os.Getenv(key)); value != "" { + return value + } + } + + return "" +} + +func loadEnvFile(name string) { + file, err := os.Open(filepath.Clean(name)) + if err != nil { + return + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + key, value, ok := strings.Cut(line, "=") + if !ok { + continue + } + + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + if key == "" { + continue + } + + if _, exists := os.LookupEnv(key); exists { + continue + } + + _ = os.Setenv(key, value) + } +} + +func (c Config) MongoURI() string { + return fmt.Sprintf( + "mongodb://%s:%s@%s:%d/?authSource=%s", + c.MongoUsername, + c.MongoPassword, + c.MongoHost, + c.MongoPort, + c.MongoAuthSource, + ) +} + +func getenv(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} + +func getenvInt(key string, fallback int) int { + raw := getenv(key, "") + if raw == "" { + return fallback + } + + value, err := strconv.Atoi(raw) + if err != nil { + return fallback + } + + return value +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 0000000..a2bcd1a --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,21 @@ +package config + +import "testing" + +func TestMongoURI(t *testing.T) { + cfg := Config{ + MongoHost: "127.0.0.1", + MongoPort: 27017, + MongoUsername: "user", + MongoPassword: "pass", + MongoAuthSource: "admin", + BusinessMongoDatabase: "openim_v3", + SchedulerMongoDatabase: "scheduler_center", + } + + got := cfg.MongoURI() + want := "mongodb://user:pass@127.0.0.1:27017/?authSource=admin" + if got != want { + t.Fatalf("MongoURI() = %q, want %q", got, want) + } +} diff --git a/pkg/log/log.go b/pkg/log/log.go new file mode 100644 index 0000000..9f04451 --- /dev/null +++ b/pkg/log/log.go @@ -0,0 +1,12 @@ +package log + +import ( + "log/slog" + "os" +) + +type Logger = *slog.Logger + +func New() Logger { + return slog.New(slog.NewTextHandler(os.Stdout, nil)) +} diff --git a/scripts/seed-sample-job.sh b/scripts/seed-sample-job.sh new file mode 100755 index 0000000..90565e0 --- /dev/null +++ b/scripts/seed-sample-job.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." +GOWORK=off go run ./cmd/tools/seed-sample-job diff --git a/scripts/start-api.sh b/scripts/start-api.sh new file mode 100755 index 0000000..2969d86 --- /dev/null +++ b/scripts/start-api.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." +GOWORK=off go run ./cmd/api diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100755 index 0000000..b61077a --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." +GOWORK=off go run ./cmd/worker diff --git a/seed-sample-job b/seed-sample-job new file mode 100755 index 0000000..3dc3ca1 Binary files /dev/null and b/seed-sample-job differ diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..929f3d6 --- /dev/null +++ b/start.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")" && pwd)" +RUN_DIR="$ROOT_DIR/run" +LOG_DIR="$ROOT_DIR/logs" +API_PID_FILE="$RUN_DIR/api.pid" +WORKER_PID_FILE="$RUN_DIR/worker.pid" +API_LOG_FILE="$LOG_DIR/api.log" +WORKER_LOG_FILE="$LOG_DIR/worker.log" +API_BIN="$RUN_DIR/api" +WORKER_BIN="$RUN_DIR/worker" +HTTP_PORT=16811 +WORKER_HTTP_PORT=16812 + +mkdir -p "$RUN_DIR" "$LOG_DIR" + +load_env_config() { + local env_file + + for env_file in ".env"; do + if [[ -f "$ROOT_DIR/$env_file" ]]; then + # shellcheck disable=SC1090 + source "$ROOT_DIR/$env_file" + fi + done + + if [[ -n "${APP_ENV:-}" && -f "$ROOT_DIR/.env.$APP_ENV" ]]; then + # shellcheck disable=SC1090 + source "$ROOT_DIR/.env.$APP_ENV" + elif [[ -n "${GO_ENV:-}" && -f "$ROOT_DIR/.env.$GO_ENV" ]]; then + # shellcheck disable=SC1090 + source "$ROOT_DIR/.env.$GO_ENV" + elif [[ -n "${ENV:-}" && -f "$ROOT_DIR/.env.$ENV" ]]; then + # shellcheck disable=SC1090 + source "$ROOT_DIR/.env.$ENV" + fi + + if [[ -f "$ROOT_DIR/.env.local" ]]; then + # shellcheck disable=SC1090 + source "$ROOT_DIR/.env.local" + fi +} + +port_pid() { + local port="$1" + lsof -tiTCP:"$port" -sTCP:LISTEN 2>/dev/null | head -n 1 || true +} + +is_scheduler_process() { + local pid="$1" + local cmd + + cmd="$(ps -p "$pid" -o command= 2>/dev/null || true)" + [[ "$cmd" == *"/scheduler-backend/run/api"* || "$cmd" == *"/scheduler-backend/run/worker"* || "$cmd" == *"go run ./cmd/api"* || "$cmd" == *"go run ./cmd/worker"* || "$cmd" == *"/scheduler-backend/api"* || "$cmd" == *"/scheduler-backend/worker"* || "$cmd" == *"/exe/api"* || "$cmd" == *"/exe/worker"* ]] +} + +is_running() { + local pid_file="$1" + if [[ ! -f "$pid_file" ]]; then + return 1 + fi + + local pid + pid="$(cat "$pid_file")" + if [[ -z "$pid" ]]; then + return 1 + fi + + kill -0 "$pid" 2>/dev/null +} + +ensure_port_free() { + local name="$1" + local port="$2" + local pid + + pid="$(port_pid "$port")" + if [[ -z "$pid" ]]; then + return 0 + fi + + if is_scheduler_process "$pid"; then + kill "$pid" 2>/dev/null || true + sleep 1 + pid="$(port_pid "$port")" + if [[ -z "$pid" ]]; then + echo "$name previous process stopped, port=$port" + return 0 + fi + fi + + echo "$name port $port is already occupied by pid=$pid" >&2 + ps -p "$pid" -o pid,ppid,command >&2 || true + return 1 +} + +start_process() { + local name="$1" + local pid_file="$2" + local log_file="$3" + shift 3 + + if is_running "$pid_file"; then + echo "$name is already running, restarting pid=$(cat "$pid_file")" + kill "$(cat "$pid_file")" 2>/dev/null || true + sleep 1 + fi + + rm -f "$pid_file" + ( + cd "$ROOT_DIR" + nohup "$@" >>"$log_file" 2>&1 & + echo $! >"$pid_file" + ) + + sleep 1 + if is_running "$pid_file"; then + echo "$name started, pid=$(cat "$pid_file"), log=$log_file" + return 0 + fi + + echo "failed to start $name, check $log_file" >&2 + rm -f "$pid_file" + return 1 +} + +build_binary() { + local output="$1" + local target="$2" + + ( + cd "$ROOT_DIR" + GOWORK=off go build -o "$output" "$target" + ) +} + +stop_process() { + local name="$1" + local pid_file="$2" + + if ! is_running "$pid_file"; then + rm -f "$pid_file" + return 0 + fi + + kill "$(cat "$pid_file")" 2>/dev/null || true + rm -f "$pid_file" + echo "$name stopped" +} + +build_binary "$API_BIN" ./cmd/api +build_binary "$WORKER_BIN" ./cmd/worker + +load_env_config +ensure_port_free "api" "${HTTP_PORT}" +ensure_port_free "worker" "${WORKER_HTTP_PORT}" + +start_process "api" "$API_PID_FILE" "$API_LOG_FILE" "$API_BIN" +if ! start_process "worker" "$WORKER_PID_FILE" "$WORKER_LOG_FILE" "$WORKER_BIN"; then + stop_process "api" "$API_PID_FILE" + exit 1 +fi + +echo "scheduler-backend started" +echo "api pid: $(cat "$API_PID_FILE")" +echo "worker pid: $(cat "$WORKER_PID_FILE")" +echo "api log: $API_LOG_FILE" +echo "worker log: $WORKER_LOG_FILE" diff --git a/stop.sh b/stop.sh new file mode 100755 index 0000000..c46af17 --- /dev/null +++ b/stop.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")" && pwd)" +RUN_DIR="$ROOT_DIR/run" +API_PID_FILE="$RUN_DIR/api.pid" +WORKER_PID_FILE="$RUN_DIR/worker.pid" +HTTP_PORT=16811 +WORKER_HTTP_PORT=16812 + +load_env_config() { + local env_file + + for env_file in ".env"; do + if [[ -f "$ROOT_DIR/$env_file" ]]; then + # shellcheck disable=SC1090 + source "$ROOT_DIR/$env_file" + fi + done + + if [[ -n "${APP_ENV:-}" && -f "$ROOT_DIR/.env.$APP_ENV" ]]; then + # shellcheck disable=SC1090 + source "$ROOT_DIR/.env.$APP_ENV" + elif [[ -n "${GO_ENV:-}" && -f "$ROOT_DIR/.env.$GO_ENV" ]]; then + # shellcheck disable=SC1090 + source "$ROOT_DIR/.env.$GO_ENV" + elif [[ -n "${ENV:-}" && -f "$ROOT_DIR/.env.$ENV" ]]; then + # shellcheck disable=SC1090 + source "$ROOT_DIR/.env.$ENV" + fi + + if [[ -f "$ROOT_DIR/.env.local" ]]; then + # shellcheck disable=SC1090 + source "$ROOT_DIR/.env.local" + fi +} + +port_pid() { + local port="$1" + lsof -tiTCP:"$port" -sTCP:LISTEN 2>/dev/null | head -n 1 || true +} + +stop_port_process() { + local name="$1" + local port="$2" + local pid + + pid="$(port_pid "$port")" + if [[ -z "$pid" ]]; then + return 0 + fi + + kill "$pid" 2>/dev/null || true + echo "$name stopped by port, pid=$pid" +} + +stop_process() { + local name="$1" + local pid_file="$2" + + if [[ ! -f "$pid_file" ]]; then + echo "$name is not running" + return 0 + fi + + local pid + pid="$(cat "$pid_file")" + if [[ -z "$pid" ]]; then + rm -f "$pid_file" + echo "$name pid file is empty, cleaned" + return 0 + fi + + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" + echo "$name stopped, pid=$pid" + else + echo "$name is not running, stale pid=$pid" + fi + + rm -f "$pid_file" +} + +stop_process "worker" "$WORKER_PID_FILE" +stop_process "api" "$API_PID_FILE" +load_env_config +stop_port_process "worker" "${WORKER_HTTP_PORT}" +stop_port_process "api" "${HTTP_PORT}" diff --git a/worker b/worker new file mode 100755 index 0000000..9ec1e49 Binary files /dev/null and b/worker differ