复制项目

2026-01-14 22:16:44 +08:00
parent e2577b8cee
commit e50142a3b9
691 changed files with 97009 additions and 1 deletions
--- a/tools/README.md
+++ b/tools/README.md
@@ -0,0 +1,25 @@
+# Notes about go workspace
+
+As openim is using go1.18's [workspace feature](https://go.dev/doc/tutorial/workspaces), once you add a new module, you need to run `go work use -r .` at root directory to update the workspace synced.
+
+### Create a new extensions
+
+1. Create your tools_name directory in pkg `/tools` first and cd into it.
+2. Init the project.
+3. Then `go work use -r .` at current directory to update the workspace.
+4. Create your tools
+
+You can execute the following commands to do things above:
+
+```bash
+# edit the CRD_NAME and CRD_GROUP to your own
+export OPENIM_TOOLS_NAME=<Changeme>
+
+# copy and paste to create a new CRD and Controller
+mkdir tools/${OPENIM_TOOLS_NAME}
+cd tools/${OPENIM_TOOLS_NAME}
+go mod init github.com/openimsdk/open-im-server-deploy/tools/${OPENIM_TOOLS_NAME}
+go mod tidy
+go work use -r .
+cd ../..
+```
--- a/tools/changelog/changelog.go
+++ b/tools/changelog/changelog.go
@@ -0,0 +1,198 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+)
+
+// You can specify a tag as a command line argument to generate the changelog for a specific version.
+// Example: go run tools/changelog/changelog.go v0.0.33
+// If no tag is provided, the latest release will be used.
+
+// Setting repo owner and repo name by generate changelog
+const (
+	repoOwner = "openimsdk"
+	repoName  = "open-im-server-deploy"
+)
+
+// GitHubRepo struct represents the repo details.
+type GitHubRepo struct {
+	Owner         string
+	Repo          string
+	FullChangelog string
+}
+
+// ReleaseData represents the JSON structure for release data.
+type ReleaseData struct {
+	TagName   string `json:"tag_name"`
+	Body      string `json:"body"`
+	HtmlUrl   string `json:"html_url"`
+	Published string `json:"published_at"`
+}
+
+// Method to classify and format release notes.
+func (g *GitHubRepo) classifyReleaseNotes(body string) map[string][]string {
+	result := map[string][]string{
+		"feat":     {},
+		"fix":      {},
+		"chore":    {},
+		"refactor": {},
+		"build":    {},
+		"other":    {},
+	}
+
+	// Regular expression to extract PR number and URL (case insensitive)
+	rePR := regexp.MustCompile(`(?i)in (https://github\.com/[^\s]+/pull/(\d+))`)
+
+	// Split the body into individual lines.
+	lines := strings.Split(body, "\n")
+
+	for _, line := range lines {
+		// Skip lines that contain "deps: Merge"
+		if strings.Contains(strings.ToLower(line), "deps: merge #") {
+			continue
+		}
+
+		// Use a regular expression to extract Full Changelog link and its title (case insensitive).
+		if strings.Contains(strings.ToLower(line), "**full changelog**") {
+			matches := regexp.MustCompile(`(?i)\*\*full changelog\*\*: (https://github\.com/[^\s]+/compare/([^\s]+))`).FindStringSubmatch(line)
+			if len(matches) > 2 {
+				// Format the Full Changelog link with title
+				g.FullChangelog = fmt.Sprintf("[%s](%s)", matches[2], matches[1])
+			}
+			continue // Skip further processing for this line.
+		}
+
+		if strings.HasPrefix(line, "*") {
+			var category string
+
+			// Use strings.ToLower to make the matching case insensitive
+			lowerLine := strings.ToLower(line)
+
+			// Determine the category based on the prefix (case insensitive).
+			if strings.HasPrefix(lowerLine, "* feat") {
+				category = "feat"
+			} else if strings.HasPrefix(lowerLine, "* fix") {
+				category = "fix"
+			} else if strings.HasPrefix(lowerLine, "* chore") {
+				category = "chore"
+			} else if strings.HasPrefix(lowerLine, "* refactor") {
+				category = "refactor"
+			} else if strings.HasPrefix(lowerLine, "* build") {
+				category = "build"
+			} else {
+				category = "other"
+			}
+
+			// Extract PR number and URL (case insensitive)
+			matches := rePR.FindStringSubmatch(line)
+			if len(matches) == 3 {
+				prURL := matches[1]
+				prNumber := matches[2]
+				// Format the line with the PR link and use original content for the final result
+				formattedLine := fmt.Sprintf("* %s [#%s](%s)", strings.Split(line, " by ")[0][2:], prNumber, prURL)
+				result[category] = append(result[category], formattedLine)
+			} else {
+				// If no PR link is found, just add the line as is
+				result[category] = append(result[category], line)
+			}
+		}
+	}
+
+	return result
+}
+
+// Method to generate the final changelog.
+func (g *GitHubRepo) generateChangelog(tag, date, htmlURL, body string) string {
+	sections := g.classifyReleaseNotes(body)
+
+	// Convert ISO 8601 date to simpler format (YYYY-MM-DD)
+	formattedDate := date[:10]
+
+	// Changelog header with tag, date, and links.
+	changelog := fmt.Sprintf("## [%s](%s) \t(%s)\n\n", tag, htmlURL, formattedDate)
+
+	if len(sections["feat"]) > 0 {
+		changelog += "### New Features\n" + strings.Join(sections["feat"], "\n") + "\n\n"
+	}
+	if len(sections["fix"]) > 0 {
+		changelog += "### Bug Fixes\n" + strings.Join(sections["fix"], "\n") + "\n\n"
+	}
+	if len(sections["chore"]) > 0 {
+		changelog += "### Chores\n" + strings.Join(sections["chore"], "\n") + "\n\n"
+	}
+	if len(sections["refactor"]) > 0 {
+		changelog += "### Refactors\n" + strings.Join(sections["refactor"], "\n") + "\n\n"
+	}
+	if len(sections["build"]) > 0 {
+		changelog += "### Builds\n" + strings.Join(sections["build"], "\n") + "\n\n"
+	}
+	if len(sections["other"]) > 0 {
+		changelog += "### Others\n" + strings.Join(sections["other"], "\n") + "\n\n"
+	}
+
+	if g.FullChangelog != "" {
+		changelog += fmt.Sprintf("**Full Changelog**: %s\n", g.FullChangelog)
+	}
+
+	return changelog
+}
+
+// Method to fetch release data from GitHub API.
+func (g *GitHubRepo) fetchReleaseData(version string) (*ReleaseData, error) {
+	var apiURL string
+
+	if version == "" {
+		// Fetch the latest release.
+		apiURL = fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", g.Owner, g.Repo)
+	} else {
+		// Fetch a specific version.
+		apiURL = fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/tags/%s", g.Owner, g.Repo, version)
+	}
+
+	resp, err := http.Get(apiURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var releaseData ReleaseData
+	err = json.Unmarshal(body, &releaseData)
+	if err != nil {
+		return nil, err
+	}
+
+	return &releaseData, nil
+}
+
+func main() {
+	repo := &GitHubRepo{Owner: repoOwner, Repo: repoName}
+
+	// Get the version from command line arguments, if provided
+	var version string // Default is use latest
+
+	if len(os.Args) > 1 {
+		version = os.Args[1] // Use the provided version
+	}
+
+	// Fetch release data (either for latest or specific version)
+	releaseData, err := repo.fetchReleaseData(version)
+	if err != nil {
+		fmt.Println("Error fetching release data:", err)
+		return
+	}
+
+	// Generate and print the formatted changelog
+	changelog := repo.generateChangelog(releaseData.TagName, releaseData.Published, releaseData.HtmlUrl, releaseData.Body)
+	fmt.Println(changelog)
+}
--- a/tools/check-component/main.go
+++ b/tools/check-component/main.go
@@ -0,0 +1,194 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/config"
+	"github.com/openimsdk/tools/db/mongoutil"
+	"github.com/openimsdk/tools/db/redisutil"
+	"github.com/openimsdk/tools/discovery/etcd"
+	"github.com/openimsdk/tools/discovery/zookeeper"
+	"github.com/openimsdk/tools/mq/kafka"
+	"github.com/openimsdk/tools/s3/minio"
+	"github.com/openimsdk/tools/system/program"
+)
+
+const maxRetry = 180
+
+const (
+	MountConfigFilePath = "CONFIG_PATH"
+	DeploymentType      = "DEPLOYMENT_TYPE"
+	KUBERNETES          = "kubernetes"
+)
+
+func CheckZookeeper(ctx context.Context, config *config.ZooKeeper) error {
+	// Temporary disable logging
+	originalLogger := log.Default().Writer()
+	log.SetOutput(io.Discard)
+	defer log.SetOutput(originalLogger) // Ensure logging is restored
+	return zookeeper.Check(ctx, config.Address, config.Schema, zookeeper.WithUserNameAndPassword(config.Username, config.Password))
+}
+
+func CheckEtcd(ctx context.Context, config *config.Etcd) error {
+	return etcd.Check(ctx, config.Address, "/check_openim_component",
+		true,
+		etcd.WithDialTimeout(10*time.Second),
+		etcd.WithMaxCallSendMsgSize(20*1024*1024),
+		etcd.WithUsernameAndPassword(config.Username, config.Password))
+}
+
+func CheckMongo(ctx context.Context, config *config.Mongo) error {
+	return mongoutil.Check(ctx, config.Build())
+}
+
+func CheckRedis(ctx context.Context, config *config.Redis) error {
+	return redisutil.Check(ctx, config.Build())
+}
+
+func CheckMinIO(ctx context.Context, config *config.Minio) error {
+	return minio.Check(ctx, config.Build())
+}
+
+func CheckKafka(ctx context.Context, conf *config.Kafka) error {
+	return kafka.CheckHealth(ctx, conf.Build())
+}
+
+func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, *config.Minio, *config.Discovery, error) {
+	var (
+		mongoConfig = &config.Mongo{}
+		redisConfig = &config.Redis{}
+		kafkaConfig = &config.Kafka{}
+		minioConfig = &config.Minio{}
+		discovery   = &config.Discovery{}
+		thirdConfig = &config.Third{}
+	)
+
+	err := config.Load(configDir, config.MongodbConfigFileName, config.EnvPrefixMap[config.MongodbConfigFileName], mongoConfig)
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+
+	err = config.Load(configDir, config.RedisConfigFileName, config.EnvPrefixMap[config.RedisConfigFileName], redisConfig)
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+
+	err = config.Load(configDir, config.KafkaConfigFileName, config.EnvPrefixMap[config.KafkaConfigFileName], kafkaConfig)
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+
+	err = config.Load(configDir, config.OpenIMRPCThirdCfgFileName, config.EnvPrefixMap[config.OpenIMRPCThirdCfgFileName], thirdConfig)
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+
+	if thirdConfig.Object.Enable == "minio" {
+		err = config.Load(configDir, config.MinioConfigFileName, config.EnvPrefixMap[config.MinioConfigFileName], minioConfig)
+		if err != nil {
+			return nil, nil, nil, nil, nil, err
+		}
+	} else {
+		minioConfig = nil
+	}
+	err = config.Load(configDir, config.DiscoveryConfigFilename, config.EnvPrefixMap[config.DiscoveryConfigFilename], discovery)
+	if err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
+	return mongoConfig, redisConfig, kafkaConfig, minioConfig, discovery, nil
+}
+
+func main() {
+	var index int
+	var configDir string
+	flag.IntVar(&index, "i", 0, "Index number")
+	defaultConfigDir := filepath.Join("..", "..", "..", "..", "..", "config")
+	flag.StringVar(&configDir, "c", defaultConfigDir, "Configuration dir")
+	flag.Parse()
+
+	fmt.Printf("%s Index: %d, Config Path: %s\n", filepath.Base(os.Args[0]), index, configDir)
+
+	mongoConfig, redisConfig, kafkaConfig, minioConfig, zookeeperConfig, err := initConfig(configDir)
+	if err != nil {
+		program.ExitWithError(err)
+	}
+
+	ctx := context.Background()
+	err = performChecks(ctx, mongoConfig, redisConfig, kafkaConfig, minioConfig, zookeeperConfig, maxRetry)
+	if err != nil {
+		// Assume program.ExitWithError logs the error and exits.
+		// Replace with your error handling logic as necessary.
+		program.ExitWithError(err)
+	}
+}
+
+func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig *config.Redis, kafkaConfig *config.Kafka, minioConfig *config.Minio, discovery *config.Discovery, maxRetry int) error {
+	checksDone := make(map[string]bool)
+
+	checks := map[string]func(ctx context.Context) error{
+		"Mongo": func(ctx context.Context) error {
+			return CheckMongo(ctx, mongoConfig)
+		},
+		"Redis": func(ctx context.Context) error {
+			return CheckRedis(ctx, redisConfig)
+		},
+		"Kafka": func(ctx context.Context) error {
+			return CheckKafka(ctx, kafkaConfig)
+		},
+	}
+	if minioConfig != nil {
+		checks["MinIO"] = func(ctx context.Context) error {
+			return CheckMinIO(ctx, minioConfig)
+		}
+	}
+	if discovery.Enable == "etcd" {
+		checks["Etcd"] = func(ctx context.Context) error {
+			return CheckEtcd(ctx, &discovery.Etcd)
+		}
+	}
+
+	for i := 0; i < maxRetry; i++ {
+		allSuccess := true
+		for name, check := range checks {
+			if !checksDone[name] {
+				if err := check(ctx); err != nil {
+					fmt.Printf("%s check failed: %v\n", name, err)
+					allSuccess = false
+				} else {
+					fmt.Printf("%s check succeeded.\n", name)
+					checksDone[name] = true
+				}
+			}
+		}
+
+		if allSuccess {
+			fmt.Println("All components checks passed successfully.")
+			return nil
+		}
+
+		time.Sleep(1 * time.Second)
+	}
+
+	return fmt.Errorf("not all components checks passed successfully after %d attempts", maxRetry)
+}
--- a/tools/check-free-memory/main.go
+++ b/tools/check-free-memory/main.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/shirou/gopsutil/mem"
+)
+
+func main() {
+	vMem, err := mem.VirtualMemory()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to get virtual memory info: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Use the Available field to get the available memory
+	availableMemoryGB := float64(vMem.Available) / float64(1024*1024*1024)
+
+	if availableMemoryGB < 1.0 {
+		fmt.Fprintf(os.Stderr, "System available memory is less than 1GB: %.2fGB\n", availableMemoryGB)
+		os.Exit(1)
+	} else {
+		fmt.Printf("System available memory is sufficient: %.2fGB\n", availableMemoryGB)
+	}
+}
--- a/tools/imctl/.gitignore
+++ b/tools/imctl/.gitignore
@@ -0,0 +1,50 @@
+# Copyright © 2023 OpenIMSDK.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ==============================================================================
+# For the entire design of.gitignore, ignore git commits and ignore files
+#===============================================================================
+#
+
+### OpenIM developer supplement ###
+logs
+.devcontainer
+components
+out-test
+Dockerfile.cross
+
+### Makefile ###
+tmp/
+bin/
+output/
+_output/
+
+### OpenIM Config ###
+config/config.yaml
+./config/config.yaml
+.env
+./.env
+
+# files used by the developer
+.idea.md
+.todo.md
+.note.md
+
+# ==============================================================================
+# Created by https://www.toptal.com/developers/gitignore/api/go,git,vim,tags,test,emacs,backup,jetbrains
+# Edit at https://www.toptal.com/developers/gitignore?templates=go,git,vim,tags,test,emacs,backup,jetbrains
+
+cmd/
+internal/
+pkg/
--- a/tools/imctl/README.md
+++ b/tools/imctl/README.md
@@ -0,0 +1,89 @@
+# [RFC #0005] OpenIM CTL Module Proposal
+
+## Meta
+
+- Name: OpenIM CTL Module Enhancement
+- Start Date: 2023-08-23
+- Author(s): @cubxxw
+- Status: Draft
+- RFC Pull Request: (leave blank)
+- OpenIMSDK Pull Request: (leave blank)
+- OpenIMSDK Issue: https://github.com/openimsdk/open-im-server-deploy/issues/924
+- Supersedes: N/A
+
+## 📇Topics
+
+- RFC #0000 OpenIMSDK CTL Module Proposal
+  - [Meta](#meta)
+  - [Summary](#summary)
+  - [Definitions](#definitions)
+  - [Motivation](#motivation)
+  - [What it is](#what-it-is)
+  - [How it Works](#how-it-works)
+  - [Migration](#migration)
+  - [Drawbacks](#drawbacks)
+  - [Alternatives](#alternatives)
+  - [Prior Art](#prior-art)
+  - [Unresolved Questions](#unresolved-questions)
+  - [Spec. Changes (OPTIONAL)](#spec-changes-optional)
+  - [History](#history)
+
+## Summary
+
+The OpenIM CTL module proposal aims to provide an integrated tool for the OpenIM system, offering utilities for user management, system monitoring, debugging, configuration, and more. This tool will enhance the extensibility of the OpenIM system and reduce dependencies on individual modules.
+
+## Definitions
+
+- **OpenIM**: An Instant Messaging system.
+- **`imctl`**: The control command-line tool for OpenIM.
+- **E2E Testing**: End-to-End Testing.
+- **API**: Application Programming Interface.
+
+## Motivation
+
+- Improve the OpenIM system's extensibility and reduce dependencies on individual modules.
+- Simplify the process for testers to perform automated tests.
+- Enhance interaction with scripts and reduce the system's coupling.
+- Implement a consistent tool similar to kubectl for a streamlined user experience.
+
+## What it is
+
+`imctl` is a command-line utility designed for OpenIM to provide functionalities including:
+
+- User Management: Add, delete, or disable user accounts.
+- System Monitoring: View metrics like online users, message transfer rate.
+- Debugging: View logs, adjust log levels, check system states.
+- Configuration Management: Update system settings, manage plugins/modules.
+- Data Management: Backup, restore, import, or export data.
+- System Maintenance: Update, restart services, or maintenance mode.
+
+## How it Works
+
+`imctl`, inspired by kubectl, will have sub-commands and options for the functionalities mentioned. Developers, operations, and testers can invoke these commands to manage and monitor the OpenIM system.
+
+## Migration
+
+Currently, the `imctl` will be housed in `tools/imctl`, and later on, the plan is to move it to `cmd/imctl`. Migration guidelines will be provided to ensure smooth transitions.
+
+## Drawbacks
+
+- Overhead in learning and adapting to a new tool for existing users.
+- Potential complexities in implementing some of the advanced functionalities.
+
+## Alternatives
+
+- Continue using individual modules for OpenIM management.
+- Utilize third-party tools or platforms with similar functionalities, customizing them for OpenIM.
+
+## Prior Art
+
+Kubectl from Kubernetes is a significant inspiration for `imctl`, offering a comprehensive command-line tool for managing clusters.
+
+## Unresolved Questions
+
+- What other functionalities might be required in future versions of `imctl`?
+- What's the expected timeline for transitioning from `tools/imctl` to `cmd/imctl`?
+
+## Spec. Changes (OPTIONAL)
+
+As of now, there are no proposed changes to the core specifications or extensions. Future changes based on community feedback might necessitate spec changes, which will be documented accordingly.
--- a/tools/imctl/main.go
+++ b/tools/imctl/main.go
@@ -0,0 +1,22 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import "fmt"
+
+func main() {
+
+	fmt.Println("imctl")
+}
--- a/tools/infra/main.go
+++ b/tools/infra/main.go
@@ -0,0 +1,52 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+)
+
+// Define a function to print important link information
+func printLinks() {
+	blue := color.New(color.FgBlue).SprintFunc()
+	fmt.Printf("OpenIM Github: %s\n", blue("https://github.com/OpenIMSDK/Open-IM-Server"))
+	fmt.Printf("Slack Invitation: %s\n", blue("https://openimsdk.slack.com"))
+	fmt.Printf("Follow Twitter: %s\n", blue("https://twitter.com/founder_im63606"))
+}
+
+func main() {
+	yellow := color.New(color.FgYellow)
+	blue := color.New(color.FgBlue, color.Bold)
+
+	yellow.Println("Please use the release branch or tag for production environments!")
+
+	message := `
+____                       _____  __  __ 
+/ __ \                     |_   _||  \/  |
+| |  | | _ __    ___  _ __    | |  | \  / |
+| |  | || '_ \  / _ \| '_ \   | |  | |\/| |
+| |__| || |_) ||  __/| | | | _| |_ | |  | |
+\____/ | .__/  \___||_| |_||_____||_|  |_|
+	   | |                                
+	   |_|                                
+
+Keep checking for updates!
+`
+
+	blue.Println(message)
+	printLinks() // Call the function to print the link information
+}
--- a/tools/ncpu/README.md
+++ b/tools/ncpu/README.md
@@ -0,0 +1,39 @@
+# ncpu
+
+**ncpu** is a simple utility to fetch the number of CPU cores across different operating systems.
+
+## Introduction
+
+In various scenarios, especially while compiling code, it's beneficial to know the number of available CPU cores to optimize the build process. However, the command to fetch the CPU core count differs between operating systems. For example, on Linux, we use `nproc`, while on macOS, it's `sysctl -n hw.ncpu`. The `ncpu` utility provides a unified way to obtain this number, regardless of the platform.
+
+## Usage
+
+To retrieve the number of CPU cores, simply use the `ncpu` command:
+
+```bash
+$ ncpu
+```
+
+This will return an integer representing the number of available CPU cores.
+
+### Example:
+
+Let's say you're compiling a project using `make`. To utilize all the CPU cores for the compilation process, you can use:
+
+```bash
+$ make -j $(ncpu) build # or any other build command
+```
+
+The above command will ensure the build process takes advantage of all the available CPU cores, thereby potentially speeding up the compilation.
+
+## Why use `ncpu`?
+
+- **Cross-platform compatibility**: No need to remember or detect which OS-specific command to use. Just use `ncpu`!
+  
+- **Ease of use**: A simple and intuitive command that's easy to incorporate into scripts or command-line operations.
+
+- **Consistency**: Ensures consistent behavior and output across different systems and environments.
+
+## Installation
+
+(Include installation steps here, e.g., how to clone the repo, build the tool, or install via package manager.)
--- a/tools/ncpu/main.go
+++ b/tools/ncpu/main.go
@@ -0,0 +1,32 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"runtime"
+
+	"go.uber.org/automaxprocs/maxprocs"
+)
+
+func main() {
+	// Set maxprocs with a custom logger that does nothing to ignore logs.
+	maxprocs.Set(maxprocs.Logger(func(string, ...interface{}) {
+		// Intentionally left blank to suppress all log output from automaxprocs.
+	}))
+
+	// Now this will print the GOMAXPROCS value without printing the automaxprocs log message.
+	fmt.Println(runtime.GOMAXPROCS(0))
+}
--- a/tools/ncpu/main_test.go
+++ b/tools/ncpu/main_test.go
@@ -0,0 +1,35 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import "testing"
+
+func Test_main(t *testing.T) {
+	tests := []struct {
+		name string
+	}{
+		{
+			name: "Test_main",
+		},
+		{
+			name: "Test_main2",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			main()
+		})
+	}
+}
--- a/tools/s3/README.md
+++ b/tools/s3/README.md
@@ -0,0 +1,12 @@
+# After s3 switches the storage engine, convert the data
+
+- build
+```shell
+go build -o s3convert main.go
+```
+
+- start
+```shell
+./s3convert -config <config dir path> -name <old s3 name>
+# ./s3convert -config ./../../config -name minio
+```
--- a/tools/s3/internal/conversion.go
+++ b/tools/s3/internal/conversion.go
@@ -0,0 +1,203 @@
+package internal
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/config"
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/storage/cache/redis"
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/storage/database"
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/storage/database/mgo"
+	"github.com/mitchellh/mapstructure"
+	"github.com/openimsdk/tools/db/mongoutil"
+	"github.com/openimsdk/tools/db/redisutil"
+	"github.com/openimsdk/tools/s3"
+	"github.com/openimsdk/tools/s3/aws"
+	"github.com/openimsdk/tools/s3/cos"
+	"github.com/openimsdk/tools/s3/kodo"
+	"github.com/openimsdk/tools/s3/minio"
+	"github.com/openimsdk/tools/s3/oss"
+	"github.com/spf13/viper"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+const defaultTimeout = time.Second * 10
+
+func readConf(path string, val any) error {
+	v := viper.New()
+	v.SetConfigFile(path)
+	if err := v.ReadInConfig(); err != nil {
+		return err
+	}
+	fn := func(config *mapstructure.DecoderConfig) {
+		config.TagName = "mapstructure"
+	}
+	return v.Unmarshal(val, fn)
+}
+
+func getS3(path string, name string, thirdConf *config.Third) (s3.Interface, error) {
+	switch name {
+	case "minio":
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
+		defer cancel()
+		var minioConf config.Minio
+		if err := readConf(filepath.Join(path, minioConf.GetConfigFileName()), &minioConf); err != nil {
+			return nil, err
+		}
+		var redisConf config.Redis
+		if err := readConf(filepath.Join(path, redisConf.GetConfigFileName()), &redisConf); err != nil {
+			return nil, err
+		}
+		rdb, err := redisutil.NewRedisClient(ctx, redisConf.Build())
+		if err != nil {
+			return nil, err
+		}
+		return minio.NewMinio(ctx, redis.NewMinioCache(rdb), *minioConf.Build())
+	case "cos":
+		return cos.NewCos(*thirdConf.Object.Cos.Build())
+	case "oss":
+		return oss.NewOSS(*thirdConf.Object.Oss.Build())
+	case "kodo":
+		return kodo.NewKodo(*thirdConf.Object.Kodo.Build())
+	case "aws":
+		return aws.NewAws(*thirdConf.Object.Aws.Build())
+	default:
+		return nil, fmt.Errorf("invalid object enable: %s", name)
+	}
+}
+
+func getMongo(path string) (database.ObjectInfo, error) {
+	var mongoConf config.Mongo
+	if err := readConf(filepath.Join(path, mongoConf.GetConfigFileName()), &mongoConf); err != nil {
+		return nil, err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
+	defer cancel()
+	mgocli, err := mongoutil.NewMongoDB(ctx, mongoConf.Build())
+	if err != nil {
+		return nil, err
+	}
+	return mgo.NewS3Mongo(mgocli.GetDB())
+}
+
+func Main(path string, engine string) error {
+	var thirdConf config.Third
+	if err := readConf(filepath.Join(path, thirdConf.GetConfigFileName()), &thirdConf); err != nil {
+		return err
+	}
+	if thirdConf.Object.Enable == engine {
+		return errors.New("same s3 storage")
+	}
+	s3db, err := getMongo(path)
+	if err != nil {
+		return err
+	}
+	oldS3, err := getS3(path, engine, &thirdConf)
+	if err != nil {
+		return err
+	}
+	newS3, err := getS3(path, thirdConf.Object.Enable, &thirdConf)
+	if err != nil {
+		return err
+	}
+	count, err := getEngineCount(s3db, oldS3.Engine())
+	if err != nil {
+		return err
+	}
+	log.Printf("engine %s count: %d", oldS3.Engine(), count)
+	var skip int
+	for i := 1; i <= count+1; i++ {
+		log.Printf("start %d/%d", i, count)
+		start := time.Now()
+		res, err := doObject(s3db, newS3, oldS3, skip)
+		if err != nil {
+			log.Printf("end [%s] %d/%d error %s", time.Since(start), i, count, err)
+			return err
+		}
+		log.Printf("end [%s] %d/%d result %+v", time.Since(start), i, count, *res)
+		if res.Skip {
+			skip++
+		}
+		if res.End {
+			break
+		}
+	}
+	return nil
+}
+
+func getEngineCount(db database.ObjectInfo, name string) (int, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
+	defer cancel()
+	count, err := db.GetEngineCount(ctx, name)
+	if err != nil {
+		return 0, err
+	}
+	return int(count), nil
+}
+
+func doObject(db database.ObjectInfo, newS3, oldS3 s3.Interface, skip int) (*Result, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
+	defer cancel()
+	infos, err := db.GetEngineInfo(ctx, oldS3.Engine(), 1, skip)
+	if err != nil {
+		return nil, err
+	}
+	if len(infos) == 0 {
+		return &Result{End: true}, nil
+	}
+	obj := infos[0]
+	if _, err := db.Take(ctx, newS3.Engine(), obj.Name); err == nil {
+		return &Result{Skip: true}, nil
+	} else if !errors.Is(err, mongo.ErrNoDocuments) {
+		return nil, err
+	}
+	downloadURL, err := oldS3.AccessURL(ctx, obj.Key, time.Hour, &s3.AccessURLOption{})
+	if err != nil {
+		return nil, err
+	}
+	putURL, err := newS3.PresignedPutObject(ctx, obj.Key, time.Hour, &s3.PutOption{ContentType: obj.ContentType})
+	if err != nil {
+		return nil, err
+	}
+	downloadResp, err := http.Get(downloadURL)
+	if err != nil {
+		return nil, err
+	}
+	defer downloadResp.Body.Close()
+	switch downloadResp.StatusCode {
+	case http.StatusNotFound:
+		return &Result{Skip: true}, nil
+	case http.StatusOK:
+	default:
+		return nil, fmt.Errorf("download object failed %s", downloadResp.Status)
+	}
+	log.Printf("file size %d", obj.Size)
+	request, err := http.NewRequest(http.MethodPut, putURL.URL, downloadResp.Body)
+	if err != nil {
+		return nil, err
+	}
+	putResp, err := http.DefaultClient.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer putResp.Body.Close()
+	if putResp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("put object failed %s", putResp.Status)
+	}
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTimeout)
+	defer cancel()
+	if err := db.UpdateEngine(ctx, obj.Engine, obj.Name, newS3.Engine()); err != nil {
+		return nil, err
+	}
+	return &Result{}, nil
+}
+
+type Result struct {
+	Skip bool
+	End  bool
+}
--- a/tools/s3/main.go
+++ b/tools/s3/main.go
@@ -0,0 +1,24 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"git.imall.cloud/openim/open-im-server-deploy/tools/s3/internal"
+)
+
+func main() {
+	var (
+		name   string
+		config string
+	)
+	flag.StringVar(&name, "name", "", "old previous storage name")
+	flag.StringVar(&config, "config", "", "config directory")
+	flag.Parse()
+	if err := internal.Main(config, name); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	fmt.Fprintln(os.Stdout, "success")
+}
--- a/tools/seq/internal/seq.go
+++ b/tools/seq/internal/seq.go
@@ -0,0 +1,347 @@
+package internal
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/config"
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/storage/database/mgo"
+	"github.com/mitchellh/mapstructure"
+	"github.com/openimsdk/tools/db/mongoutil"
+	"github.com/openimsdk/tools/db/redisutil"
+	"github.com/openimsdk/tools/utils/runtimeenv"
+	"github.com/redis/go-redis/v9"
+	"github.com/spf13/viper"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+const StructTagName = "yaml"
+
+const (
+	MaxSeq                 = "MAX_SEQ:"
+	MinSeq                 = "MIN_SEQ:"
+	ConversationUserMinSeq = "CON_USER_MIN_SEQ:"
+	HasReadSeq             = "HAS_READ_SEQ:"
+)
+
+const (
+	batchSize             = 100
+	dataVersionCollection = "data_version"
+	seqKey                = "seq"
+	seqVersion            = 38
+)
+
+func readConfig[T any](dir string, name string) (*T, error) {
+	if runtimeenv.RuntimeEnvironment() == config.KUBERNETES {
+		dir = os.Getenv(config.MountConfigFilePath)
+	}
+	v := viper.New()
+	v.SetEnvPrefix(config.EnvPrefixMap[name])
+	v.AutomaticEnv()
+	v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	v.SetConfigFile(filepath.Join(dir, name))
+	if err := v.ReadInConfig(); err != nil {
+		return nil, err
+	}
+
+	var conf T
+	if err := v.Unmarshal(&conf, func(config *mapstructure.DecoderConfig) {
+		config.TagName = StructTagName
+	}); err != nil {
+		return nil, err
+	}
+
+	return &conf, nil
+}
+
+func Main(conf string, del time.Duration) error {
+	redisConfig, err := readConfig[config.Redis](conf, config.RedisConfigFileName)
+	if err != nil {
+		return err
+	}
+
+	mongodbConfig, err := readConfig[config.Mongo](conf, config.MongodbConfigFileName)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+	rdb, err := redisutil.NewRedisClient(ctx, redisConfig.Build())
+	if err != nil {
+		return err
+	}
+	mgocli, err := mongoutil.NewMongoDB(ctx, mongodbConfig.Build())
+	if err != nil {
+		return err
+	}
+	versionColl := mgocli.GetDB().Collection(dataVersionCollection)
+	converted, err := CheckVersion(versionColl, seqKey, seqVersion)
+	if err != nil {
+		return err
+	}
+	if converted {
+		fmt.Println("[seq] seq data has been converted")
+		return nil
+	}
+	if _, err := mgo.NewSeqConversationMongo(mgocli.GetDB()); err != nil {
+		return err
+	}
+	cSeq, err := mgo.NewSeqConversationMongo(mgocli.GetDB())
+	if err != nil {
+		return err
+	}
+	uSeq, err := mgo.NewSeqUserMongo(mgocli.GetDB())
+	if err != nil {
+		return err
+	}
+	uSpitHasReadSeq := func(id string) (conversationID string, userID string, err error) {
+		// HasReadSeq + userID + ":" + conversationID
+		arr := strings.Split(id, ":")
+		if len(arr) != 2 || arr[0] == "" || arr[1] == "" {
+			return "", "", fmt.Errorf("invalid has read seq id %s", id)
+		}
+		userID = arr[0]
+		conversationID = arr[1]
+		return
+	}
+	uSpitConversationUserMinSeq := func(id string) (conversationID string, userID string, err error) {
+		// ConversationUserMinSeq + conversationID + "u:" + userID
+		arr := strings.Split(id, "u:")
+		if len(arr) != 2 || arr[0] == "" || arr[1] == "" {
+			return "", "", fmt.Errorf("invalid has read seq id %s", id)
+		}
+		conversationID = arr[0]
+		userID = arr[1]
+		return
+	}
+
+	ts := []*taskSeq{
+		{
+			Prefix: MaxSeq,
+			GetSeq: cSeq.GetMaxSeq,
+			SetSeq: cSeq.SetMaxSeq,
+		},
+		{
+			Prefix: MinSeq,
+			GetSeq: cSeq.GetMinSeq,
+			SetSeq: cSeq.SetMinSeq,
+		},
+		{
+			Prefix: HasReadSeq,
+			GetSeq: func(ctx context.Context, id string) (int64, error) {
+				conversationID, userID, err := uSpitHasReadSeq(id)
+				if err != nil {
+					return 0, err
+				}
+				return uSeq.GetUserReadSeq(ctx, conversationID, userID)
+			},
+			SetSeq: func(ctx context.Context, id string, seq int64) error {
+				conversationID, userID, err := uSpitHasReadSeq(id)
+				if err != nil {
+					return err
+				}
+				return uSeq.SetUserReadSeq(ctx, conversationID, userID, seq)
+			},
+		},
+		{
+			Prefix: ConversationUserMinSeq,
+			GetSeq: func(ctx context.Context, id string) (int64, error) {
+				conversationID, userID, err := uSpitConversationUserMinSeq(id)
+				if err != nil {
+					return 0, err
+				}
+				return uSeq.GetUserMinSeq(ctx, conversationID, userID)
+			},
+			SetSeq: func(ctx context.Context, id string, seq int64) error {
+				conversationID, userID, err := uSpitConversationUserMinSeq(id)
+				if err != nil {
+					return err
+				}
+				return uSeq.SetUserMinSeq(ctx, conversationID, userID, seq)
+			},
+		},
+	}
+
+	cancel()
+	ctx = context.Background()
+
+	var wg sync.WaitGroup
+	wg.Add(len(ts))
+
+	for i := range ts {
+		go func(task *taskSeq) {
+			defer wg.Done()
+			err := seqRedisToMongo(ctx, rdb, task.GetSeq, task.SetSeq, task.Prefix, del, &task.Count)
+			task.End = time.Now()
+			task.Error = err
+		}(ts[i])
+	}
+	start := time.Now()
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGTERM)
+
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	var buf bytes.Buffer
+
+	printTaskInfo := func(now time.Time) {
+		buf.Reset()
+		buf.WriteString(now.Format(time.DateTime))
+		buf.WriteString(" \n")
+		for i := range ts {
+			task := ts[i]
+			if task.Error == nil {
+				if task.End.IsZero() {
+					buf.WriteString(fmt.Sprintf("[%s] converting %s* count %d", now.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count)))
+				} else {
+					buf.WriteString(fmt.Sprintf("[%s] success %s* count %d", task.End.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count)))
+				}
+			} else {
+				buf.WriteString(fmt.Sprintf("[%s] failed %s* count %d error %s", task.End.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count), task.Error))
+			}
+			buf.WriteString("\n")
+		}
+		fmt.Println(buf.String())
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case s := <-sigs:
+			return fmt.Errorf("exit by signal %s", s)
+		case <-done:
+			errs := make([]error, 0, len(ts))
+			for i := range ts {
+				task := ts[i]
+				if task.Error != nil {
+					errs = append(errs, fmt.Errorf("seq %s failed %w", task.Prefix, task.Error))
+				}
+			}
+			if len(errs) > 0 {
+				return errors.Join(errs...)
+			}
+			printTaskInfo(time.Now())
+			if err := SetVersion(versionColl, seqKey, seqVersion); err != nil {
+				return fmt.Errorf("set mongodb seq version %w", err)
+			}
+			return nil
+		case now := <-ticker.C:
+			printTaskInfo(now)
+		}
+	}
+}
+
+type taskSeq struct {
+	Prefix string
+	Count  int64
+	Error  error
+	End    time.Time
+	GetSeq func(ctx context.Context, id string) (int64, error)
+	SetSeq func(ctx context.Context, id string, seq int64) error
+}
+
+func seqRedisToMongo(ctx context.Context, rdb redis.UniversalClient, getSeq func(ctx context.Context, id string) (int64, error), setSeq func(ctx context.Context, id string, seq int64) error, prefix string, delAfter time.Duration, count *int64) error {
+	var (
+		cursor uint64
+		keys   []string
+		err    error
+	)
+	for {
+		keys, cursor, err = rdb.Scan(ctx, cursor, prefix+"*", batchSize).Result()
+		if err != nil {
+			return err
+		}
+		if len(keys) > 0 {
+			for _, key := range keys {
+				seqStr, err := rdb.Get(ctx, key).Result()
+				if err != nil {
+					return fmt.Errorf("redis get %s failed %w", key, err)
+				}
+				seq, err := strconv.Atoi(seqStr)
+				if err != nil {
+					return fmt.Errorf("invalid %s seq %s", key, seqStr)
+				}
+				if seq < 0 {
+					return fmt.Errorf("invalid %s seq %s", key, seqStr)
+				}
+				id := strings.TrimPrefix(key, prefix)
+				redisSeq := int64(seq)
+				mongoSeq, err := getSeq(ctx, id)
+				if err != nil {
+					return fmt.Errorf("get mongo seq %s failed %w", key, err)
+				}
+				if mongoSeq < redisSeq {
+					if err := setSeq(ctx, id, redisSeq); err != nil {
+						return fmt.Errorf("set mongo seq %s failed %w", key, err)
+					}
+				}
+				if delAfter > 0 {
+					if err := rdb.Expire(ctx, key, delAfter).Err(); err != nil {
+						return fmt.Errorf("redis expire key %s failed %w", key, err)
+					}
+				} else {
+					if err := rdb.Del(ctx, key).Err(); err != nil {
+						return fmt.Errorf("redis del key %s failed %w", key, err)
+					}
+				}
+				atomic.AddInt64(count, 1)
+			}
+		}
+		if cursor == 0 {
+			return nil
+		}
+	}
+}
+
+func CheckVersion(coll *mongo.Collection, key string, currentVersion int) (converted bool, err error) {
+	type VersionTable struct {
+		Key   string `bson:"key"`
+		Value string `bson:"value"`
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	res, err := mongoutil.FindOne[VersionTable](ctx, coll, bson.M{"key": key})
+	if err == nil {
+		ver, err := strconv.Atoi(res.Value)
+		if err != nil {
+			return false, fmt.Errorf("version %s parse error %w", res.Value, err)
+		}
+		if ver >= currentVersion {
+			return true, nil
+		}
+		return false, nil
+	} else if errors.Is(err, mongo.ErrNoDocuments) {
+		return false, nil
+	} else {
+		return false, err
+	}
+}
+
+func SetVersion(coll *mongo.Collection, key string, version int) error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	option := options.Update().SetUpsert(true)
+	filter := bson.M{"key": key}
+	update := bson.M{"$set": bson.M{"key": key, "value": strconv.Itoa(version)}}
+	return mongoutil.UpdateOne(ctx, coll, filter, update, false, option)
+}
--- a/tools/seq/main.go
+++ b/tools/seq/main.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	"git.imall.cloud/openim/open-im-server-deploy/tools/seq/internal"
+)
+
+func main() {
+	var (
+		config string
+		second int
+	)
+	flag.StringVar(&config, "c", "", "config directory")
+	flag.IntVar(&second, "sec", 3600*24, "delayed deletion of the original seq key after conversion")
+	flag.Parse()
+	if err := internal.Main(config, time.Duration(second)*time.Second); err != nil {
+		fmt.Println("seq task", err)
+		os.Exit(1)
+		return
+	}
+	fmt.Println("seq task success!")
+}
--- a/tools/url2im/main.go
+++ b/tools/url2im/main.go
@@ -0,0 +1,119 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+
+	"git.imall.cloud/openim/open-im-server-deploy/tools/url2im/pkg"
+)
+
+/*take.txt
+{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
+{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
+{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
+*/
+
+func main() {
+	var conf pkg.Config // Configuration object, '*' denotes required fields
+
+	// *Required*: Path for the task log file
+	flag.StringVar(&conf.TaskPath, "task", "take.txt", "Path for the task log file")
+
+	// Optional: Path for the progress log file
+	flag.StringVar(&conf.ProgressPath, "progress", "", "Path for the progress log file")
+
+	// Number of concurrent operations
+	flag.IntVar(&conf.Concurrency, "concurrency", 1, "Number of concurrent operations")
+
+	// Number of retry attempts
+	flag.IntVar(&conf.Retry, "retry", 1, "Number of retry attempts")
+
+	// Optional: Path for the temporary directory
+	flag.StringVar(&conf.TempDir, "temp", "", "Path for the temporary directory")
+
+	// Cache size in bytes (downloads move to disk when exceeded)
+	flag.Int64Var(&conf.CacheSize, "cache", 1024*1024*100, "Cache size in bytes")
+
+	// Request timeout in milliseconds
+	flag.Int64Var((*int64)(&conf.Timeout), "timeout", 5000, "Request timeout in milliseconds")
+
+	// *Required*: API endpoint for the IM service
+	flag.StringVar(&conf.Api, "api", "http://127.0.0.1:10002", "API endpoint for the IM service")
+
+	// IM administrator's user ID
+	flag.StringVar(&conf.UserID, "userID", "openIM123456", "IM administrator's user ID")
+
+	// Secret for the IM configuration
+	flag.StringVar(&conf.Secret, "secret", "openIM123", "Secret for the IM configuration")
+
+	flag.Parse()
+	if !filepath.IsAbs(conf.TaskPath) {
+		var err error
+		conf.TaskPath, err = filepath.Abs(conf.TaskPath)
+		if err != nil {
+			log.Println("get abs path err:", err)
+			return
+		}
+	}
+	if conf.ProgressPath == "" {
+		conf.ProgressPath = conf.TaskPath + ".progress.txt"
+	} else if !filepath.IsAbs(conf.ProgressPath) {
+		var err error
+		conf.ProgressPath, err = filepath.Abs(conf.ProgressPath)
+		if err != nil {
+			log.Println("get abs path err:", err)
+			return
+		}
+	}
+	if conf.TempDir == "" {
+		conf.TempDir = conf.TaskPath + ".temp"
+	}
+	if info, err := os.Stat(conf.TempDir); err == nil {
+		if !info.IsDir() {
+			log.Printf("temp dir %s is not dir\n", err)
+			return
+		}
+	} else if os.IsNotExist(err) {
+		if err := os.MkdirAll(conf.TempDir, os.ModePerm); err != nil {
+			log.Printf("mkdir temp dir %s err %+v\n", conf.TempDir, err)
+			return
+		}
+		defer os.RemoveAll(conf.TempDir)
+	} else {
+		log.Println("get temp dir err:", err)
+		return
+	}
+	if conf.Concurrency <= 0 {
+		conf.Concurrency = 1
+	}
+	if conf.Retry <= 0 {
+		conf.Retry = 1
+	}
+	if conf.CacheSize <= 0 {
+		conf.CacheSize = 1024 * 1024 * 100 // 100M
+	}
+	if conf.Timeout <= 0 {
+		conf.Timeout = 5000
+	}
+	conf.Timeout = conf.Timeout * time.Millisecond
+	if err := pkg.Run(conf); err != nil {
+		log.Println("main err:", err)
+	}
+}
--- a/tools/url2im/pkg/api.go
+++ b/tools/url2im/pkg/api.go
@@ -0,0 +1,124 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"git.imall.cloud/openim/protocol/auth"
+	"git.imall.cloud/openim/protocol/third"
+	"github.com/openimsdk/tools/errs"
+)
+
+type Api struct {
+	Api    string
+	UserID string
+	Secret string
+	Token  string
+	Client *http.Client
+}
+
+func (a *Api) apiPost(ctx context.Context, path string, req any, resp any) error {
+	operationID, _ := ctx.Value("operationID").(string)
+	if operationID == "" {
+		return errs.New("call api operationID is empty")
+	}
+	reqBody, err := json.Marshal(req)
+	if err != nil {
+		return err
+	}
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, a.Api+path, bytes.NewReader(reqBody))
+	if err != nil {
+		return err
+	}
+	DefaultRequestHeader(request.Header)
+	request.ContentLength = int64(len(reqBody))
+	request.Header.Set("Content-Type", "application/json")
+	request.Header.Set("operationID", operationID)
+	if a.Token != "" {
+		request.Header.Set("token", a.Token)
+	}
+	response, err := a.Client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	body, err := io.ReadAll(response.Body)
+	if err != nil {
+		return err
+	}
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("api %s status %s body %s", path, response.Status, body)
+	}
+	var baseResponse struct {
+		ErrCode int             `json:"errCode"`
+		ErrMsg  string          `json:"errMsg"`
+		ErrDlt  string          `json:"errDlt"`
+		Data    json.RawMessage `json:"data"`
+	}
+	if err := json.Unmarshal(body, &baseResponse); err != nil {
+		return err
+	}
+	if baseResponse.ErrCode != 0 {
+		return fmt.Errorf("api %s errCode %d errMsg %s errDlt %s", path, baseResponse.ErrCode, baseResponse.ErrMsg, baseResponse.ErrDlt)
+	}
+	if resp != nil {
+		if err := json.Unmarshal(baseResponse.Data, resp); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (a *Api) GetAdminToken(ctx context.Context) (string, error) {
+	req := auth.GetAdminTokenReq{
+		UserID: a.UserID,
+		Secret: a.Secret,
+	}
+	var resp auth.GetAdminTokenResp
+	if err := a.apiPost(ctx, "/auth/get_admin_token", &req, &resp); err != nil {
+		return "", err
+	}
+	return resp.Token, nil
+}
+
+func (a *Api) GetPartLimit(ctx context.Context) (*third.PartLimitResp, error) {
+	var resp third.PartLimitResp
+	if err := a.apiPost(ctx, "/object/part_limit", &third.PartLimitReq{}, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (a *Api) InitiateMultipartUpload(ctx context.Context, req *third.InitiateMultipartUploadReq) (*third.InitiateMultipartUploadResp, error) {
+	var resp third.InitiateMultipartUploadResp
+	if err := a.apiPost(ctx, "/object/initiate_multipart_upload", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (a *Api) CompleteMultipartUpload(ctx context.Context, req *third.CompleteMultipartUploadReq) (string, error) {
+	var resp third.CompleteMultipartUploadResp
+	if err := a.apiPost(ctx, "/object/complete_multipart_upload", req, &resp); err != nil {
+		return "", err
+	}
+	return resp.Url, nil
+}
--- a/tools/url2im/pkg/buffer.go
+++ b/tools/url2im/pkg/buffer.go
@@ -0,0 +1,110 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"bytes"
+	"io"
+	"os"
+)
+
+type ReadSeekSizeCloser interface {
+	io.ReadSeekCloser
+	Size() int64
+}
+
+func NewReader(r io.Reader, max int64, path string) (ReadSeekSizeCloser, error) {
+	buf := make([]byte, max+1)
+	n, err := io.ReadFull(r, buf)
+	if err == nil {
+		f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o666)
+		if err != nil {
+			return nil, err
+		}
+		var ok bool
+		defer func() {
+			if !ok {
+				_ = f.Close()
+				_ = os.Remove(path)
+			}
+		}()
+		if _, err := f.Write(buf[:n]); err != nil {
+			return nil, err
+		}
+		cn, err := io.Copy(f, r)
+		if err != nil {
+			return nil, err
+		}
+		if _, err := f.Seek(0, io.SeekStart); err != nil {
+			return nil, err
+		}
+		ok = true
+		return &fileBuffer{
+			f: f,
+			n: cn + int64(n),
+		}, nil
+	} else if err == io.EOF || err == io.ErrUnexpectedEOF {
+		return &memoryBuffer{
+			r: bytes.NewReader(buf[:n]),
+		}, nil
+	} else {
+		return nil, err
+	}
+}
+
+type fileBuffer struct {
+	n int64
+	f *os.File
+}
+
+func (r *fileBuffer) Read(p []byte) (n int, err error) {
+	return r.f.Read(p)
+}
+
+func (r *fileBuffer) Seek(offset int64, whence int) (int64, error) {
+	return r.f.Seek(offset, whence)
+}
+
+func (r *fileBuffer) Size() int64 {
+	return r.n
+}
+
+func (r *fileBuffer) Close() error {
+	name := r.f.Name()
+	if err := r.f.Close(); err != nil {
+		return err
+	}
+	return os.Remove(name)
+}
+
+type memoryBuffer struct {
+	r *bytes.Reader
+}
+
+func (r *memoryBuffer) Read(p []byte) (n int, err error) {
+	return r.r.Read(p)
+}
+
+func (r *memoryBuffer) Seek(offset int64, whence int) (int64, error) {
+	return r.r.Seek(offset, whence)
+}
+
+func (r *memoryBuffer) Close() error {
+	return nil
+}
+
+func (r *memoryBuffer) Size() int64 {
+	return r.r.Size()
+}
--- a/tools/url2im/pkg/config.go
+++ b/tools/url2im/pkg/config.go
@@ -0,0 +1,30 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import "time"
+
+type Config struct {
+	TaskPath     string
+	ProgressPath string
+	Concurrency  int
+	Retry        int
+	Timeout      time.Duration
+	Api          string
+	UserID       string
+	Secret       string
+	TempDir      string
+	CacheSize    int64
+}
--- a/tools/url2im/pkg/http.go
+++ b/tools/url2im/pkg/http.go
@@ -0,0 +1,21 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import "net/http"
+
+func DefaultRequestHeader(header http.Header) {
+	header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36")
+}
--- a/tools/url2im/pkg/manage.go
+++ b/tools/url2im/pkg/manage.go
@@ -0,0 +1,400 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"bufio"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/openimsdk/tools/errs"
+
+	"git.imall.cloud/openim/protocol/third"
+)
+
+type Upload struct {
+	URL         string `json:"url"`
+	Name        string `json:"name"`
+	ContentType string `json:"contentType"`
+}
+
+type Task struct {
+	Index  int
+	Upload Upload
+}
+
+type PartInfo struct {
+	ContentType string
+	PartSize    int64
+	PartNum     int
+	FileMd5     string
+	PartMd5     string
+	PartSizes   []int64
+	PartMd5s    []string
+}
+
+func Run(conf Config) error {
+	m := &Manage{
+		prefix: time.Now().Format("20060102150405"),
+		conf:   &conf,
+		ctx:    context.Background(),
+	}
+	return m.Run()
+}
+
+type Manage struct {
+	conf      *Config
+	ctx       context.Context
+	api       *Api
+	partLimit *third.PartLimitResp
+	prefix    string
+	tasks     chan Task
+	id        uint64
+	success   int64
+	failed    int64
+}
+
+func (m *Manage) tempFilePath() string {
+	return filepath.Join(m.conf.TempDir, fmt.Sprintf("%s_%d", m.prefix, atomic.AddUint64(&m.id, 1)))
+}
+
+func (m *Manage) Run() error {
+	defer func(start time.Time) {
+		log.Printf("run time %s\n", time.Since(start))
+	}(time.Now())
+	m.api = &Api{
+		Api:    m.conf.Api,
+		UserID: m.conf.UserID,
+		Secret: m.conf.Secret,
+		Client: &http.Client{Timeout: m.conf.Timeout},
+	}
+	var err error
+	ctx := context.WithValue(m.ctx, "operationID", fmt.Sprintf("%s_init", m.prefix))
+	m.api.Token, err = m.api.GetAdminToken(ctx)
+	if err != nil {
+		return err
+	}
+	m.partLimit, err = m.api.GetPartLimit(ctx)
+	if err != nil {
+		return err
+	}
+	progress, err := ReadProgress(m.conf.ProgressPath)
+	if err != nil {
+		return err
+	}
+	progressFile, err := os.OpenFile(m.conf.ProgressPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return err
+	}
+	var mutex sync.Mutex
+	writeSuccessIndex := func(index int) {
+		mutex.Lock()
+		defer mutex.Unlock()
+		if _, err := progressFile.Write([]byte(strconv.Itoa(index) + "\n")); err != nil {
+			log.Printf("write progress err: %v\n", err)
+		}
+	}
+	file, err := os.Open(m.conf.TaskPath)
+	if err != nil {
+		return err
+	}
+	m.tasks = make(chan Task, m.conf.Concurrency*2)
+	go func() {
+		defer file.Close()
+		defer close(m.tasks)
+		scanner := bufio.NewScanner(file)
+		var (
+			index int
+			num   int
+		)
+		for scanner.Scan() {
+			line := strings.TrimSpace(scanner.Text())
+			if line == "" {
+				continue
+			}
+			index++
+			if progress.IsUploaded(index) {
+				log.Printf("index: %d already uploaded %s\n", index, line)
+				continue
+			}
+			var upload Upload
+			if err := json.Unmarshal([]byte(line), &upload); err != nil {
+				log.Printf("index: %d json.Unmarshal(%s) err: %v", index, line, err)
+				continue
+			}
+			num++
+			m.tasks <- Task{
+				Index:  index,
+				Upload: upload,
+			}
+		}
+		if num == 0 {
+			log.Println("mark all completed")
+		}
+	}()
+	var wg sync.WaitGroup
+	wg.Add(m.conf.Concurrency)
+	for i := 0; i < m.conf.Concurrency; i++ {
+		go func(tid int) {
+			defer wg.Done()
+			for task := range m.tasks {
+				var success bool
+				for n := 0; n < m.conf.Retry; n++ {
+					ctx := context.WithValue(m.ctx, "operationID", fmt.Sprintf("%s_%d_%d_%d", m.prefix, tid, task.Index, n+1))
+					if urlRaw, err := m.RunTask(ctx, task); err == nil {
+						writeSuccessIndex(task.Index)
+						log.Println("index:", task.Index, "upload success", "urlRaw", urlRaw)
+						success = true
+						break
+					} else {
+						log.Printf("index: %d upload: %+v err: %v", task.Index, task.Upload, err)
+					}
+				}
+				if success {
+					atomic.AddInt64(&m.success, 1)
+				} else {
+					atomic.AddInt64(&m.failed, 1)
+					log.Printf("index: %d upload: %+v failed", task.Index, task.Upload)
+				}
+			}
+		}(i + 1)
+	}
+	wg.Wait()
+	log.Printf("execution completed success %d failed %d\n", m.success, m.failed)
+	return nil
+}
+
+func (m *Manage) RunTask(ctx context.Context, task Task) (string, error) {
+	resp, err := m.HttpGet(ctx, task.Upload.URL)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	reader, err := NewReader(resp.Body, m.conf.CacheSize, m.tempFilePath())
+	if err != nil {
+		return "", err
+	}
+	defer reader.Close()
+	part, err := m.getPartInfo(ctx, reader, reader.Size())
+	if err != nil {
+		return "", err
+	}
+	var contentType string
+	if task.Upload.ContentType == "" {
+		contentType = part.ContentType
+	} else {
+		contentType = task.Upload.ContentType
+	}
+	initiateMultipartUploadResp, err := m.api.InitiateMultipartUpload(ctx, &third.InitiateMultipartUploadReq{
+		Hash:        part.PartMd5,
+		Size:        reader.Size(),
+		PartSize:    part.PartSize,
+		MaxParts:    -1,
+		Cause:       "batch-import",
+		Name:        task.Upload.Name,
+		ContentType: contentType,
+	})
+	if err != nil {
+		return "", err
+	}
+	if initiateMultipartUploadResp.Upload == nil {
+		return initiateMultipartUploadResp.Url, nil
+	}
+	if _, err := reader.Seek(0, io.SeekStart); err != nil {
+		return "", err
+	}
+	uploadParts := make([]*third.SignPart, part.PartNum)
+	for _, part := range initiateMultipartUploadResp.Upload.Sign.Parts {
+		uploadParts[part.PartNumber-1] = part
+	}
+	for i, currentPartSize := range part.PartSizes {
+		md5Reader := NewMd5Reader(io.LimitReader(reader, currentPartSize))
+		if err := m.doPut(ctx, m.api.Client, initiateMultipartUploadResp.Upload.Sign, uploadParts[i], md5Reader, currentPartSize); err != nil {
+			return "", err
+		}
+		if md5val := md5Reader.Md5(); md5val != part.PartMd5s[i] {
+			return "", fmt.Errorf("upload part %d failed, md5 not match, expect %s, got %s", i, part.PartMd5s[i], md5val)
+		}
+	}
+	urlRaw, err := m.api.CompleteMultipartUpload(ctx, &third.CompleteMultipartUploadReq{
+		UploadID:    initiateMultipartUploadResp.Upload.UploadID,
+		Parts:       part.PartMd5s,
+		Name:        task.Upload.Name,
+		ContentType: contentType,
+		Cause:       "batch-import",
+	})
+	if err != nil {
+		return "", err
+	}
+	return urlRaw, nil
+}
+
+func (m *Manage) partSize(size int64) (int64, error) {
+	if size <= 0 {
+		return 0, errs.New("size must be greater than 0")
+	}
+	if size > m.partLimit.MaxPartSize*int64(m.partLimit.MaxNumSize) {
+		return 0, errs.New("size must be less than", "size", m.partLimit.MaxPartSize*int64(m.partLimit.MaxNumSize))
+	}
+	if size <= m.partLimit.MinPartSize*int64(m.partLimit.MaxNumSize) {
+		return m.partLimit.MinPartSize, nil
+	}
+	partSize := size / int64(m.partLimit.MaxNumSize)
+	if size%int64(m.partLimit.MaxNumSize) != 0 {
+		partSize++
+	}
+	return partSize, nil
+}
+
+func (m *Manage) partMD5(parts []string) string {
+	s := strings.Join(parts, ",")
+	md5Sum := md5.Sum([]byte(s))
+	return hex.EncodeToString(md5Sum[:])
+}
+
+func (m *Manage) getPartInfo(ctx context.Context, r io.Reader, fileSize int64) (*PartInfo, error) {
+	partSize, err := m.partSize(fileSize)
+	if err != nil {
+		return nil, err
+	}
+	partNum := int(fileSize / partSize)
+	if fileSize%partSize != 0 {
+		partNum++
+	}
+	partSizes := make([]int64, partNum)
+	for i := 0; i < partNum; i++ {
+		partSizes[i] = partSize
+	}
+	partSizes[partNum-1] = fileSize - partSize*(int64(partNum)-1)
+	partMd5s := make([]string, partNum)
+	buf := make([]byte, 1024*8)
+	fileMd5 := md5.New()
+	var contentType string
+	for i := 0; i < partNum; i++ {
+		h := md5.New()
+		r := io.LimitReader(r, partSize)
+		for {
+			if n, err := r.Read(buf); err == nil {
+				if contentType == "" {
+					contentType = http.DetectContentType(buf[:n])
+				}
+				h.Write(buf[:n])
+				fileMd5.Write(buf[:n])
+			} else if err == io.EOF {
+				break
+			} else {
+				return nil, err
+			}
+		}
+		partMd5s[i] = hex.EncodeToString(h.Sum(nil))
+	}
+	partMd5Val := m.partMD5(partMd5s)
+	fileMd5val := hex.EncodeToString(fileMd5.Sum(nil))
+	return &PartInfo{
+		ContentType: contentType,
+		PartSize:    partSize,
+		PartNum:     partNum,
+		FileMd5:     fileMd5val,
+		PartMd5:     partMd5Val,
+		PartSizes:   partSizes,
+		PartMd5s:    partMd5s,
+	}, nil
+}
+
+func (m *Manage) doPut(ctx context.Context, client *http.Client, sign *third.AuthSignParts, part *third.SignPart, reader io.Reader, size int64) error {
+	rawURL := part.Url
+	if rawURL == "" {
+		rawURL = sign.Url
+	}
+	if len(sign.Query)+len(part.Query) > 0 {
+		u, err := url.Parse(rawURL)
+		if err != nil {
+			return err
+		}
+		query := u.Query()
+		for i := range sign.Query {
+			v := sign.Query[i]
+			query[v.Key] = v.Values
+		}
+		for i := range part.Query {
+			v := part.Query[i]
+			query[v.Key] = v.Values
+		}
+		u.RawQuery = query.Encode()
+		rawURL = u.String()
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, reader)
+	if err != nil {
+		return err
+	}
+	for i := range sign.Header {
+		v := sign.Header[i]
+		req.Header[v.Key] = v.Values
+	}
+	for i := range part.Header {
+		v := part.Header[i]
+		req.Header[v.Key] = v.Values
+	}
+	req.ContentLength = size
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode/200 != 1 {
+		return fmt.Errorf("PUT %s part %d failed, status code %d, body %s", rawURL, part.PartNumber, resp.StatusCode, string(body))
+	}
+	return nil
+}
+
+func (m *Manage) HttpGet(ctx context.Context, url string) (*http.Response, error) {
+	reqUrl := url
+	for {
+		request, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl, nil)
+		if err != nil {
+			return nil, err
+		}
+		DefaultRequestHeader(request.Header)
+		response, err := m.api.Client.Do(request)
+		if err != nil {
+			return nil, err
+		}
+		if response.StatusCode != http.StatusOK {
+			_ = response.Body.Close()
+			return nil, fmt.Errorf("webhook get %s status %s", url, response.Status)
+		}
+		return response, nil
+	}
+}
--- a/tools/url2im/pkg/md5.go
+++ b/tools/url2im/pkg/md5.go
@@ -0,0 +1,43 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"hash"
+	"io"
+)
+
+func NewMd5Reader(r io.Reader) *Md5Reader {
+	return &Md5Reader{h: md5.New(), r: r}
+}
+
+type Md5Reader struct {
+	h hash.Hash
+	r io.Reader
+}
+
+func (r *Md5Reader) Read(p []byte) (n int, err error) {
+	n, err = r.r.Read(p)
+	if err == nil && n > 0 {
+		r.h.Write(p[:n])
+	}
+	return
+}
+
+func (r *Md5Reader) Md5() string {
+	return hex.EncodeToString(r.h.Sum(nil))
+}
--- a/tools/url2im/pkg/progress.go
+++ b/tools/url2im/pkg/progress.go
@@ -0,0 +1,55 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"bufio"
+	"os"
+	"strconv"
+
+	"github.com/kelindar/bitmap"
+)
+
+func ReadProgress(path string) (*Progress, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return &Progress{}, nil
+		}
+		return nil, err
+	}
+	defer file.Close()
+	scanner := bufio.NewScanner(file)
+	var upload bitmap.Bitmap
+	for scanner.Scan() {
+		index, err := strconv.Atoi(scanner.Text())
+		if err != nil || index < 0 {
+			continue
+		}
+		upload.Set(uint32(index))
+	}
+	return &Progress{upload: upload}, nil
+}
+
+type Progress struct {
+	upload bitmap.Bitmap
+}
+
+func (p *Progress) IsUploaded(index int) bool {
+	if p == nil {
+		return false
+	}
+	return p.upload.Contains(uint32(index))
+}
--- a/tools/versionchecker/main.go
+++ b/tools/versionchecker/main.go
@@ -0,0 +1,113 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"runtime"
+
+	"github.com/fatih/color"
+	"github.com/openimsdk/tools/utils/timeutil"
+)
+
+func ExecuteCommand(cmdName string, args ...string) (string, error) {
+	cmd := exec.Command(cmdName, args...)
+	var out bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	if err != nil {
+		return "", fmt.Errorf("error executing %s: %v, stderr: %s", cmdName, err, stderr.String())
+	}
+	return out.String(), nil
+}
+
+func printTime() string {
+	formattedTime := timeutil.GetCurrentTimeFormatted()
+	return fmt.Sprintf("Current Date & Time: %s", formattedTime)
+}
+
+func getGoVersion() string {
+	version := runtime.Version()
+	goos := runtime.GOOS
+	goarch := runtime.GOARCH
+	return fmt.Sprintf("Go Version: %s\nOS: %s\nArchitecture: %s", version, goos, goarch)
+}
+
+func getDockerVersion() string {
+	version, err := ExecuteCommand("docker", "--version")
+	if err != nil {
+		return "Docker is not installed. Please install it to get the version."
+	}
+	return version
+}
+
+func getKubernetesVersion() string {
+	version, err := ExecuteCommand("kubectl", "version", "--client", "--short")
+	if err != nil {
+		return "Kubernetes is not installed. Please install it to get the version."
+	}
+	return version
+}
+
+func getGitVersion() string {
+	version, err := ExecuteCommand("git", "branch", "--show-current")
+	if err != nil {
+		return "Git is not installed. Please install it to get the version."
+	}
+	return version
+}
+
+// // NOTE: You'll need to provide appropriate commands for OpenIM versions.
+// func getOpenIMServerVersion() string {
+// 	// Placeholder
+// 	openimVersion := version.GetSingleVersion()
+// 	return "OpenIM Server: " + openimVersion + "\n"
+// }
+
+// func getOpenIMClientVersion() (string, error) {
+// 	openIMClientVersion, err := version.GetClientVersion()
+// 	if err != nil {
+// 		return "", err
+// 	}
+// 	return "OpenIM Client: " + openIMClientVersion.ClientVersion + "\n", nil
+// }
+
+func main() {
+	// red := color.New(color.FgRed).SprintFunc()
+	//	green := color.New(color.FgGreen).SprintFunc()
+	blue := color.New(color.FgBlue).SprintFunc()
+	//	yellow := color.New(color.FgYellow).SprintFunc()
+	fmt.Println(blue("## Go Version"))
+	fmt.Println(getGoVersion())
+	fmt.Println(blue("## Branch Type"))
+	fmt.Println(getGitVersion())
+	fmt.Println(blue("## Docker Version"))
+	fmt.Println(getDockerVersion())
+	fmt.Println(blue("## Kubernetes Version"))
+	fmt.Println(getKubernetesVersion())
+	// fmt.Println(blue("## OpenIM Versions"))
+	// fmt.Println(getOpenIMServerVersion())
+	// clientVersion, err := getOpenIMClientVersion()
+	// if err != nil {
+	// 	fmt.Println(red("Error getting OpenIM Client Version: "), err)
+	// } else {
+	// 	fmt.Println(clientVersion)
+	// }
+}
--- a/tools/yamlfmt/main.go
+++ b/tools/yamlfmt/main.go
@@ -0,0 +1,72 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// OPENIM plan on prow tools
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+func main() {
+	// Prow OWNERs file defines the default indent as 2 spaces.
+	indent := flag.Int("indent", 2, "default indent")
+	flag.Parse()
+	for _, path := range flag.Args() {
+		sourceYaml, err := os.ReadFile(path)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
+			continue
+		}
+		rootNode, err := fetchYaml(sourceYaml)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
+			continue
+		}
+		writer, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o666)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
+			continue
+		}
+		err = streamYaml(writer, indent, rootNode)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
+			continue
+		}
+	}
+}
+
+func fetchYaml(sourceYaml []byte) (*yaml.Node, error) {
+	rootNode := yaml.Node{}
+	err := yaml.Unmarshal(sourceYaml, &rootNode)
+	if err != nil {
+		return nil, err
+	}
+	return &rootNode, nil
+}
+
+func streamYaml(writer io.Writer, indent *int, in *yaml.Node) error {
+	encoder := yaml.NewEncoder(writer)
+	encoder.SetIndent(*indent)
+	err := encoder.Encode(in)
+	if err != nil {
+		return err
+	}
+	return encoder.Close()
+}
--- a/tools/yamlfmt/main_test.go
+++ b/tools/yamlfmt/main_test.go
@@ -0,0 +1,158 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"reflect"
+	"testing"
+
+	"github.com/likexian/gokit/assert"
+	"gopkg.in/yaml.v3"
+)
+
+func Test_main(t *testing.T) {
+	sourceYaml := ` # See the OWNERS docs at https://go.k8s.io/owners
+approvers:
+- dep-approvers
+- thockin         # Network
+- liggitt
+
+labels:
+- sig/architecture
+`
+
+	outputYaml := `# See the OWNERS docs at https://go.k8s.io/owners
+approvers:
+  - dep-approvers
+  - thockin # Network
+  - liggitt
+labels:
+  - sig/architecture
+`
+	node, _ := fetchYaml([]byte(sourceYaml))
+	var output bytes.Buffer
+	indent := 2
+	writer := bufio.NewWriter(&output)
+	_ = streamYaml(writer, &indent, node)
+	_ = writer.Flush()
+	assert.Equal(t, outputYaml, string(output.Bytes()), "yaml was not formatted correctly")
+}
+
+func Test_fetchYaml(t *testing.T) {
+	type args struct {
+		sourceYaml []byte
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *yaml.Node
+		wantErr bool
+	}{
+		{
+			name: "Valid YAML",
+			args: args{sourceYaml: []byte("key: value")},
+			want: &yaml.Node{
+				Kind:  yaml.MappingNode,
+				Tag:   "!!map",
+				Value: "",
+				Content: []*yaml.Node{
+					{
+						Kind:  yaml.ScalarNode,
+						Tag:   "!!str",
+						Value: "key",
+					},
+					{
+						Kind:  yaml.ScalarNode,
+						Tag:   "!!str",
+						Value: "value",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name:    "Invalid YAML",
+			args:    args{sourceYaml: []byte("key:")},
+			want:    nil,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := fetchYaml(tt.args.sourceYaml)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("fetchYaml() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("fetchYaml() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_streamYaml(t *testing.T) {
+	type args struct {
+		indent *int
+		in     *yaml.Node
+	}
+	defaultIndent := 2
+	tests := []struct {
+		name       string
+		args       args
+		wantWriter string
+		wantErr    bool
+	}{
+		{
+			name: "Valid YAML node with default indent",
+			args: args{
+				indent: &defaultIndent,
+				in: &yaml.Node{
+					Kind:  yaml.MappingNode,
+					Tag:   "!!map",
+					Value: "",
+					Content: []*yaml.Node{
+						{
+							Kind:  yaml.ScalarNode,
+							Tag:   "!!str",
+							Value: "key",
+						},
+						{
+							Kind:  yaml.ScalarNode,
+							Tag:   "!!str",
+							Value: "value",
+						},
+					},
+				},
+			},
+			wantWriter: "key: value\n",
+			wantErr:    false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			writer := &bytes.Buffer{}
+			if err := streamYaml(writer, tt.args.indent, tt.args.in); (err != nil) != tt.wantErr {
+				t.Errorf("streamYaml() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if gotWriter := writer.String(); gotWriter != tt.wantWriter {
+				t.Errorf("streamYaml() = %v, want %v", gotWriter, tt.wantWriter)
+			}
+		})
+	}
+}