复制项目

2026-01-14 22:16:44 +08:00
parent e2577b8cee
commit e50142a3b9
691 changed files with 97009 additions and 1 deletions
--- a/tools/url2im/main.go
+++ b/tools/url2im/main.go
@@ -0,0 +1,119 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+
+	"git.imall.cloud/openim/open-im-server-deploy/tools/url2im/pkg"
+)
+
+/*take.txt
+{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
+{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
+{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
+*/
+
+func main() {
+	var conf pkg.Config // Configuration object, '*' denotes required fields
+
+	// *Required*: Path for the task log file
+	flag.StringVar(&conf.TaskPath, "task", "take.txt", "Path for the task log file")
+
+	// Optional: Path for the progress log file
+	flag.StringVar(&conf.ProgressPath, "progress", "", "Path for the progress log file")
+
+	// Number of concurrent operations
+	flag.IntVar(&conf.Concurrency, "concurrency", 1, "Number of concurrent operations")
+
+	// Number of retry attempts
+	flag.IntVar(&conf.Retry, "retry", 1, "Number of retry attempts")
+
+	// Optional: Path for the temporary directory
+	flag.StringVar(&conf.TempDir, "temp", "", "Path for the temporary directory")
+
+	// Cache size in bytes (downloads move to disk when exceeded)
+	flag.Int64Var(&conf.CacheSize, "cache", 1024*1024*100, "Cache size in bytes")
+
+	// Request timeout in milliseconds
+	flag.Int64Var((*int64)(&conf.Timeout), "timeout", 5000, "Request timeout in milliseconds")
+
+	// *Required*: API endpoint for the IM service
+	flag.StringVar(&conf.Api, "api", "http://127.0.0.1:10002", "API endpoint for the IM service")
+
+	// IM administrator's user ID
+	flag.StringVar(&conf.UserID, "userID", "openIM123456", "IM administrator's user ID")
+
+	// Secret for the IM configuration
+	flag.StringVar(&conf.Secret, "secret", "openIM123", "Secret for the IM configuration")
+
+	flag.Parse()
+	if !filepath.IsAbs(conf.TaskPath) {
+		var err error
+		conf.TaskPath, err = filepath.Abs(conf.TaskPath)
+		if err != nil {
+			log.Println("get abs path err:", err)
+			return
+		}
+	}
+	if conf.ProgressPath == "" {
+		conf.ProgressPath = conf.TaskPath + ".progress.txt"
+	} else if !filepath.IsAbs(conf.ProgressPath) {
+		var err error
+		conf.ProgressPath, err = filepath.Abs(conf.ProgressPath)
+		if err != nil {
+			log.Println("get abs path err:", err)
+			return
+		}
+	}
+	if conf.TempDir == "" {
+		conf.TempDir = conf.TaskPath + ".temp"
+	}
+	if info, err := os.Stat(conf.TempDir); err == nil {
+		if !info.IsDir() {
+			log.Printf("temp dir %s is not dir\n", err)
+			return
+		}
+	} else if os.IsNotExist(err) {
+		if err := os.MkdirAll(conf.TempDir, os.ModePerm); err != nil {
+			log.Printf("mkdir temp dir %s err %+v\n", conf.TempDir, err)
+			return
+		}
+		defer os.RemoveAll(conf.TempDir)
+	} else {
+		log.Println("get temp dir err:", err)
+		return
+	}
+	if conf.Concurrency <= 0 {
+		conf.Concurrency = 1
+	}
+	if conf.Retry <= 0 {
+		conf.Retry = 1
+	}
+	if conf.CacheSize <= 0 {
+		conf.CacheSize = 1024 * 1024 * 100 // 100M
+	}
+	if conf.Timeout <= 0 {
+		conf.Timeout = 5000
+	}
+	conf.Timeout = conf.Timeout * time.Millisecond
+	if err := pkg.Run(conf); err != nil {
+		log.Println("main err:", err)
+	}
+}
--- a/tools/url2im/pkg/api.go
+++ b/tools/url2im/pkg/api.go
@@ -0,0 +1,124 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"git.imall.cloud/openim/protocol/auth"
+	"git.imall.cloud/openim/protocol/third"
+	"github.com/openimsdk/tools/errs"
+)
+
+type Api struct {
+	Api    string
+	UserID string
+	Secret string
+	Token  string
+	Client *http.Client
+}
+
+func (a *Api) apiPost(ctx context.Context, path string, req any, resp any) error {
+	operationID, _ := ctx.Value("operationID").(string)
+	if operationID == "" {
+		return errs.New("call api operationID is empty")
+	}
+	reqBody, err := json.Marshal(req)
+	if err != nil {
+		return err
+	}
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, a.Api+path, bytes.NewReader(reqBody))
+	if err != nil {
+		return err
+	}
+	DefaultRequestHeader(request.Header)
+	request.ContentLength = int64(len(reqBody))
+	request.Header.Set("Content-Type", "application/json")
+	request.Header.Set("operationID", operationID)
+	if a.Token != "" {
+		request.Header.Set("token", a.Token)
+	}
+	response, err := a.Client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	body, err := io.ReadAll(response.Body)
+	if err != nil {
+		return err
+	}
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("api %s status %s body %s", path, response.Status, body)
+	}
+	var baseResponse struct {
+		ErrCode int             `json:"errCode"`
+		ErrMsg  string          `json:"errMsg"`
+		ErrDlt  string          `json:"errDlt"`
+		Data    json.RawMessage `json:"data"`
+	}
+	if err := json.Unmarshal(body, &baseResponse); err != nil {
+		return err
+	}
+	if baseResponse.ErrCode != 0 {
+		return fmt.Errorf("api %s errCode %d errMsg %s errDlt %s", path, baseResponse.ErrCode, baseResponse.ErrMsg, baseResponse.ErrDlt)
+	}
+	if resp != nil {
+		if err := json.Unmarshal(baseResponse.Data, resp); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (a *Api) GetAdminToken(ctx context.Context) (string, error) {
+	req := auth.GetAdminTokenReq{
+		UserID: a.UserID,
+		Secret: a.Secret,
+	}
+	var resp auth.GetAdminTokenResp
+	if err := a.apiPost(ctx, "/auth/get_admin_token", &req, &resp); err != nil {
+		return "", err
+	}
+	return resp.Token, nil
+}
+
+func (a *Api) GetPartLimit(ctx context.Context) (*third.PartLimitResp, error) {
+	var resp third.PartLimitResp
+	if err := a.apiPost(ctx, "/object/part_limit", &third.PartLimitReq{}, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (a *Api) InitiateMultipartUpload(ctx context.Context, req *third.InitiateMultipartUploadReq) (*third.InitiateMultipartUploadResp, error) {
+	var resp third.InitiateMultipartUploadResp
+	if err := a.apiPost(ctx, "/object/initiate_multipart_upload", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (a *Api) CompleteMultipartUpload(ctx context.Context, req *third.CompleteMultipartUploadReq) (string, error) {
+	var resp third.CompleteMultipartUploadResp
+	if err := a.apiPost(ctx, "/object/complete_multipart_upload", req, &resp); err != nil {
+		return "", err
+	}
+	return resp.Url, nil
+}
--- a/tools/url2im/pkg/buffer.go
+++ b/tools/url2im/pkg/buffer.go
@@ -0,0 +1,110 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"bytes"
+	"io"
+	"os"
+)
+
+type ReadSeekSizeCloser interface {
+	io.ReadSeekCloser
+	Size() int64
+}
+
+func NewReader(r io.Reader, max int64, path string) (ReadSeekSizeCloser, error) {
+	buf := make([]byte, max+1)
+	n, err := io.ReadFull(r, buf)
+	if err == nil {
+		f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o666)
+		if err != nil {
+			return nil, err
+		}
+		var ok bool
+		defer func() {
+			if !ok {
+				_ = f.Close()
+				_ = os.Remove(path)
+			}
+		}()
+		if _, err := f.Write(buf[:n]); err != nil {
+			return nil, err
+		}
+		cn, err := io.Copy(f, r)
+		if err != nil {
+			return nil, err
+		}
+		if _, err := f.Seek(0, io.SeekStart); err != nil {
+			return nil, err
+		}
+		ok = true
+		return &fileBuffer{
+			f: f,
+			n: cn + int64(n),
+		}, nil
+	} else if err == io.EOF || err == io.ErrUnexpectedEOF {
+		return &memoryBuffer{
+			r: bytes.NewReader(buf[:n]),
+		}, nil
+	} else {
+		return nil, err
+	}
+}
+
+type fileBuffer struct {
+	n int64
+	f *os.File
+}
+
+func (r *fileBuffer) Read(p []byte) (n int, err error) {
+	return r.f.Read(p)
+}
+
+func (r *fileBuffer) Seek(offset int64, whence int) (int64, error) {
+	return r.f.Seek(offset, whence)
+}
+
+func (r *fileBuffer) Size() int64 {
+	return r.n
+}
+
+func (r *fileBuffer) Close() error {
+	name := r.f.Name()
+	if err := r.f.Close(); err != nil {
+		return err
+	}
+	return os.Remove(name)
+}
+
+type memoryBuffer struct {
+	r *bytes.Reader
+}
+
+func (r *memoryBuffer) Read(p []byte) (n int, err error) {
+	return r.r.Read(p)
+}
+
+func (r *memoryBuffer) Seek(offset int64, whence int) (int64, error) {
+	return r.r.Seek(offset, whence)
+}
+
+func (r *memoryBuffer) Close() error {
+	return nil
+}
+
+func (r *memoryBuffer) Size() int64 {
+	return r.r.Size()
+}
--- a/tools/url2im/pkg/config.go
+++ b/tools/url2im/pkg/config.go
@@ -0,0 +1,30 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import "time"
+
+type Config struct {
+	TaskPath     string
+	ProgressPath string
+	Concurrency  int
+	Retry        int
+	Timeout      time.Duration
+	Api          string
+	UserID       string
+	Secret       string
+	TempDir      string
+	CacheSize    int64
+}
--- a/tools/url2im/pkg/http.go
+++ b/tools/url2im/pkg/http.go
@@ -0,0 +1,21 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import "net/http"
+
+func DefaultRequestHeader(header http.Header) {
+	header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36")
+}
--- a/tools/url2im/pkg/manage.go
+++ b/tools/url2im/pkg/manage.go
@@ -0,0 +1,400 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"bufio"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/openimsdk/tools/errs"
+
+	"git.imall.cloud/openim/protocol/third"
+)
+
+type Upload struct {
+	URL         string `json:"url"`
+	Name        string `json:"name"`
+	ContentType string `json:"contentType"`
+}
+
+type Task struct {
+	Index  int
+	Upload Upload
+}
+
+type PartInfo struct {
+	ContentType string
+	PartSize    int64
+	PartNum     int
+	FileMd5     string
+	PartMd5     string
+	PartSizes   []int64
+	PartMd5s    []string
+}
+
+func Run(conf Config) error {
+	m := &Manage{
+		prefix: time.Now().Format("20060102150405"),
+		conf:   &conf,
+		ctx:    context.Background(),
+	}
+	return m.Run()
+}
+
+type Manage struct {
+	conf      *Config
+	ctx       context.Context
+	api       *Api
+	partLimit *third.PartLimitResp
+	prefix    string
+	tasks     chan Task
+	id        uint64
+	success   int64
+	failed    int64
+}
+
+func (m *Manage) tempFilePath() string {
+	return filepath.Join(m.conf.TempDir, fmt.Sprintf("%s_%d", m.prefix, atomic.AddUint64(&m.id, 1)))
+}
+
+func (m *Manage) Run() error {
+	defer func(start time.Time) {
+		log.Printf("run time %s\n", time.Since(start))
+	}(time.Now())
+	m.api = &Api{
+		Api:    m.conf.Api,
+		UserID: m.conf.UserID,
+		Secret: m.conf.Secret,
+		Client: &http.Client{Timeout: m.conf.Timeout},
+	}
+	var err error
+	ctx := context.WithValue(m.ctx, "operationID", fmt.Sprintf("%s_init", m.prefix))
+	m.api.Token, err = m.api.GetAdminToken(ctx)
+	if err != nil {
+		return err
+	}
+	m.partLimit, err = m.api.GetPartLimit(ctx)
+	if err != nil {
+		return err
+	}
+	progress, err := ReadProgress(m.conf.ProgressPath)
+	if err != nil {
+		return err
+	}
+	progressFile, err := os.OpenFile(m.conf.ProgressPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return err
+	}
+	var mutex sync.Mutex
+	writeSuccessIndex := func(index int) {
+		mutex.Lock()
+		defer mutex.Unlock()
+		if _, err := progressFile.Write([]byte(strconv.Itoa(index) + "\n")); err != nil {
+			log.Printf("write progress err: %v\n", err)
+		}
+	}
+	file, err := os.Open(m.conf.TaskPath)
+	if err != nil {
+		return err
+	}
+	m.tasks = make(chan Task, m.conf.Concurrency*2)
+	go func() {
+		defer file.Close()
+		defer close(m.tasks)
+		scanner := bufio.NewScanner(file)
+		var (
+			index int
+			num   int
+		)
+		for scanner.Scan() {
+			line := strings.TrimSpace(scanner.Text())
+			if line == "" {
+				continue
+			}
+			index++
+			if progress.IsUploaded(index) {
+				log.Printf("index: %d already uploaded %s\n", index, line)
+				continue
+			}
+			var upload Upload
+			if err := json.Unmarshal([]byte(line), &upload); err != nil {
+				log.Printf("index: %d json.Unmarshal(%s) err: %v", index, line, err)
+				continue
+			}
+			num++
+			m.tasks <- Task{
+				Index:  index,
+				Upload: upload,
+			}
+		}
+		if num == 0 {
+			log.Println("mark all completed")
+		}
+	}()
+	var wg sync.WaitGroup
+	wg.Add(m.conf.Concurrency)
+	for i := 0; i < m.conf.Concurrency; i++ {
+		go func(tid int) {
+			defer wg.Done()
+			for task := range m.tasks {
+				var success bool
+				for n := 0; n < m.conf.Retry; n++ {
+					ctx := context.WithValue(m.ctx, "operationID", fmt.Sprintf("%s_%d_%d_%d", m.prefix, tid, task.Index, n+1))
+					if urlRaw, err := m.RunTask(ctx, task); err == nil {
+						writeSuccessIndex(task.Index)
+						log.Println("index:", task.Index, "upload success", "urlRaw", urlRaw)
+						success = true
+						break
+					} else {
+						log.Printf("index: %d upload: %+v err: %v", task.Index, task.Upload, err)
+					}
+				}
+				if success {
+					atomic.AddInt64(&m.success, 1)
+				} else {
+					atomic.AddInt64(&m.failed, 1)
+					log.Printf("index: %d upload: %+v failed", task.Index, task.Upload)
+				}
+			}
+		}(i + 1)
+	}
+	wg.Wait()
+	log.Printf("execution completed success %d failed %d\n", m.success, m.failed)
+	return nil
+}
+
+func (m *Manage) RunTask(ctx context.Context, task Task) (string, error) {
+	resp, err := m.HttpGet(ctx, task.Upload.URL)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	reader, err := NewReader(resp.Body, m.conf.CacheSize, m.tempFilePath())
+	if err != nil {
+		return "", err
+	}
+	defer reader.Close()
+	part, err := m.getPartInfo(ctx, reader, reader.Size())
+	if err != nil {
+		return "", err
+	}
+	var contentType string
+	if task.Upload.ContentType == "" {
+		contentType = part.ContentType
+	} else {
+		contentType = task.Upload.ContentType
+	}
+	initiateMultipartUploadResp, err := m.api.InitiateMultipartUpload(ctx, &third.InitiateMultipartUploadReq{
+		Hash:        part.PartMd5,
+		Size:        reader.Size(),
+		PartSize:    part.PartSize,
+		MaxParts:    -1,
+		Cause:       "batch-import",
+		Name:        task.Upload.Name,
+		ContentType: contentType,
+	})
+	if err != nil {
+		return "", err
+	}
+	if initiateMultipartUploadResp.Upload == nil {
+		return initiateMultipartUploadResp.Url, nil
+	}
+	if _, err := reader.Seek(0, io.SeekStart); err != nil {
+		return "", err
+	}
+	uploadParts := make([]*third.SignPart, part.PartNum)
+	for _, part := range initiateMultipartUploadResp.Upload.Sign.Parts {
+		uploadParts[part.PartNumber-1] = part
+	}
+	for i, currentPartSize := range part.PartSizes {
+		md5Reader := NewMd5Reader(io.LimitReader(reader, currentPartSize))
+		if err := m.doPut(ctx, m.api.Client, initiateMultipartUploadResp.Upload.Sign, uploadParts[i], md5Reader, currentPartSize); err != nil {
+			return "", err
+		}
+		if md5val := md5Reader.Md5(); md5val != part.PartMd5s[i] {
+			return "", fmt.Errorf("upload part %d failed, md5 not match, expect %s, got %s", i, part.PartMd5s[i], md5val)
+		}
+	}
+	urlRaw, err := m.api.CompleteMultipartUpload(ctx, &third.CompleteMultipartUploadReq{
+		UploadID:    initiateMultipartUploadResp.Upload.UploadID,
+		Parts:       part.PartMd5s,
+		Name:        task.Upload.Name,
+		ContentType: contentType,
+		Cause:       "batch-import",
+	})
+	if err != nil {
+		return "", err
+	}
+	return urlRaw, nil
+}
+
+func (m *Manage) partSize(size int64) (int64, error) {
+	if size <= 0 {
+		return 0, errs.New("size must be greater than 0")
+	}
+	if size > m.partLimit.MaxPartSize*int64(m.partLimit.MaxNumSize) {
+		return 0, errs.New("size must be less than", "size", m.partLimit.MaxPartSize*int64(m.partLimit.MaxNumSize))
+	}
+	if size <= m.partLimit.MinPartSize*int64(m.partLimit.MaxNumSize) {
+		return m.partLimit.MinPartSize, nil
+	}
+	partSize := size / int64(m.partLimit.MaxNumSize)
+	if size%int64(m.partLimit.MaxNumSize) != 0 {
+		partSize++
+	}
+	return partSize, nil
+}
+
+func (m *Manage) partMD5(parts []string) string {
+	s := strings.Join(parts, ",")
+	md5Sum := md5.Sum([]byte(s))
+	return hex.EncodeToString(md5Sum[:])
+}
+
+func (m *Manage) getPartInfo(ctx context.Context, r io.Reader, fileSize int64) (*PartInfo, error) {
+	partSize, err := m.partSize(fileSize)
+	if err != nil {
+		return nil, err
+	}
+	partNum := int(fileSize / partSize)
+	if fileSize%partSize != 0 {
+		partNum++
+	}
+	partSizes := make([]int64, partNum)
+	for i := 0; i < partNum; i++ {
+		partSizes[i] = partSize
+	}
+	partSizes[partNum-1] = fileSize - partSize*(int64(partNum)-1)
+	partMd5s := make([]string, partNum)
+	buf := make([]byte, 1024*8)
+	fileMd5 := md5.New()
+	var contentType string
+	for i := 0; i < partNum; i++ {
+		h := md5.New()
+		r := io.LimitReader(r, partSize)
+		for {
+			if n, err := r.Read(buf); err == nil {
+				if contentType == "" {
+					contentType = http.DetectContentType(buf[:n])
+				}
+				h.Write(buf[:n])
+				fileMd5.Write(buf[:n])
+			} else if err == io.EOF {
+				break
+			} else {
+				return nil, err
+			}
+		}
+		partMd5s[i] = hex.EncodeToString(h.Sum(nil))
+	}
+	partMd5Val := m.partMD5(partMd5s)
+	fileMd5val := hex.EncodeToString(fileMd5.Sum(nil))
+	return &PartInfo{
+		ContentType: contentType,
+		PartSize:    partSize,
+		PartNum:     partNum,
+		FileMd5:     fileMd5val,
+		PartMd5:     partMd5Val,
+		PartSizes:   partSizes,
+		PartMd5s:    partMd5s,
+	}, nil
+}
+
+func (m *Manage) doPut(ctx context.Context, client *http.Client, sign *third.AuthSignParts, part *third.SignPart, reader io.Reader, size int64) error {
+	rawURL := part.Url
+	if rawURL == "" {
+		rawURL = sign.Url
+	}
+	if len(sign.Query)+len(part.Query) > 0 {
+		u, err := url.Parse(rawURL)
+		if err != nil {
+			return err
+		}
+		query := u.Query()
+		for i := range sign.Query {
+			v := sign.Query[i]
+			query[v.Key] = v.Values
+		}
+		for i := range part.Query {
+			v := part.Query[i]
+			query[v.Key] = v.Values
+		}
+		u.RawQuery = query.Encode()
+		rawURL = u.String()
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, reader)
+	if err != nil {
+		return err
+	}
+	for i := range sign.Header {
+		v := sign.Header[i]
+		req.Header[v.Key] = v.Values
+	}
+	for i := range part.Header {
+		v := part.Header[i]
+		req.Header[v.Key] = v.Values
+	}
+	req.ContentLength = size
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode/200 != 1 {
+		return fmt.Errorf("PUT %s part %d failed, status code %d, body %s", rawURL, part.PartNumber, resp.StatusCode, string(body))
+	}
+	return nil
+}
+
+func (m *Manage) HttpGet(ctx context.Context, url string) (*http.Response, error) {
+	reqUrl := url
+	for {
+		request, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl, nil)
+		if err != nil {
+			return nil, err
+		}
+		DefaultRequestHeader(request.Header)
+		response, err := m.api.Client.Do(request)
+		if err != nil {
+			return nil, err
+		}
+		if response.StatusCode != http.StatusOK {
+			_ = response.Body.Close()
+			return nil, fmt.Errorf("webhook get %s status %s", url, response.Status)
+		}
+		return response, nil
+	}
+}
--- a/tools/url2im/pkg/md5.go
+++ b/tools/url2im/pkg/md5.go
@@ -0,0 +1,43 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"hash"
+	"io"
+)
+
+func NewMd5Reader(r io.Reader) *Md5Reader {
+	return &Md5Reader{h: md5.New(), r: r}
+}
+
+type Md5Reader struct {
+	h hash.Hash
+	r io.Reader
+}
+
+func (r *Md5Reader) Read(p []byte) (n int, err error) {
+	n, err = r.r.Read(p)
+	if err == nil && n > 0 {
+		r.h.Write(p[:n])
+	}
+	return
+}
+
+func (r *Md5Reader) Md5() string {
+	return hex.EncodeToString(r.h.Sum(nil))
+}
--- a/tools/url2im/pkg/progress.go
+++ b/tools/url2im/pkg/progress.go
@@ -0,0 +1,55 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"bufio"
+	"os"
+	"strconv"
+
+	"github.com/kelindar/bitmap"
+)
+
+func ReadProgress(path string) (*Progress, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return &Progress{}, nil
+		}
+		return nil, err
+	}
+	defer file.Close()
+	scanner := bufio.NewScanner(file)
+	var upload bitmap.Bitmap
+	for scanner.Scan() {
+		index, err := strconv.Atoi(scanner.Text())
+		if err != nil || index < 0 {
+			continue
+		}
+		upload.Set(uint32(index))
+	}
+	return &Progress{upload: upload}, nil
+}
+
+type Progress struct {
+	upload bitmap.Bitmap
+}
+
+func (p *Progress) IsUploaded(index int) bool {
+	if p == nil {
+		return false
+	}
+	return p.upload.Contains(uint32(index))
+}