复制项目

internal/msgtransfer/init.go

@@ -0,0 +1,188 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package msgtransfer
+
+import (
+	"context"
+	"time"
+
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/storage/cache"
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/storage/cache/mcache"
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/storage/cache/redis"
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/storage/database/mgo"
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/dbbuild"
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/mqbuild"
+	"github.com/openimsdk/tools/discovery"
+	"github.com/openimsdk/tools/mq"
+	"github.com/openimsdk/tools/utils/runtimeenv"
+
+	conf "git.imall.cloud/openim/open-im-server-deploy/pkg/common/config"
+	"git.imall.cloud/openim/open-im-server-deploy/pkg/common/storage/controller"
+	"github.com/openimsdk/tools/log"
+	"google.golang.org/grpc"
+)
+
+type MsgTransfer struct {
+	historyConsumer      mq.Consumer
+	historyMongoConsumer mq.Consumer
+	// This consumer aggregated messages, subscribed to the topic:toRedis,
+	//  the message is stored in redis, Incr Redis, and then the message is sent to toPush topic for push,
+	// and the message is sent to toMongo topic for persistence
+	historyHandler *OnlineHistoryRedisConsumerHandler
+	//This consumer handle message to mongo
+	historyMongoHandler *OnlineHistoryMongoConsumerHandler
+	ctx                 context.Context
+	//cancel              context.CancelFunc
+}
+
+type Config struct {
+	MsgTransfer    conf.MsgTransfer
+	RedisConfig    conf.Redis
+	MongodbConfig  conf.Mongo
+	KafkaConfig    conf.Kafka
+	Share          conf.Share
+	WebhooksConfig conf.Webhooks
+	Discovery      conf.Discovery
+	Index          conf.Index
+}
+
+func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error {
+	builder := mqbuild.NewBuilder(&config.KafkaConfig)
+
+	log.CInfo(ctx, "MSG-TRANSFER server is initializing", "runTimeEnv", runtimeenv.RuntimeEnvironment(), "prometheusPorts",
+		config.MsgTransfer.Prometheus.Ports, "index", config.Index)
+	dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig)
+	mgocli, err := dbb.Mongo(ctx)
+	if err != nil {
+		return err
+	}
+	rdb, err := dbb.Redis(ctx)
+	if err != nil {
+		return err
+	}
+
+	//if config.Discovery.Enable == conf.ETCD {
+	//	cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), []string{
+	//		config.MsgTransfer.GetConfigFileName(),
+	//		config.RedisConfig.GetConfigFileName(),
+	//		config.MongodbConfig.GetConfigFileName(),
+	//		config.KafkaConfig.GetConfigFileName(),
+	//		config.Share.GetConfigFileName(),
+	//		config.WebhooksConfig.GetConfigFileName(),
+	//		config.Discovery.GetConfigFileName(),
+	//		conf.LogConfigFileName,
+	//	})
+	//	cm.Watch(ctx)
+	//}
+	mongoProducer, err := builder.GetTopicProducer(ctx, config.KafkaConfig.ToMongoTopic)
+	if err != nil {
+		return err
+	}
+	pushProducer, err := builder.GetTopicProducer(ctx, config.KafkaConfig.ToPushTopic)
+	if err != nil {
+		return err
+	}
+	msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB())
+	if err != nil {
+		return err
+	}
+	var msgModel cache.MsgCache
+	if rdb == nil {
+		cm, err := mgo.NewCacheMgo(mgocli.GetDB())
+		if err != nil {
+			return err
+		}
+		msgModel = mcache.NewMsgCache(cm, msgDocModel)
+	} else {
+		msgModel = redis.NewMsgCache(rdb, msgDocModel)
+	}
+	seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB())
+	if err != nil {
+		return err
+	}
+	seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation)
+	seqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB())
+	if err != nil {
+		return err
+	}
+	seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser)
+	msgTransferDatabase, err := controller.NewMsgTransferDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, mongoProducer, pushProducer)
+	if err != nil {
+		return err
+	}
+	historyConsumer, err := builder.GetTopicConsumer(ctx, config.KafkaConfig.ToRedisTopic)
+	if err != nil {
+		return err
+	}
+	historyMongoConsumer, err := builder.GetTopicConsumer(ctx, config.KafkaConfig.ToMongoTopic)
+	if err != nil {
+		return err
+	}
+	historyHandler, err := NewOnlineHistoryRedisConsumerHandler(ctx, client, config, msgTransferDatabase)
+	if err != nil {
+		return err
+	}
+	historyMongoHandler := NewOnlineHistoryMongoConsumerHandler(msgTransferDatabase, config)
+
+	msgTransfer := &MsgTransfer{
+		historyConsumer:      historyConsumer,
+		historyMongoConsumer: historyMongoConsumer,
+		historyHandler:       historyHandler,
+		historyMongoHandler:  historyMongoHandler,
+	}
+
+	return msgTransfer.Start(ctx)
+}
+
+func (m *MsgTransfer) Start(ctx context.Context) error {
+	m.ctx = ctx
+
+	go func() {
+		for {
+			if err := m.historyConsumer.Subscribe(m.ctx, m.historyHandler.HandlerRedisMessage); err != nil {
+				log.ZError(m.ctx, "historyConsumer err, will retry in 5 seconds", err)
+				time.Sleep(5 * time.Second)
+				continue
+			}
+			// Subscribe returned normally (possibly due to context cancellation), retry immediately
+			log.ZWarn(m.ctx, "historyConsumer Subscribe returned normally, will retry immediately", nil)
+		}
+	}()
+
+	go func() {
+		fn := func(msg mq.Message) error {
+			m.historyMongoHandler.HandleChatWs2Mongo(msg)
+			return nil
+		}
+		for {
+			if err := m.historyMongoConsumer.Subscribe(m.ctx, fn); err != nil {
+				log.ZError(m.ctx, "historyMongoConsumer err, will retry in 5 seconds", err)
+				time.Sleep(5 * time.Second)
+				continue
+			}
+			// Subscribe returned normally (possibly due to context cancellation), retry immediately
+			log.ZWarn(m.ctx, "historyMongoConsumer Subscribe returned normally, will retry immediately", nil)
+		}
+	}()
+
+	go m.historyHandler.HandleUserHasReadSeqMessages(m.ctx)
+
+	err := m.historyHandler.redisMessageBatches.Start()
+	if err != nil {
+		return err
+	}
+	<-m.ctx.Done()
+	return m.ctx.Err()
+}