Compare commits
No commits in common. "7d09ba028650f253b64cc6f0eff786a533ee987d" and "6fc3412837f36d015021b28c0fbfcf81da916131" have entirely different histories.
7d09ba0286
...
6fc3412837
|
|
@ -36,7 +36,7 @@ func NewStartCmd() *cobra.Command {
|
|||
Short: "Start hpds_web application",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
var (
|
||||
cfg *config.ControlCenterConfig
|
||||
cfg *config.WebConfig
|
||||
err error
|
||||
)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
|
@ -80,7 +80,7 @@ func NewStartCmd() *cobra.Command {
|
|||
consulCfg, err := discover.NewConsulConfig(fmt.Sprintf("%s:%d", cfg.Consul.Host, cfg.Consul.Port), cfg.Name, cfg.Name, cfg.Consul.Host, cfg.Consul.Port,
|
||||
tags, 300, 300, 300)
|
||||
must(err)
|
||||
config.Cfg = cfg
|
||||
|
||||
//连接数据库
|
||||
model.New(cfg.Db.DriveName, cfg.Db.Conn, cfg.Mode == "dev")
|
||||
|
||||
|
|
|
|||
|
|
@ -27,12 +27,6 @@ cache:
|
|||
port: 6379
|
||||
db: 0
|
||||
pool_size: 10
|
||||
minio:
|
||||
protocol: http
|
||||
endpoint: 127.0.0.1:9000
|
||||
accessKeyId: root
|
||||
secretAccessKey: OIxv7QptYBO3
|
||||
bucket: jky-data
|
||||
node:
|
||||
host: 127.0.0.1
|
||||
port: 27188
|
||||
|
|
@ -42,14 +36,8 @@ functions:
|
|||
dataTag: 12
|
||||
mqType: 2
|
||||
- name: task-response
|
||||
dataTag: 24
|
||||
mqType: 2
|
||||
dataTag: 14
|
||||
mqType: 1
|
||||
- name: task-execute
|
||||
dataTag: 16
|
||||
mqType: 1
|
||||
- name: task-execute-log
|
||||
dataTag: 26
|
||||
mqType: 2
|
||||
- name: task-log
|
||||
dataTag: 28
|
||||
mqType: 1
|
||||
|
|
@ -10,11 +10,7 @@ import (
|
|||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var (
|
||||
Cfg *ControlCenterConfig
|
||||
)
|
||||
|
||||
type ControlCenterConfig struct {
|
||||
type WebConfig struct {
|
||||
Name string `yaml:"name,omitempty"`
|
||||
Host string `yaml:"host,omitempty"`
|
||||
Port int `yaml:"port,omitempty"`
|
||||
|
|
@ -81,7 +77,7 @@ type FuncConfig struct {
|
|||
MqType uint `yaml:"mqType"` //消息类型， 发布，1；订阅；2
|
||||
}
|
||||
|
||||
func ParseConfigByFile(path string) (cfg *ControlCenterConfig, err error) {
|
||||
func ParseConfigByFile(path string) (cfg *WebConfig, err error) {
|
||||
buffer, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -89,10 +85,10 @@ func ParseConfigByFile(path string) (cfg *ControlCenterConfig, err error) {
|
|||
return load(buffer)
|
||||
}
|
||||
|
||||
func load(buf []byte) (cfg *ControlCenterConfig, err error) {
|
||||
func load(buf []byte) (cfg *WebConfig, err error) {
|
||||
cViper := viper.New()
|
||||
cViper.SetConfigType("yaml")
|
||||
cfg = new(ControlCenterConfig)
|
||||
cfg = new(WebConfig)
|
||||
cfg.Funcs = make([]FuncConfig, 0)
|
||||
//cViper.ReadConfig(bytes.NewBuffer(buf))
|
||||
err = yaml.Unmarshal(buf, cfg)
|
||||
|
|
@ -103,7 +99,7 @@ func load(buf []byte) (cfg *ControlCenterConfig, err error) {
|
|||
return
|
||||
}
|
||||
|
||||
func UpdateLocalConfig(cfg *ControlCenterConfig, fn string) error {
|
||||
func UpdateLocalConfig(cfg *WebConfig, fn string) error {
|
||||
data, err := yaml.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -112,7 +108,7 @@ func UpdateLocalConfig(cfg *ControlCenterConfig, fn string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
func UpdateRemoteConfig(cfg *ControlCenterConfig) error {
|
||||
func UpdateRemoteConfig(cfg *WebConfig) error {
|
||||
consulClient, err := consulapi.NewClient(&consulapi.Config{Address: fmt.Sprintf("%s:%d", cfg.Consul.Host, cfg.Consul.Port)})
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -128,7 +124,7 @@ func UpdateRemoteConfig(cfg *ControlCenterConfig) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func GetRemoteConfig(remoteAddr, path string) (cfg *ControlCenterConfig, err error) {
|
||||
func GetRemoteConfig(remoteAddr, path string) (cfg *WebConfig, err error) {
|
||||
consulClient, err := consulapi.NewClient(&consulapi.Config{Address: remoteAddr})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ consul:
|
|||
timeout: 5
|
||||
deregister: 1
|
||||
db:
|
||||
conn: root:OIxv7QptYBO3@tcp(192.168.53.10:3306)/diagnosis?charset=utf8mb4
|
||||
conn: root:123456@tcp(127.0.0.1:3306)/hpds_jky?charset=utf8mb4
|
||||
drive_name: mysql
|
||||
cache:
|
||||
host: 127.0.0.1
|
||||
|
|
@ -29,12 +29,11 @@ cache:
|
|||
pool_size: 10
|
||||
minio:
|
||||
protocol: http
|
||||
endpoint: 192.168.53.10:9000
|
||||
endpoint: 127.0.0.1:9000
|
||||
accessKeyId: root
|
||||
secretAccessKey: OIxv7QptYBO3
|
||||
bucket: diagnosis
|
||||
node:
|
||||
host: 192.168.53.10
|
||||
host: 114.55.236.153
|
||||
port: 9188
|
||||
token: 06d36c6f5705507dae778fdce90d0767
|
||||
functions:
|
||||
|
|
@ -42,14 +41,8 @@ functions:
|
|||
dataTag: 12
|
||||
mqType: 2
|
||||
- name: task-response
|
||||
dataTag: 24
|
||||
dataTag: 14
|
||||
mqType: 2
|
||||
- name: task-execute
|
||||
dataTag: 16
|
||||
mqType: 1
|
||||
- name: task-execute-log
|
||||
dataTag: 26
|
||||
mqType: 2
|
||||
- name: task-log
|
||||
dataTag: 28
|
||||
mqType: 1
|
||||
37
go.mod
37
go.mod
|
|
@ -4,12 +4,10 @@ go 1.19
|
|||
|
||||
require (
|
||||
git.hpds.cc/Component/logging v0.0.0-20230106105738-e378e873921b
|
||||
git.hpds.cc/Component/network v0.0.0-20230405135741-a4ea724bab76
|
||||
git.hpds.cc/pavement/hpds_node v0.0.0-20230405153516-9403c4d01e12
|
||||
git.hpds.cc/pavement/hpds_node v0.0.0-20230402152619-41414aafa930
|
||||
github.com/go-sql-driver/mysql v1.7.0
|
||||
github.com/hashicorp/consul/api v1.20.0
|
||||
github.com/minio/minio-go v6.0.14+incompatible
|
||||
github.com/minio/minio-go/v7 v7.0.52
|
||||
github.com/spf13/cobra v1.6.1
|
||||
github.com/spf13/viper v1.15.0
|
||||
go.uber.org/zap v1.23.0
|
||||
|
|
@ -24,10 +22,10 @@ require (
|
|||
cloud.google.com/go/firestore v1.9.0 // indirect
|
||||
cloud.google.com/go/longrunning v0.3.0 // indirect
|
||||
git.hpds.cc/Component/mq_coder v0.0.0-20221010064749-174ae7ae3340 // indirect
|
||||
git.hpds.cc/Component/network v0.0.0-20221012021659-2433c68452d5 // indirect
|
||||
github.com/armon/go-metrics v0.4.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
|
|
@ -39,8 +37,6 @@ require (
|
|||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
|
|
@ -52,28 +48,22 @@ require (
|
|||
github.com/hashicorp/serf v0.10.1 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.1 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.16.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/lucas-clemente/quic-go v0.29.1 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
|
||||
github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect
|
||||
github.com/matoous/go-nanoid/v2 v2.0.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/minio/sha256-simd v1.0.0 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/nxadm/tail v1.4.8 // indirect
|
||||
github.com/onsi/ginkgo v1.16.4 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.2.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/quic-go/qtls-go1-19 v0.2.1 // indirect
|
||||
github.com/quic-go/qtls-go1-20 v0.1.1 // indirect
|
||||
github.com/quic-go/quic-go v0.33.0 // indirect
|
||||
github.com/rs/xid v1.4.0 // indirect
|
||||
github.com/sagikazarmark/crypt v0.9.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/spf13/afero v1.9.3 // indirect
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
|
|
@ -87,16 +77,16 @@ require (
|
|||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
golang.org/x/crypto v0.6.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
|
||||
golang.org/x/mod v0.6.0 // indirect
|
||||
golang.org/x/net v0.7.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
|
||||
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
|
||||
golang.org/x/net v0.4.0 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/sys v0.5.0 // indirect
|
||||
golang.org/x/text v0.7.0 // indirect
|
||||
golang.org/x/sys v0.3.0 // indirect
|
||||
golang.org/x/text v0.5.0 // indirect
|
||||
golang.org/x/time v0.1.0 // indirect
|
||||
golang.org/x/tools v0.2.0 // indirect
|
||||
golang.org/x/tools v0.1.12 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/api v0.107.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
|
|
@ -105,5 +95,6 @@ require (
|
|||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
|
||||
xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978 // indirect
|
||||
)
|
||||
|
|
|
|||
|
|
@ -30,16 +30,16 @@ func (s UInt32Slice) Swap(i, j int) {
|
|||
type ConsistentHashBalance struct {
|
||||
mux sync.RWMutex
|
||||
hash Hash
|
||||
replicas int //复制因子
|
||||
keys UInt32Slice //已排序的节点hash切片
|
||||
hashMap map[uint32]*model.NodeLastStateItem //节点哈希和key的map, 键是hash值，值是节点key
|
||||
replicas int //复制因子
|
||||
keys UInt32Slice //已排序的节点hash切片
|
||||
hashMap map[uint32]int64 //节点哈希和key的map, 键是hash值，值是节点key
|
||||
}
|
||||
|
||||
func NewConsistentHashBalance(replicas int, fn Hash) *ConsistentHashBalance {
|
||||
m := &ConsistentHashBalance{
|
||||
replicas: replicas,
|
||||
hash: fn,
|
||||
hashMap: make(map[uint32]*model.NodeLastStateItem),
|
||||
hashMap: make(map[uint32]int64),
|
||||
}
|
||||
if m.hash == nil {
|
||||
//最多32位，保证是一个2^32-1环
|
||||
|
|
@ -62,7 +62,7 @@ func (c *ConsistentHashBalance) Add(params model.NodeLastStateItem) error {
|
|||
for i := 0; i < c.replicas; i++ {
|
||||
hash := c.hash([]byte(strconv.Itoa(i) + fmt.Sprintf("%d", params.NodeId)))
|
||||
c.keys = append(c.keys, hash)
|
||||
c.hashMap[hash] = ¶ms
|
||||
c.hashMap[hash] = params.NodeId
|
||||
}
|
||||
|
||||
// 对所有虚拟节点的哈希值进行排序，方便之后进行二分查找
|
||||
|
|
@ -71,9 +71,9 @@ func (c *ConsistentHashBalance) Add(params model.NodeLastStateItem) error {
|
|||
}
|
||||
|
||||
// Get 方法根据给定的对象获取最靠近它的那个节点
|
||||
func (c *ConsistentHashBalance) Get(key int64) (*model.NodeLastStateItem, error) {
|
||||
func (c *ConsistentHashBalance) Get(key int64) (int64, error) {
|
||||
if c.IsEmpty() {
|
||||
return nil, errors.New("node is empty")
|
||||
return 0, errors.New("node is empty")
|
||||
}
|
||||
hash := c.hash([]byte(fmt.Sprintf("%d", key)))
|
||||
|
||||
|
|
|
|||
|
|
@ -4,5 +4,5 @@ import "hpds_control_center/model"
|
|||
|
||||
type LoadBalance interface {
|
||||
Add(model.NodeLastStateItem) error
|
||||
Get(int64) (*model.NodeLastStateItem, error)
|
||||
Get(int64) (int64, error)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,23 +9,24 @@ import (
|
|||
type RandomBalance struct {
|
||||
curIndex int
|
||||
|
||||
rss []*model.NodeLastStateItem
|
||||
rss []int64
|
||||
}
|
||||
|
||||
func (r *RandomBalance) Add(params model.NodeLastStateItem) error {
|
||||
r.rss = append(r.rss, ¶ms)
|
||||
nodeId := params.NodeId
|
||||
r.rss = append(r.rss, nodeId)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *RandomBalance) Next() *model.NodeLastStateItem {
|
||||
func (r *RandomBalance) Next() int64 {
|
||||
if len(r.rss) == 0 {
|
||||
return nil
|
||||
return 0
|
||||
}
|
||||
r.curIndex = rand.Intn(len(r.rss))
|
||||
return r.rss[r.curIndex]
|
||||
}
|
||||
|
||||
func (r *RandomBalance) Get(int64) (*model.NodeLastStateItem, error) {
|
||||
func (r *RandomBalance) Get(int64) (int64, error) {
|
||||
return r.Next(), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,17 +7,18 @@ import (
|
|||
// RoundRobinBalance 轮询负载均衡
|
||||
type RoundRobinBalance struct {
|
||||
curIndex int
|
||||
rss []*model.NodeLastStateItem
|
||||
rss []int64
|
||||
}
|
||||
|
||||
func (r *RoundRobinBalance) Add(params model.NodeLastStateItem) error {
|
||||
r.rss = append(r.rss, ¶ms)
|
||||
nodeId := params.NodeId
|
||||
r.rss = append(r.rss, nodeId)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *RoundRobinBalance) Next() *model.NodeLastStateItem {
|
||||
func (r *RoundRobinBalance) Next() int64 {
|
||||
if len(r.rss) == 0 {
|
||||
return nil
|
||||
return 0
|
||||
}
|
||||
lens := len(r.rss)
|
||||
if r.curIndex >= lens {
|
||||
|
|
@ -29,6 +30,6 @@ func (r *RoundRobinBalance) Next() *model.NodeLastStateItem {
|
|||
return curNode
|
||||
}
|
||||
|
||||
func (r *RoundRobinBalance) Get(int64) (*model.NodeLastStateItem, error) {
|
||||
func (r *RoundRobinBalance) Get(int64) (int64, error) {
|
||||
return r.Next(), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,11 +8,11 @@ import (
|
|||
type WeightRoundRobinBalance struct {
|
||||
curIndex int
|
||||
rss []*WeightNode
|
||||
rsw []*model.NodeLastStateItem
|
||||
rsw []int
|
||||
}
|
||||
|
||||
type WeightNode struct {
|
||||
node *model.NodeLastStateItem
|
||||
node model.NodeLastStateItem
|
||||
Weight int //初始化时对节点约定的权重
|
||||
currentWeight int //节点临时权重，每轮都会变化
|
||||
effectiveWeight int //有效权重, 默认与weight相同 , totalWeight = sum(effectiveWeight) //出现故障就-1
|
||||
|
|
@ -33,7 +33,7 @@ func (r *WeightRoundRobinBalance) Add(params model.NodeLastStateItem) error {
|
|||
weightTask = 100
|
||||
}
|
||||
node := &WeightNode{
|
||||
node: ¶ms,
|
||||
node: params,
|
||||
Weight: int(400 - weightCpu - weightMem - weightSwap - weightTask),
|
||||
}
|
||||
node.effectiveWeight = node.Weight
|
||||
|
|
@ -41,7 +41,7 @@ func (r *WeightRoundRobinBalance) Add(params model.NodeLastStateItem) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (r *WeightRoundRobinBalance) Next() *model.NodeLastStateItem {
|
||||
func (r *WeightRoundRobinBalance) Next() int64 {
|
||||
var best *WeightNode
|
||||
total := 0
|
||||
for i := 0; i < len(r.rss); i++ {
|
||||
|
|
@ -62,14 +62,14 @@ func (r *WeightRoundRobinBalance) Next() *model.NodeLastStateItem {
|
|||
}
|
||||
|
||||
if best == nil {
|
||||
return nil
|
||||
return 0
|
||||
}
|
||||
//5 变更临时权重为 临时权重-有效权重之和
|
||||
best.currentWeight -= total
|
||||
return best.node
|
||||
return best.node.NodeId
|
||||
}
|
||||
|
||||
func (r *WeightRoundRobinBalance) Get(int64) (*model.NodeLastStateItem, error) {
|
||||
func (r *WeightRoundRobinBalance) Get(int64) (int64, error) {
|
||||
return r.Next(), nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,8 @@
|
|||
package minio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.hpds.cc/Component/logging"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"io"
|
||||
"github.com/minio/minio-go"
|
||||
)
|
||||
|
||||
type MinClient struct {
|
||||
|
|
@ -14,11 +11,7 @@ type MinClient struct {
|
|||
}
|
||||
|
||||
func NewClient(ak, sak, ep string, useSSL bool, logger *logging.Logger) *MinClient {
|
||||
opt := &minio.Options{
|
||||
Creds: credentials.NewStaticV4(ak, sak, ""),
|
||||
Secure: useSSL,
|
||||
}
|
||||
client, err := minio.New(ep, opt)
|
||||
client, err := minio.New(ep, ak, sak, useSSL)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -29,18 +22,9 @@ func NewClient(ak, sak, ep string, useSSL bool, logger *logging.Logger) *MinClie
|
|||
}
|
||||
|
||||
func (cli *MinClient) UploadObject(fn, dst, bucket string) error {
|
||||
_, err := cli.Client.FPutObject(context.Background(), bucket, dst, fn, minio.PutObjectOptions{})
|
||||
_, err := cli.Client.FPutObject(bucket, dst, fn, minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cli *MinClient) GetObject(dstUrl, bucket string) ([]byte, error) {
|
||||
f, err := cli.Client.GetObject(context.Background(), bucket, dstUrl, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
imgByte, _ := io.ReadAll(f)
|
||||
return imgByte, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
package proto
|
||||
|
||||
type TaskLogPayload struct {
|
||||
PayloadType int `json:"payloadType"`
|
||||
TaskId int64 `json:"taskId"`
|
||||
TaskCode string `json:"taskCode"`
|
||||
NodeId int64 `json:"nodeId"`
|
||||
NodeGuid string `json:"nodeGuid"`
|
||||
TaskContent string `json:"taskContent"`
|
||||
Status int `json:"status"` //1：执行成功;2:执行失败
|
||||
EventTime int64 `json:"eventTime"`
|
||||
}
|
||||
type TaskLogProgress struct {
|
||||
PayloadType int `json:"payloadType"`
|
||||
TaskId int64 `json:"taskId"`
|
||||
TotalCount int64 `json:"totalCount"`
|
||||
CompletedCount int64 `json:"completedCount"`
|
||||
FailingCount int64 `json:"failingCount"`
|
||||
UnfinishedCount int64 `json:"unfinishedCount"`
|
||||
}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
package model
|
||||
|
||||
type TaskLog struct {
|
||||
TaskLogId int64 `xorm:"not null pk autoincr BIGINT(11)" json:"taskLogId"`
|
||||
TaskId int64 `xorm:"INT(11) index" json:"taskId"`
|
||||
NodeId int64 `xorm:"INT(11) index" json:"nodeId"`
|
||||
Content string `xorm:"LANGTEXT" json:"content"`
|
||||
CreateAt int64 `xorm:"created" json:"createAt"`
|
||||
UpdateAt int64 `xorm:"updated" json:"updateAt"`
|
||||
}
|
||||
|
||||
func InsertLog(taskLog *TaskLog) {
|
||||
_, _ = DB.Insert(taskLog)
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
package model
|
||||
|
||||
type Dataset struct {
|
||||
DatasetId int64 `xorm:"not null pk autoincr INT(11)" json:"datasetId"`
|
||||
DatasetName string `xorm:"varchar(200) not null" json:"datasetName"`
|
||||
DatasetDesc string `xorm:"varchar(200)" json:"datasetDesc"`
|
||||
StoreName string `xorm:"varchar(200)" json:"storeName"`
|
||||
CategoryId int `xorm:"not null SMALLINT default 1" json:"categoryId"` //业务分类, 1:道路 2:桥梁 3:隧道 4:边坡
|
||||
ProjectId int64 `xorm:"INT(11) not null default 0 index" json:"projectId"`
|
||||
OwnerId int64 `xorm:"INT(11) not null default 0 index" json:"ownerId"`
|
||||
Creator int64 `xorm:"INT(11) not null default 0 index" json:"creator"`
|
||||
Status int `xorm:"SMALLINT not null default 1" json:"status"`
|
||||
CreateAt int64 `xorm:"created" json:"createAt"`
|
||||
UpdateAt int64 `xorm:"updated" json:"updateAt"`
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
package model
|
||||
|
||||
type FileManager struct {
|
||||
FileId int64 `xorm:"not null pk autoincr INT(11)" json:"fileId"` //文件编号
|
||||
FileName string `xorm:"VARCHAR(200)" json:"fileName"` //文件名
|
||||
AccessUrl string `xorm:"VARCHAR(400)" json:"url"` //访问路径
|
||||
Scene string `xorm:"VARCHAR(40)" json:"scene"` //应用场景，0 : 其他, 1: 道路; 2: 桥梁; 3:隧道; 4: 边坡
|
||||
DataType int `xorm:"TINYINT index default 0" json:"dataType"` //数据类型，1:数据集;2:病害库;3:应用;0:其他
|
||||
DatasetId int64 `xorm:"INT(11) index default 0" json:"datasetId"` //数据集
|
||||
FileSize int64 `xorm:"BIGINT" json:"fileSize"` //文件大小
|
||||
FileMd5 string `xorm:"VARCHAR(64)" json:"fileMd5"` //文件MD5
|
||||
Creator int64 `xorm:"INT(11) index" json:"creator"` //上传人
|
||||
CreateAt int64 `xorm:"created" json:"createAt"` //上传时间
|
||||
UpdateAt int64 `xorm:"updated" json:"updateAt"` //更新时间
|
||||
}
|
||||
|
|
@ -24,7 +24,6 @@ func New(driveName, dsn string, showSql bool) {
|
|||
&Node{},
|
||||
&NodeState{},
|
||||
&Task{},
|
||||
&TaskLog{},
|
||||
&TaskResult{},
|
||||
)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -44,12 +44,3 @@ func GetAllNode(modelId int64) []Node {
|
|||
}
|
||||
return list
|
||||
}
|
||||
|
||||
func GetNodeById(nodeId int64) *Node {
|
||||
item := new(Node)
|
||||
h, err := DB.ID(nodeId).Get(item)
|
||||
if err != nil || !h {
|
||||
return nil
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,5 @@
|
|||
package model
|
||||
|
||||
import (
|
||||
"hpds_control_center/internal/proto"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Task struct {
|
||||
TaskId int64 `xorm:"not null pk autoincr INT(11)" json:"taskId"`
|
||||
ModelId int64 `xorm:"INT(11) index" json:"modelId"`
|
||||
|
|
@ -16,10 +11,6 @@ type Task struct {
|
|||
AppointmentTime string `xorm:"VARCHAR(30)" json:"appointmentTime"`
|
||||
StartTime int64 `xorm:"BIGINT" json:"startTime"`
|
||||
FinishTime int64 `xorm:"BIGINT" json:"finishTime"`
|
||||
TotalCount int64 `xorm:"INT" json:"totalCount"`
|
||||
FailingCount int64 `xorm:"INT" json:"failingCount"`
|
||||
CompletedCount int64 `xorm:"INT" json:"completedCount"`
|
||||
UnfinishedCount int64 `xorm:"INT" json:"unfinishedCount"`
|
||||
Status int `xorm:"not null SMALLINT default 0" json:"status"` // 1:等待执行; 2:执行中; 3:执行完成; 4:任务分配失败; 5:任务执行失败
|
||||
CreateAt int64 `xorm:"created" json:"createAt"`
|
||||
UpdateAt int64 `xorm:"updated" json:"updateAt"`
|
||||
|
|
@ -31,60 +22,3 @@ func UpdateTaskExecuteNode(id, nodeId int64) {
|
|||
item.NodeId = nodeId
|
||||
_, _ = DB.ID(id).Cols("node_id").Update(item)
|
||||
}
|
||||
|
||||
func UpdateTaskProgress(taskProgress *proto.TaskLogProgress) {
|
||||
task := new(Task)
|
||||
h, err := DB.ID(taskProgress.TaskId).Get(task)
|
||||
if err != nil || !h {
|
||||
return
|
||||
}
|
||||
if taskProgress.CompletedCount > task.CompletedCount {
|
||||
task.CompletedCount = taskProgress.CompletedCount
|
||||
}
|
||||
if taskProgress.FailingCount > task.FailingCount {
|
||||
task.FailingCount = taskProgress.FailingCount
|
||||
}
|
||||
if taskProgress.TotalCount != task.TotalCount && taskProgress.TotalCount > 0 {
|
||||
task.TotalCount = taskProgress.TotalCount
|
||||
}
|
||||
if taskProgress.UnfinishedCount != task.UnfinishedCount && taskProgress.UnfinishedCount >= 0 {
|
||||
task.UnfinishedCount = taskProgress.UnfinishedCount
|
||||
}
|
||||
if task.CompletedCount+task.FailingCount >= task.TotalCount {
|
||||
task.Status = 3
|
||||
} else {
|
||||
task.Status = 2
|
||||
}
|
||||
task.UpdateAt = time.Now().Unix()
|
||||
_, err = DB.ID(taskProgress.TaskId).Cols("completed_count", "failing_count", "total_count", "unfinished_count", "update_at").Update(task)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func UpdateTaskProgressByLog(res *TaskResult, isFailing bool) float64 {
|
||||
ret := -1.0
|
||||
item := new(Task)
|
||||
h, err := DB.ID(res.TaskId).Get(item)
|
||||
if err != nil || !h {
|
||||
return ret
|
||||
}
|
||||
if isFailing {
|
||||
item.FailingCount += 1
|
||||
} else {
|
||||
item.CompletedCount += 1
|
||||
}
|
||||
item.UnfinishedCount -= 1
|
||||
if item.CompletedCount+item.FailingCount >= item.TotalCount {
|
||||
item.FinishTime = time.Now().Unix()
|
||||
item.UnfinishedCount = 0
|
||||
item.Status = 3
|
||||
ret = 1.0
|
||||
}
|
||||
item.UpdateAt = time.Now().Unix()
|
||||
_, _ = DB.ID(res.TaskId).Cols("completed_count", "failing_count", "total_count", "unfinished_count", "update_at", "finish_time", "status").Update(item)
|
||||
if item.TotalCount > 0 {
|
||||
return 1 - float64(item.CompletedCount)/float64(item.TotalCount)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package model
|
|||
type TaskResult struct {
|
||||
ResultId int64 `xorm:"not null pk autoincr INT(11)" json:"resultId"`
|
||||
TaskId int64 `xorm:"INT(11) index" json:"taskId"`
|
||||
FileId int64 `xorm:"INT(11) index" json:"fileId"`
|
||||
TaskCode string `xorm:"varchar(200)" json:"taskCode"`
|
||||
ModelId int64 `xorm:"INT(11)" json:"modelId"`
|
||||
NodeId int64 `xorm:"INT(11)" json:"nodeId"`
|
||||
|
|
@ -12,5 +11,5 @@ type TaskResult struct {
|
|||
SubDataset string `xorm:"varchar(200)" json:"subDataset"`
|
||||
DatasetId int64 `xorm:"INT(11)" json:"datasetId"`
|
||||
SrcPath string `xorm:"varchar(500)" json:"srcPath"`
|
||||
Result string `xorm:"LONGTEXT" json:"result"`
|
||||
Result string `xorm:"TEXT" json:"result"`
|
||||
}
|
||||
|
|
|
|||
443
mq/index.go
443
mq/index.go
|
|
@ -1,31 +1,20 @@
|
|||
package mq
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.hpds.cc/Component/logging"
|
||||
"git.hpds.cc/Component/network/frame"
|
||||
"github.com/google/uuid"
|
||||
"go.uber.org/zap"
|
||||
"hpds_control_center/config"
|
||||
"hpds_control_center/internal/balance"
|
||||
"hpds_control_center/internal/minio"
|
||||
"hpds_control_center/internal/proto"
|
||||
"hpds_control_center/model"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.hpds.cc/pavement/hpds_node"
|
||||
)
|
||||
|
||||
var (
|
||||
MqList []HpdsMqNode
|
||||
TaskList = make(map[int64]*TaskItem)
|
||||
)
|
||||
var MqList []HpdsMqNode
|
||||
|
||||
type HpdsMqNode struct {
|
||||
MqType uint
|
||||
|
|
@ -35,15 +24,6 @@ type HpdsMqNode struct {
|
|||
Logger *logging.Logger
|
||||
}
|
||||
|
||||
type TaskItem struct {
|
||||
TaskId int64
|
||||
TotalCount int64
|
||||
CompletedCount int64
|
||||
FailingCount int64
|
||||
UnfinishedCount int64
|
||||
LastSendTime int64
|
||||
}
|
||||
|
||||
func must(logger *logging.Logger, err error) {
|
||||
if err != nil {
|
||||
if logger != nil {
|
||||
|
|
@ -63,7 +43,7 @@ func NewMqClient(funcs []config.FuncConfig, node config.HpdsNode, logger *loggin
|
|||
sf := hpds_node.NewStreamFunction(
|
||||
v.Name,
|
||||
hpds_node.WithMqAddr(fmt.Sprintf("%s:%d", node.Host, node.Port)),
|
||||
hpds_node.WithObserveDataTags(frame.Tag(v.DataTag)),
|
||||
hpds_node.WithObserveDataTags(v.DataTag),
|
||||
hpds_node.WithCredential(node.Token),
|
||||
)
|
||||
err = sf.Connect()
|
||||
|
|
@ -77,10 +57,6 @@ func NewMqClient(funcs []config.FuncConfig, node config.HpdsNode, logger *loggin
|
|||
switch v.Name {
|
||||
case "task-request":
|
||||
_ = sf.SetHandler(TaskRequestHandler)
|
||||
case "task-response":
|
||||
_ = sf.SetHandler(TaskResponseHandler)
|
||||
case "task-execute-log":
|
||||
_ = sf.SetHandler(TaskExecuteLogHandler)
|
||||
default:
|
||||
|
||||
}
|
||||
|
|
@ -99,7 +75,7 @@ func NewMqClient(funcs []config.FuncConfig, node config.HpdsNode, logger *loggin
|
|||
EndPoint: ap,
|
||||
}
|
||||
must(logger, err)
|
||||
ap.SetDataTag(frame.Tag(v.DataTag))
|
||||
ap.SetDataTag(v.DataTag)
|
||||
mqList = append(mqList, nodeInfo)
|
||||
}
|
||||
|
||||
|
|
@ -125,7 +101,8 @@ func GenerateAndSendData(stream hpds_node.AccessPoint, data []byte) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func TaskRequestHandler(data []byte) (frame.Tag, []byte) {
|
||||
func TaskRequestHandler(data []byte) (byte, []byte) {
|
||||
fmt.Println("接收数据", string(data))
|
||||
cmd := new(InstructionReq)
|
||||
err := json.Unmarshal(data, cmd)
|
||||
if err != nil {
|
||||
|
|
@ -134,225 +111,70 @@ func TaskRequestHandler(data []byte) (frame.Tag, []byte) {
|
|||
switch cmd.Command {
|
||||
case TaskAdd:
|
||||
payload := cmd.Payload.(map[string]interface{})
|
||||
if len(payload["subDataset"].(string)) > 0 {
|
||||
if payload["nodeId"].(float64) == 0 {
|
||||
//根据业务属性进行分配节点
|
||||
m := model.GetModelById(int64(payload["modelId"].(float64)))
|
||||
var nodeList []model.Node
|
||||
//todo 需要增加模型下发记录
|
||||
if m.IsLightWeight {
|
||||
nodeList = model.GetLightWeight(m.ModelId)
|
||||
} else {
|
||||
nodeList = model.GetAllNode(m.ModelId)
|
||||
}
|
||||
if nodeList != nil {
|
||||
if len(nodeList) > 1 {
|
||||
//这里采用加权算法，权重采用CPU占用+mem使用+任务执行状态
|
||||
list := model.GetNodeState(nodeList)
|
||||
lb := balance.LoadBalanceFactory(balance.LbWeightRoundRobin)
|
||||
for _, v := range list {
|
||||
_ = lb.Add(v)
|
||||
}
|
||||
nodeId, _ := lb.Get(0)
|
||||
if nodeId == nil {
|
||||
//todo 需要增加未能获取的处理
|
||||
}
|
||||
payload["nodeId"] = nodeId.NodeId
|
||||
payload["nodeGuid"] = nodeId.NodeGuid
|
||||
|
||||
cmd := &InstructionReq{
|
||||
Command: TaskExecute,
|
||||
Payload: payload,
|
||||
}
|
||||
|
||||
pData, _ := json.Marshal(cmd)
|
||||
cli := GetMqClient("task-execute", 1)
|
||||
if cli != nil {
|
||||
_ = GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), pData)
|
||||
}
|
||||
model.UpdateTaskExecuteNode(int64(payload["taskId"].(float64)), nodeId.NodeId)
|
||||
} else {
|
||||
payload["nodeId"] = nodeList[0].NodeId
|
||||
issue := new(model.IssueModel)
|
||||
h, _ := model.DB.Where("model_id=? and node_id =?", int64(payload["modelId"].(float64)), nodeList[0].NodeId).Get(issue)
|
||||
if !h {
|
||||
|
||||
}
|
||||
payload["issueResult"] = issue.IssueResult
|
||||
cmd := &InstructionReq{
|
||||
Command: TaskExecute,
|
||||
Payload: payload,
|
||||
}
|
||||
pData, _ := json.Marshal(cmd)
|
||||
cli := GetMqClient("task-execute", 1)
|
||||
if cli != nil {
|
||||
_ = GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), pData)
|
||||
}
|
||||
model.UpdateTaskExecuteNode(int64(payload["taskId"].(float64)), nodeList[0].NodeId)
|
||||
}
|
||||
} else {
|
||||
|
||||
}
|
||||
|
||||
if payload["nodeId"].(float64) == 0 {
|
||||
//根据业务属性进行分配节点
|
||||
m := model.GetModelById(int64(payload["modelId"].(float64)))
|
||||
var nodeList []model.Node
|
||||
//todo 需要增加模型下发记录
|
||||
if m.IsLightWeight {
|
||||
nodeList = model.GetLightWeight(m.ModelId)
|
||||
} else {
|
||||
cmd := &InstructionReq{
|
||||
Command: TaskExecute,
|
||||
Payload: payload,
|
||||
}
|
||||
pData, _ := json.Marshal(cmd)
|
||||
cli := GetMqClient("task-execute", 1)
|
||||
if cli != nil {
|
||||
_ = GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), pData)
|
||||
}
|
||||
nodeList = model.GetAllNode(m.ModelId)
|
||||
}
|
||||
} else {
|
||||
if len(payload["datasetArr"].(string)) > 0 {
|
||||
GoroutinueChan := make(chan bool, 5)
|
||||
if payload["nodeId"].(float64) == 0 {
|
||||
//根据业务属性进行分配节点
|
||||
m := model.GetModelById(int64(payload["modelId"].(float64)))
|
||||
var nodeList []model.Node
|
||||
//todo 需要增加模型下发记录
|
||||
if m.IsLightWeight {
|
||||
nodeList = model.GetLightWeight(m.ModelId)
|
||||
} else {
|
||||
nodeList = model.GetAllNode(m.ModelId)
|
||||
if nodeList != nil {
|
||||
if len(nodeList) > 1 {
|
||||
//这里采用加权算法，权重采用CPU占用+mem使用+任务执行状态
|
||||
list := model.GetNodeState(nodeList)
|
||||
lb := balance.LoadBalanceFactory(balance.LbWeightRoundRobin)
|
||||
for _, v := range list {
|
||||
_ = lb.Add(v)
|
||||
}
|
||||
if nodeList != nil {
|
||||
if len(nodeList) > 1 {
|
||||
//这里采用加权算法，权重采用CPU占用+mem使用+任务执行状态
|
||||
list := model.GetNodeState(nodeList)
|
||||
lb := balance.LoadBalanceFactory(balance.LbWeightRoundRobin)
|
||||
for _, v := range list {
|
||||
_ = lb.Add(v)
|
||||
}
|
||||
nodeId, _ := lb.Get(0)
|
||||
if nodeId == nil {
|
||||
//todo 需要增加未能获取的处理
|
||||
}
|
||||
payload["nodeId"] = nodeId.NodeId
|
||||
payload["nodeGuid"] = nodeId.NodeGuid
|
||||
|
||||
cmd := &InstructionReq{
|
||||
Command: TaskExecute,
|
||||
Payload: payload,
|
||||
}
|
||||
|
||||
pData, _ := json.Marshal(cmd)
|
||||
cli := GetMqClient("task-execute", 1)
|
||||
if cli != nil {
|
||||
_ = GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), pData)
|
||||
}
|
||||
model.UpdateTaskExecuteNode(int64(payload["taskId"].(float64)), nodeId.NodeId)
|
||||
} else {
|
||||
payload["nodeId"] = nodeList[0].NodeId
|
||||
issue := new(model.IssueModel)
|
||||
h, _ := model.DB.Where("model_id=? and node_id =?", int64(payload["modelId"].(float64)), nodeList[0].NodeId).Get(issue)
|
||||
if !h {
|
||||
|
||||
}
|
||||
payload["issueResult"] = issue.IssueResult
|
||||
cmd := &InstructionReq{
|
||||
Command: TaskExecute,
|
||||
Payload: payload,
|
||||
}
|
||||
pData, _ := json.Marshal(cmd)
|
||||
cli := GetMqClient("task-execute", 1)
|
||||
if cli != nil {
|
||||
_ = GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), pData)
|
||||
}
|
||||
model.UpdateTaskExecuteNode(int64(payload["taskId"].(float64)), nodeList[0].NodeId)
|
||||
}
|
||||
} else {
|
||||
nodeId, _ := lb.Get(0)
|
||||
|
||||
payload["nodeId"] = nodeId
|
||||
cmd := &InstructionReq{
|
||||
Command: TaskExecute,
|
||||
Payload: payload,
|
||||
}
|
||||
|
||||
pData, _ := json.Marshal(cmd)
|
||||
cli := GetMqClient("task-execute", 1)
|
||||
if cli != nil {
|
||||
_ = GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), pData)
|
||||
}
|
||||
model.UpdateTaskExecuteNode(int64(payload["taskId"].(float64)), nodeId)
|
||||
} else {
|
||||
node := model.GetNodeById(int64(payload["nodeId"].(float64)))
|
||||
if node != nil {
|
||||
payload["nodeGuid"] = node.NodeGuid
|
||||
payload["nodeId"] = nodeList[0].NodeId
|
||||
issue := new(model.IssueModel)
|
||||
h, _ := model.DB.Where("model_id=? and node_id =?", int64(payload["modelId"].(float64)), nodeList[0].NodeId).Get(issue)
|
||||
if !h {
|
||||
|
||||
}
|
||||
payload["issueResult"] = issue.IssueResult
|
||||
cmd := &InstructionReq{
|
||||
Command: TaskExecute,
|
||||
Payload: payload,
|
||||
}
|
||||
pData, _ := json.Marshal(cmd)
|
||||
cli := GetMqClient("task-execute", 1)
|
||||
if cli != nil {
|
||||
_ = GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), pData)
|
||||
}
|
||||
model.UpdateTaskExecuteNode(int64(payload["taskId"].(float64)), nodeList[0].NodeId)
|
||||
}
|
||||
//数据集处理
|
||||
datasetArr := strings.Split(payload["datasetArr"].(string), ",")
|
||||
//for _, val := range datasetArr {
|
||||
// dId, err := strconv.ParseInt(val, 10, 64)
|
||||
// if err != nil {
|
||||
// continue
|
||||
// }
|
||||
// dt := new(model.Dataset)
|
||||
// _, _ = model.DB.ID(dId).Get(dt)
|
||||
fileList := make([]model.FileManager, 0)
|
||||
err = model.DB.In("dataset_id", datasetArr).
|
||||
Find(&fileList)
|
||||
if err != nil {
|
||||
} else {
|
||||
|
||||
}
|
||||
item := &TaskItem{
|
||||
TaskId: int64(payload["taskId"].(float64)),
|
||||
TotalCount: int64(len(fileList)),
|
||||
CompletedCount: 0,
|
||||
FailingCount: 0,
|
||||
UnfinishedCount: int64(len(fileList)),
|
||||
LastSendTime: time.Now().Unix(),
|
||||
}
|
||||
TaskList[int64(payload["taskId"].(float64))] = item
|
||||
//获取任务总数，并入库
|
||||
taskProgress := &proto.TaskLogProgress{
|
||||
PayloadType: 1,
|
||||
TaskId: int64(payload["taskId"].(float64)),
|
||||
TotalCount: int64(len(fileList)),
|
||||
CompletedCount: 0,
|
||||
FailingCount: 0,
|
||||
UnfinishedCount: int64(len(fileList)),
|
||||
}
|
||||
model.UpdateTaskProgress(taskProgress)
|
||||
taskLog := &model.TaskLog{
|
||||
TaskId: int64(payload["taskId"].(float64)),
|
||||
NodeId: int64(payload["nodeId"].(float64)),
|
||||
Content: fmt.Sprintf("[%s] 在节点[%s]上开始执行任务，任务数量共[%d]", time.Now().Format("2006-01-02 15:04:05"), payload["nodeGuid"].(string), taskProgress.TotalCount),
|
||||
CreateAt: time.Now().Unix(),
|
||||
UpdateAt: time.Now().Unix(),
|
||||
}
|
||||
model.InsertLog(taskLog)
|
||||
taskProgressCmd := &InstructionReq{
|
||||
Command: TaskLog,
|
||||
Payload: taskProgress,
|
||||
}
|
||||
deliver("task-log", 1, taskProgressCmd)
|
||||
}
|
||||
|
||||
//数据集处理
|
||||
minioCli := minio.NewClient(config.Cfg.Minio.AccessKeyId, config.Cfg.Minio.SecretAccessKey, config.Cfg.Minio.Endpoint, false, logging.L())
|
||||
for _, v := range fileList {
|
||||
GoroutinueChan <- true
|
||||
go func(fa model.FileManager, payload map[string]interface{}) {
|
||||
p := make(map[string]interface{})
|
||||
for key, val := range payload {
|
||||
p[key] = val
|
||||
}
|
||||
dstPath := strings.Replace(fa.AccessUrl, fmt.Sprintf("%s://%s/", config.Cfg.Minio.Protocol, config.Cfg.Minio.Endpoint), "", 1)
|
||||
|
||||
dstPath = strings.Replace(dstPath, config.Cfg.Minio.Bucket, "", 1)
|
||||
imgByte, _ := minioCli.GetObject(dstPath, config.Cfg.Minio.Bucket)
|
||||
fc := FileCapture{
|
||||
FileId: fa.FileId,
|
||||
FileName: fa.FileName,
|
||||
File: base64.StdEncoding.EncodeToString(imgByte),
|
||||
DatasetName: p["datasetName"].(string),
|
||||
CaptureTime: fa.CreateAt,
|
||||
}
|
||||
p["single"] = fc
|
||||
taskCode, _ := uuid.NewUUID()
|
||||
p["taskCode"] = taskCode.String()
|
||||
cmd := &InstructionReq{
|
||||
Command: TaskExecute,
|
||||
Payload: p,
|
||||
}
|
||||
deliver("task-execute", 1, cmd)
|
||||
<-GoroutinueChan
|
||||
}(v, payload)
|
||||
}
|
||||
//}
|
||||
} else {
|
||||
cmd := &InstructionReq{
|
||||
Command: TaskExecute,
|
||||
Payload: payload,
|
||||
}
|
||||
pData, _ := json.Marshal(cmd)
|
||||
cli := GetMqClient("task-execute", 1)
|
||||
if cli != nil {
|
||||
_ = GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), pData)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -387,160 +209,25 @@ func TaskRequestHandler(data []byte) (frame.Tag, []byte) {
|
|||
item.UpdateAt = time.Now().Unix()
|
||||
_, _ = model.DB.Insert(item)
|
||||
}
|
||||
//case TaskResponse:
|
||||
// payload := cmd.Payload.(map[string]interface{})
|
||||
// item := new(model.TaskResult)
|
||||
// item.TaskId = int64(payload["taskId"].(float64))
|
||||
// item.TaskCode = payload["taskCode"].(string)
|
||||
// item.NodeId = int64(payload["nodeId"].(float64))
|
||||
// item.ModelId = int64(payload["modelId"].(float64))
|
||||
// item.StartTime = int64(payload["startTime"].(float64))
|
||||
// item.FinishTime = int64(payload["finishTime"].(float64))
|
||||
// item.SubDataset = payload["subDataset"].(string)
|
||||
// item.DatasetId = int64(payload["datasetArr"].(float64))
|
||||
// item.SrcPath = payload["srcPath"].(string)
|
||||
// item.Result = payload["body"].(string)
|
||||
// _, _ = model.DB.Insert(item)
|
||||
// //fn := payload["fileName"].(string)
|
||||
// //dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(payload["file"].(string)))
|
||||
|
||||
default:
|
||||
|
||||
}
|
||||
return frame.Tag(cmd.Command), nil
|
||||
}
|
||||
|
||||
func TaskResponseHandler(data []byte) (frame.Tag, []byte) {
|
||||
cmd := new(InstructionReq)
|
||||
err := json.Unmarshal(data, cmd)
|
||||
if err != nil {
|
||||
return 0x0B, []byte(err.Error())
|
||||
}
|
||||
switch cmd.Command {
|
||||
case TaskResponse:
|
||||
payload := cmd.Payload.(map[string]interface{})
|
||||
item := new(model.TaskResult)
|
||||
item.TaskId = int64(payload["taskId"].(float64))
|
||||
if _, ok := payload["taskCode"]; ok && payload["taskCode"] != nil {
|
||||
item.TaskCode = payload["taskCode"].(string)
|
||||
}
|
||||
if _, ok := payload["fileId"]; ok {
|
||||
item.FileId = int64(payload["fileId"].(float64))
|
||||
}
|
||||
item.TaskCode = payload["taskCode"].(string)
|
||||
item.NodeId = int64(payload["nodeId"].(float64))
|
||||
item.ModelId = int64(payload["modelId"].(float64))
|
||||
item.StartTime = int64(payload["startTime"].(float64))
|
||||
item.FinishTime = int64(payload["finishTime"].(float64))
|
||||
if _, ok := payload["subDataset"]; ok {
|
||||
item.SubDataset = payload["subDataset"].(string)
|
||||
}
|
||||
item.DatasetId, _ = strconv.ParseInt(payload["datasetArr"].(string), 10, 64)
|
||||
if _, ok := payload["srcPath"]; ok && payload["srcPath"] != nil {
|
||||
item.SrcPath = payload["srcPath"].(string)
|
||||
}
|
||||
if _, ok := payload["body"]; ok {
|
||||
item.Result = payload["body"].(string)
|
||||
}
|
||||
isFailing := false
|
||||
if _, ok := payload["code"]; ok && int(payload["code"].(float64)) == 500 {
|
||||
item.Result = payload["msg"].(string)
|
||||
isFailing = true
|
||||
}
|
||||
_, err = model.DB.Insert(item)
|
||||
if err != nil {
|
||||
fmt.Println("接收TaskResponse数据出错", err)
|
||||
}
|
||||
//更新运行进度
|
||||
rat := model.UpdateTaskProgressByLog(item, isFailing)
|
||||
var (
|
||||
ratStr string
|
||||
)
|
||||
if rat > 0 && rat < 1 {
|
||||
ratStr = fmt.Sprintf("[已处理%2.f,剩余%2.f未处理]", 1-rat, rat)
|
||||
} else if rat == 1 {
|
||||
ratStr = "[已全部处理]"
|
||||
}
|
||||
taskLog := new(model.TaskLog)
|
||||
taskLog.TaskId = item.TaskId
|
||||
taskLog.NodeId = item.NodeId
|
||||
if len(item.SrcPath) > 0 {
|
||||
taskLog.Content = fmt.Sprintf("[%s] 图片%s处理完成 %s ", time.Unix(item.FinishTime, 0).Format("2006-01-02 15:04:05"),
|
||||
item.SrcPath, ratStr)
|
||||
} else {
|
||||
taskLog.Content = fmt.Sprintf("[%s] %s", time.Unix(item.FinishTime, 0).Format("2006-01-02 15:04:05"),
|
||||
ratStr)
|
||||
}
|
||||
model.InsertLog(taskLog)
|
||||
item.SubDataset = payload["subDataset"].(string)
|
||||
item.DatasetId = int64(payload["datasetArr"].(float64))
|
||||
item.SrcPath = payload["srcPath"].(string)
|
||||
item.Result = payload["body"].(string)
|
||||
_, _ = model.DB.Insert(item)
|
||||
//fn := payload["fileName"].(string)
|
||||
//dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(payload["file"].(string)))
|
||||
|
||||
default:
|
||||
}
|
||||
return frame.Tag(cmd.Command), nil
|
||||
}
|
||||
|
||||
func deliver(topic string, mqType uint, payload interface{}) {
|
||||
cli := GetMqClient(topic, mqType)
|
||||
pData, _ := json.Marshal(payload)
|
||||
_ = GenerateAndSendData(cli.EndPoint.(hpds_node.AccessPoint), pData)
|
||||
}
|
||||
|
||||
func TaskExecuteLogHandler(data []byte) (frame.Tag, []byte) {
|
||||
cmd := new(InstructionReq)
|
||||
err := json.Unmarshal(data, cmd)
|
||||
if err != nil {
|
||||
return 0x0B, []byte(err.Error())
|
||||
}
|
||||
payload := cmd.Payload.(map[string]interface{})
|
||||
var l sync.Mutex
|
||||
l.Lock()
|
||||
taskId := int64(payload["taskId"].(float64))
|
||||
if item, ok := TaskList[taskId]; ok {
|
||||
item.UnfinishedCount -= 1
|
||||
if int(payload["status"].(float64)) == 1 {
|
||||
item.CompletedCount += 1
|
||||
}
|
||||
if int(payload["status"].(float64)) == 2 {
|
||||
item.FailingCount += 1
|
||||
}
|
||||
if item.UnfinishedCount <= 0 || time.Now().Unix()-item.LastSendTime > 5000 {
|
||||
//发送完成消息
|
||||
taskProgress := &proto.TaskLogProgress{
|
||||
PayloadType: 1,
|
||||
TaskId: item.TaskId,
|
||||
TotalCount: item.TotalCount,
|
||||
CompletedCount: item.CompletedCount,
|
||||
FailingCount: item.FailingCount,
|
||||
UnfinishedCount: item.UnfinishedCount,
|
||||
}
|
||||
//model.UpdateTaskProgress(taskProgress)
|
||||
taskProgressCmd := &InstructionReq{
|
||||
Command: TaskLog,
|
||||
Payload: taskProgress,
|
||||
}
|
||||
deliver("task-log", 1, taskProgressCmd)
|
||||
if item.UnfinishedCount <= 0 {
|
||||
delete(TaskList, item.TaskId)
|
||||
} else {
|
||||
item.LastSendTime = time.Now().Unix()
|
||||
}
|
||||
}
|
||||
taskLog := &proto.TaskLogPayload{
|
||||
PayloadType: 2,
|
||||
TaskId: item.TaskId,
|
||||
TaskCode: payload["taskCode"].(string),
|
||||
NodeId: int64(payload["nodeId"].(float64)),
|
||||
NodeGuid: payload["nodeGuid"].(string),
|
||||
TaskContent: payload["taskContent"].(string),
|
||||
Status: int(payload["status"].(float64)),
|
||||
EventTime: int64(payload["eventTime"].(float64)),
|
||||
}
|
||||
taskLogCmd := &InstructionReq{
|
||||
Command: TaskLog,
|
||||
Payload: taskLog,
|
||||
}
|
||||
deliver("task-log", 1, taskLogCmd)
|
||||
}
|
||||
l.Unlock()
|
||||
return frame.Tag(cmd.Command), nil
|
||||
return byte(cmd.Command), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,36 +7,9 @@ const (
|
|||
TaskResponse
|
||||
ModelIssueRepeater
|
||||
ModelIssueResponse
|
||||
TaskExecuteLog
|
||||
TaskLog
|
||||
)
|
||||
|
||||
type InstructionReq struct {
|
||||
Command int `json:"command"`
|
||||
Payload interface{} `json:"payload"`
|
||||
}
|
||||
|
||||
type FileCapture struct {
|
||||
FileId int64 `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
File string `json:"file"`
|
||||
DatasetName string `json:"datasetName"`
|
||||
CaptureTime int64 `json:"captureTime"`
|
||||
}
|
||||
|
||||
type TaskLogPayload struct {
|
||||
TaskId int64 `json:"taskId"`
|
||||
TaskCode string `json:"taskCode"`
|
||||
NodeId int64 `json:"nodeId"`
|
||||
NodeGuid string `json:"nodeGuid"`
|
||||
Payload string `json:"taskContent"`
|
||||
EventTime int64 `json:"eventTime"`
|
||||
}
|
||||
|
||||
type TaskLogProgress struct {
|
||||
TaskId int64 `json:"taskId"`
|
||||
TotalCount int64 `json:"totalCount"`
|
||||
CompletedCount int64 `json:"completedCount"`
|
||||
FailingCount int64 `json:"failingCount"`
|
||||
UnfinishedCount int64 `json:"unfinishedCount"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
package test
|
||||
|
||||
// {\"command\":1,\"payload\":{\"datasetArr\":\"2\",\"d":\"绕城道路测试\",\"datasetPath\":\"test_road\",\"httpUrl\":\"/\",\"inPath\":\"/in\",\"modelCommand\":\"\",\"modelId\":3,\"modelVersion\":\"v2.0\",\"nodeId\":1,\"h\":\"/out\",\"subDataTag\":100,\"subDataset\":\"\",\"taskId\":29,\"workflow\":\"\"}}
|
||||
//
|
||||
//func TestTaskRequestHandler(t *testing.T) {
|
||||
// data := []byte(`{"command":1,"payload":{"datasetArr":"2","d":\"绕城道路测试\",\"datasetPath\":\"test_road\",\"httpUrl\":\"/\",\"inPath\":\"/in\",\"modelCommand\":\"\",\"modelId\":3,\"modelVersion\":\"v2.0\",\"nodeId\":1,\"h\":\"/out\",\"subDataTag\":100,\"subDataset\":\"\",\"taskId\":29,\"workflow\":\"\"}}`)
|
||||
// tag, res := mq.TaskRequestHandler(data)
|
||||
// fmt.Println("tag ==== >>>> ", tag)
|
||||
// fmt.Println("res===== >>>> ", string(res))
|
||||
// assert.Nil(t, res)
|
||||
//}
|
||||
Loading…
Reference in New Issue