Skip to content

Commit 94d79c6

Browse files
author
chenjie
committed
update
1 parent d5bd461 commit 94d79c6

File tree

15 files changed

+247
-2844
lines changed

15 files changed

+247
-2844
lines changed

‎codegen/tml/cmd/template_cmd.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ help() {
3030
}
3131
3232
pb() {
33-
protoc --go_out=../ ./api/*.proto
34-
protoc --go-rpc_out=../ ./api/*.proto
35-
protoc --go-web_out=../ ./api/*.proto
33+
protoc --go_out=paths=source_relative:. ./api/*.proto
34+
protoc --go-rpc_out=paths=source_relative:. ./api/*.proto
35+
protoc --go-web_out=paths=source_relative:. ./api/*.proto
3636
go mod tidy
3737
}
3838
@@ -245,9 +245,9 @@ if "%1" == "new" (
245245
)
246246
247247
:pb
248-
protoc --go_out=../ ./api/*.proto
249-
protoc --go-grpc_out=require_unimplemented_servers=false:../ ./api/*.proto
250-
protoc --go-gin_out=../ ./api/*.proto
248+
protoc --go_out=paths=source_relative:. ./api/*.proto
249+
protoc --go-rpc_out=paths=source_relative:. ./api/*.proto
250+
protoc --go-web_out=paths=source_relative:. ./api/*.proto
251251
go mod tidy
252252
goto :end
253253

‎codegen/tml/gomod/template_gomod.go

+6-7
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,13 @@ const text = `module {{.}}
1111
go 1.16
1212
1313
require (
14-
github.com/chenjie199234/Config v0.0.4
15-
github.com/chenjie199234/Corelib v0.0.18
16-
github.com/chenjie199234/Discovery v0.0.4
14+
github.com/chenjie199234/Config v0.0.5
15+
github.com/chenjie199234/Corelib v0.0.19
16+
github.com/chenjie199234/Discovery v0.0.5
1717
github.com/fsnotify/fsnotify v1.4.9
18-
github.com/go-sql-driver/mysql v1.5.0
19-
github.com/golang/protobuf v1.5.0
20-
github.com/segmentio/kafka-go v0.4.8
21-
go.mongodb.org/mongo-driver v1.5.0
18+
github.com/go-sql-driver/mysql v1.6.0
19+
github.com/segmentio/kafka-go v0.4.14
20+
go.mongodb.org/mongo-driver v1.5.1
2221
google.golang.org/protobuf v1.26.0
2322
)`
2423

‎go.mod

-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ module github.com/chenjie199234/Corelib
33
go 1.16
44

55
require (
6-
github.com/golang/protobuf v1.5.0
76
github.com/gomodule/redigo v1.8.4
87
github.com/julienschmidt/httprouter v1.3.0
98
google.golang.org/protobuf v1.26.0

‎go.sum

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
22
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
3-
github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
43
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
54
github.com/gomodule/redigo v1.8.4 h1:Z5JUg94HMTR1XpwBaSH4vq3+PNSIykBLxMdglbw10gg=
65
github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=

‎redis/bloom.go

+100-88
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,20 @@ import (
1717
type Bloom struct {
1818
p *Pool
1919
c *BloomConfig
20-
bitnum uint64
2120
expire int64
2221
}
22+
2323
type BloomConfig struct {
2424
// make sure don't use duplicate BloomName between different blooms
2525
BloomName string
2626
//Groupnum decide there are how many keys will be used in redis for this bloom
2727
//this is useful to balance all redis nodes' request
2828
//every special key will have a name like BloomName_[0,Groupnum)
2929
Groupnum uint64 //default 10
30-
//unit is byte
30+
//unit is bit
3131
//single hash list's capacity in every group
32-
//so this bloom will use memory: Capacity * hash_func_num(6) * Groupnum
33-
Capacity uint64 //default 1024*1024(1M)
32+
//so this bloom will use memory: Capacity / 8 * hash_func_num(6) * Groupnum
33+
Capacity uint64 //default 1024*1024*8(1M)
3434
Expire time.Duration //default 30day,min 1hour
3535
}
3636

@@ -45,7 +45,7 @@ func (c *BloomConfig) validate() error {
4545
c.Groupnum = 10
4646
}
4747
if c.Capacity == 0 {
48-
c.Capacity = 1024 * 1024
48+
c.Capacity = 1024 * 1024 * 8
4949
}
5050
if c.Expire == 0 {
5151
c.Expire = 30 * 24 * time.Hour
@@ -71,29 +71,28 @@ then
7171
end
7272
redis.call("SET",KEYS[1],ARGV[1],"EX",ARGV[2])`
7373

74-
var initlua = `local e,k1,k2,k3,k4,k5,k6="{"..KEYS[1].."}_e","{"..KEYS[1].."}_bkdr","{"..KEYS[1].."}_djb","{"..KEYS[1].."}_fnv","{"..KEYS[1].."}_dek","{"..KEYS[1].."}_rs","{"..KEYS[1].."}_sdbm"
75-
if(redis.call("EXISTS",e)==0)
74+
var initlua = `if(redis.call("EXISTS",KEYS[7])==0)
7675
then
77-
local r1=redis.call("SETBIT",k1,ARGV[1],1)
78-
local r2=redis.call("SETBIT",k2,ARGV[1],1)
79-
local r3=redis.call("SETBIT",k3,ARGV[1],1)
80-
local r4=redis.call("SETBIT",k4,ARGV[1],1)
81-
local r5=redis.call("SETBIT",k5,ARGV[1],1)
82-
local r6=redis.call("SETBIT",k6,ARGV[1],1)
83-
redis.call("SET",e,1,"EX",ARGV[2])
84-
redis.call("EXPIRE",k1,ARGV[2])
85-
redis.call("EXPIRE",k2,ARGV[2])
86-
redis.call("EXPIRE",k3,ARGV[2])
87-
redis.call("EXPIRE",k4,ARGV[2])
88-
redis.call("EXPIRE",k5,ARGV[2])
89-
redis.call("EXPIRE",k6,ARGV[2])
76+
local r1=redis.call("SETBIT",KEYS[1],ARGV[1],1)
77+
local r2=redis.call("SETBIT",KEYS[2],ARGV[1],1)
78+
local r3=redis.call("SETBIT",KEYS[3],ARGV[1],1)
79+
local r4=redis.call("SETBIT",KEYS[4],ARGV[1],1)
80+
local r5=redis.call("SETBIT",KEYS[5],ARGV[1],1)
81+
local r6=redis.call("SETBIT",KEYS[6],ARGV[1],1)
82+
redis.call("SET",KEYS[7],1,"EX",ARGV[2])
83+
redis.call("EXPIRE",KEYS[1],ARGV[2])
84+
redis.call("EXPIRE",KEYS[2],ARGV[2])
85+
redis.call("EXPIRE",KEYS[3],ARGV[2])
86+
redis.call("EXPIRE",KEYS[4],ARGV[2])
87+
redis.call("EXPIRE",KEYS[5],ARGV[2])
88+
redis.call("EXPIRE",KEYS[6],ARGV[2])
9089
end`
9190

91+
var ErrBloomConflict = errors.New("bloom conflict")
92+
var ErrBloomExpired = errors.New("bloom expired")
93+
9294
// NewRedisBitFilter -
9395
func (p *Pool) NewBloom(ctx context.Context, c *BloomConfig) (*Bloom, error) {
94-
if p.p == nil {
95-
return nil, errors.New("redis client not inited")
96-
}
9796
if e := c.validate(); e != nil {
9897
return nil, e
9998
}
@@ -114,20 +113,19 @@ func (p *Pool) NewBloom(ctx context.Context, c *BloomConfig) (*Bloom, error) {
114113
}
115114
}
116115
conn.Close()
117-
//if exist,use exist's config to replace this config
118-
c = &BloomConfig{}
119116
if exist {
120-
if e = json.Unmarshal([]byte(cstr), c); e != nil {
121-
return nil, errors.New("BloomName conflict")
117+
//if exist,use exist's config to replace this config
118+
c = &BloomConfig{}
119+
if e = json.Unmarshal(common.Str2byte(cstr), c); e != nil {
120+
return nil, ErrBloomConflict
122121
}
123122
if c.Groupnum == 0 || c.Capacity == 0 || c.Expire < time.Hour {
124-
return nil, errors.New("BloomName conflict")
123+
return nil, ErrBloomConflict
125124
}
126125
}
127126
instance := &Bloom{
128127
p: p,
129128
c: c,
130-
bitnum: c.Capacity * 8,
131129
expire: int64(c.Expire.Seconds()),
132130
}
133131
//init memory in redis
@@ -136,14 +134,20 @@ func (p *Pool) NewBloom(ctx context.Context, c *BloomConfig) (*Bloom, error) {
136134
tempindex := i
137135
go func() {
138136
key := c.BloomName + "_" + strconv.FormatUint(tempindex, 10)
139-
bit := c.Capacity * 8 //capacity's unit is byte,transform to bit
137+
keybkdr := "{" + key + "}_bkdr"
138+
keydjb := "{" + key + "}_djb"
139+
keyfnv := "{" + key + "}_fnv"
140+
keydek := "{" + key + "}_dek"
141+
keyrs := "{" + key + "}_rs"
142+
keysdbm := "{" + key + "}_sdbm"
143+
keyexist := "{" + key + "}_exist"
140144
conn, e := p.GetContext(ctx)
141145
if e != nil {
142146
ch <- e
143147
return
144148
}
145149
defer conn.Close()
146-
_, e = conn.DoContext(ctx, "EVAL", initlua, 1, key, bit, int64(c.Expire.Seconds()))
150+
_, e = conn.DoContext(ctx, "EVAL", initlua, 7, keybkdr, keydjb, keyfnv, keydek, keyrs, keysdbm, keyexist, c.Capacity, instance.expire)
147151
if e != nil && e != redis.ErrNil {
148152
ch <- e
149153
return
@@ -160,30 +164,29 @@ func (p *Pool) NewBloom(ctx context.Context, c *BloomConfig) (*Bloom, error) {
160164
if string(d) == cstr {
161165
return instance, nil
162166
}
163-
return nil, errors.New("BloomName conflict")
167+
return nil, ErrBloomConflict
164168
}
165169
return instance, nil
166170
}
167171

168-
const setlua = `local e,k1,k2,k3,k4,k5,k6="{"..KEYS[1].."}_e","{"..KEYS[1].."}_bkdr","{"..KEYS[1].."}_djb","{"..KEYS[1].."}_fnv","{"..KEYS[1].."}_dek","{"..KEYS[1].."}_rs","{"..KEYS[1].."}_sdbm"
169-
if(redis.call("EXISTS",e)==0)
172+
const setlua = `if(redis.call("EXISTS",KEYS[7])==0)
170173
then
171174
return -1
172175
end
173-
local r1=redis.call("SETBIT",k1,ARGV[1],1)
174-
local r2=redis.call("SETBIT",k2,ARGV[2],1)
175-
local r3=redis.call("SETBIT",k3,ARGV[3],1)
176-
local r4=redis.call("SETBIT",k4,ARGV[4],1)
177-
local r5=redis.call("SETBIT",k5,ARGV[5],1)
178-
local r6=redis.call("SETBIT",k6,ARGV[6],1)
179-
if(redis.call("EXISTS",e)==0)
176+
local r1=redis.call("SETBIT",KEYS[1],ARGV[1],1)
177+
local r2=redis.call("SETBIT",KEYS[2],ARGV[2],1)
178+
local r3=redis.call("SETBIT",KEYS[3],ARGV[3],1)
179+
local r4=redis.call("SETBIT",KEYS[4],ARGV[4],1)
180+
local r5=redis.call("SETBIT",KEYS[5],ARGV[5],1)
181+
local r6=redis.call("SETBIT",KEYS[6],ARGV[6],1)
182+
if(redis.call("EXISTS",KEYS[7])==0)
180183
then
181-
redis.call("DEL",k1)
182-
redis.call("DEL",k2)
183-
redis.call("DEL",k3)
184-
redis.call("DEL",k4)
185-
redis.call("DEL",k5)
186-
redis.call("DEL",k6)
184+
redis.call("DEL",KEYS[1])
185+
redis.call("DEL",kEYS[2])
186+
redis.call("DEL",KEYS[3])
187+
redis.call("DEL",KEYS[4])
188+
redis.call("DEL",KEYS[5])
189+
redis.call("DEL",KEYS[6])
187190
return -1
188191
end
189192
if(r1==0 or r2==0 or r3==0 or r4==0 or r5==0 or r6==0)
@@ -197,51 +200,57 @@ var hsetlua string
197200
// Set add key into the bloom
198201
//true,this key is not in this bloom and add success
199202
//false,this key maybe already in this bloom,can't 100% confirm
200-
func (b *Bloom) Set(ctx context.Context, key string) (bool, error) {
201-
filterkey := b.getbloomkey(key)
202-
bit1 := common.BkdrhashString(key, b.bitnum)
203-
bit2 := common.DjbhashString(key, b.bitnum)
204-
bit3 := common.FnvhashString(key, b.bitnum)
205-
bit4 := common.DekhashString(key, b.bitnum)
206-
bit5 := common.RshashString(key, b.bitnum)
207-
bit6 := common.SdbmhashString(key, b.bitnum)
203+
func (b *Bloom) Set(ctx context.Context, userkey string) (bool, error) {
204+
key := b.c.BloomName + "_" + strconv.FormatUint(common.BkdrhashString(userkey, uint64(b.c.Groupnum)), 10)
205+
keybkdr := "{" + key + "}_bkdr"
206+
keydjb := "{" + key + "}_djb"
207+
keyfnv := "{" + key + "}_fnv"
208+
keydek := "{" + key + "}_dek"
209+
keyrs := "{" + key + "}_rs"
210+
keysdbm := "{" + key + "}_sdbm"
211+
keyexist := "{" + key + "}_exist"
212+
bit1 := common.BkdrhashString(key, b.c.Capacity)
213+
bit2 := common.DjbhashString(key, b.c.Capacity)
214+
bit3 := common.FnvhashString(key, b.c.Capacity)
215+
bit4 := common.DekhashString(key, b.c.Capacity)
216+
bit5 := common.RshashString(key, b.c.Capacity)
217+
bit6 := common.SdbmhashString(key, b.c.Capacity)
208218
c, e := b.p.GetContext(ctx)
209219
if e != nil {
210220
return false, e
211221
}
212222
defer c.Close()
213-
r, e := redis.Int(c.DoContext(ctx, "EVALSHA", hsetlua, 1, filterkey, bit1, bit2, bit3, bit4, bit5, bit6, b.expire))
223+
r, e := redis.Int(c.DoContext(ctx, "EVALSHA", hsetlua, 7, keybkdr, keydjb, keyfnv, keydek, keyrs, keysdbm, keyexist, bit1, bit2, bit3, bit4, bit5, bit6))
214224
if e != nil && strings.Contains(e.Error(), "NOSCRIPT") {
215-
r, e = redis.Int(c.DoContext(ctx, "EVAL", setlua, 1, filterkey, bit1, bit2, bit3, bit4, bit5, bit6, b.expire))
225+
r, e = redis.Int(c.DoContext(ctx, "EVAL", setlua, 7, keybkdr, keydjb, keyfnv, keydek, keyrs, keysdbm, keyexist, bit1, bit2, bit3, bit4, bit5, bit6))
216226
}
217227
if e != nil {
218228
return false, e
219229
}
220230
if r == -1 {
221-
return false, errors.New("bloom expired")
231+
return false, ErrBloomExpired
222232
}
223233
return r == 0, nil
224234
}
225235

226-
var checklua = `local e,k1,k2,k3,k4,k5,k6="{"..KEYS[1].."}_e","{"..KEYS[1].."}_bkdr","{"..KEYS[1].."}_djb","{"..KEYS[1].."}_fnv","{"..KEYS[1].."}_dek","{"..KEYS[1].."}_rs","{"..KEYS[1].."}_sdbm"
227-
if(redis.call("EXISTS",e)==0)
236+
var checklua = `if(redis.call("EXISTS",KEYS[7])==0)
228237
then
229238
return -1
230239
end
231-
local r1=redis.call("GETBIT",k1,ARGV[1])
232-
local r2=redis.call("GETBIT",k2,ARGV[2])
233-
local r3=redis.call("GETBIT",k3,ARGV[3])
234-
local r4=redis.call("GETBIT",k4,ARGV[4])
235-
local r5=redis.call("GETBIT",k5,ARGV[5])
236-
local r6=redis.call("GETBIT",k6,ARGV[6])
237-
if(redis.call("EXISTS",e)==0)
240+
local r1=redis.call("GETBIT",KEYS[1],ARGV[1])
241+
local r2=redis.call("GETBIT",KEYS[2],ARGV[2])
242+
local r3=redis.call("GETBIT",KEYS[3],ARGV[3])
243+
local r4=redis.call("GETBIT",KEYS[4],ARGV[4])
244+
local r5=redis.call("GETBIT",KEYS[5],ARGV[5])
245+
local r6=redis.call("GETBIT",KEYS[6],ARGV[6])
246+
if(redis.call("EXISTS",KEYS[7])==0)
238247
then
239-
redis.call("DEL",k1)
240-
redis.call("DEL",k2)
241-
redis.call("DEL",k3)
242-
redis.call("DEL",k4)
243-
redis.call("DEL",k5)
244-
redis.call("DEL",k6)
248+
redis.call("DEL",KEYS[1])
249+
redis.call("DEL",KEYS[2])
250+
redis.call("DEL",KEYS[3])
251+
redis.call("DEL",KEYS[4])
252+
redis.call("DEL",KEYS[5])
253+
redis.call("DEL",KEYS[6])
245254
return -1
246255
end
247256
if(r1==0 or r2==0 or r3==0 or r4==0 or r5==0 or r6==0)
@@ -255,32 +264,35 @@ var hchecklua string
255264
// Check -
256265
//true,this key 100% not in this bloom
257266
//false,this key maybe in this bloom,can't 100% confirm
258-
func (b *Bloom) Check(ctx context.Context, key string) (bool, error) {
259-
filterkey := b.getbloomkey(key)
260-
bit1 := common.BkdrhashString(key, b.bitnum)
261-
bit2 := common.DjbhashString(key, b.bitnum)
262-
bit3 := common.FnvhashString(key, b.bitnum)
263-
bit4 := common.DekhashString(key, b.bitnum)
264-
bit5 := common.RshashString(key, b.bitnum)
265-
bit6 := common.SdbmhashString(key, b.bitnum)
267+
func (b *Bloom) Check(ctx context.Context, userkey string) (bool, error) {
268+
key := b.c.BloomName + "_" + strconv.FormatUint(common.BkdrhashString(userkey, uint64(b.c.Groupnum)), 10)
269+
keybkdr := "{" + key + "}_bkdr"
270+
keydjb := "{" + key + "}_djb"
271+
keyfnv := "{" + key + "}_fnv"
272+
keydek := "{" + key + "}_dek"
273+
keyrs := "{" + key + "}_rs"
274+
keysdbm := "{" + key + "}_sdbm"
275+
keyexist := "{" + key + "}_exist"
276+
bit1 := common.BkdrhashString(key, b.c.Capacity)
277+
bit2 := common.DjbhashString(key, b.c.Capacity)
278+
bit3 := common.FnvhashString(key, b.c.Capacity)
279+
bit4 := common.DekhashString(key, b.c.Capacity)
280+
bit5 := common.RshashString(key, b.c.Capacity)
281+
bit6 := common.SdbmhashString(key, b.c.Capacity)
266282
c, e := b.p.GetContext(ctx)
267283
if e != nil {
268284
return false, e
269285
}
270286
defer c.Close()
271-
r, e := redis.Int(c.DoContext(ctx, "EVALSHA", hchecklua, 1, filterkey, bit1, bit2, bit3, bit4, bit5, bit6, b.expire))
287+
r, e := redis.Int(c.DoContext(ctx, "EVALSHA", hchecklua, 7, keybkdr, keydjb, keyfnv, keydek, keyrs, keysdbm, keyexist, bit1, bit2, bit3, bit4, bit5, bit6))
272288
if e != nil && strings.HasPrefix(e.Error(), "NOSCRIPT") {
273-
r, e = redis.Int(c.DoContext(ctx, "EVAL", checklua, 1, filterkey, bit1, bit2, bit3, bit4, bit5, bit6, b.expire))
289+
r, e = redis.Int(c.DoContext(ctx, "EVAL", checklua, 7, keybkdr, keydjb, keyfnv, keydek, keyrs, keysdbm, keyexist, bit1, bit2, bit3, bit4, bit5, bit6))
274290
}
275291
if e != nil {
276292
return false, e
277293
}
278294
if r == -1 {
279-
return false, errors.New("bloom expired")
295+
return false, ErrBloomExpired
280296
}
281297
return r == 0, nil
282298
}
283-
284-
func (b *Bloom) getbloomkey(userkey string) string {
285-
return b.c.BloomName + "_" + strconv.FormatUint(common.BkdrhashString(userkey, uint64(b.c.Groupnum)), 10)
286-
}

0 commit comments

Comments
 (0)