Skip to content

Commit 1a3e75c

Browse files
author
gfelber
committed
Flagid Scan and Scraper can now be enabled/disabled (use start.sh)
Flagid scanner now uses Aho-Corasick added script for testing (test.sh) seperated flagid configs
1 parent 56d5683 commit 1a3e75c

File tree

11 files changed

+197
-59
lines changed

11 files changed

+197
-59
lines changed

‎.env.example

+14-5
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,23 @@ TRAFFIC_DIR_DOCKER="/traffic"
1111
TICK_START="2018-06-27T13:00+02:00"
1212
# Tick length in ms
1313
TICK_LENGTH=180000
14+
VM_IP="10.10.3.1"
15+
16+
#PCAP_OVER_IP="host.docker.internal:1337"
17+
#For multiple PCAP_OVER_IP you can comma separate
18+
#PCAP_OVER_IP="host.docker.internal:1337,otherhost.com:5050"
19+
20+
##############################
21+
# FLAGID CONFIGS
22+
##############################
23+
24+
# enable flagid scrapping
25+
FLAGID_SCRAPE=1
26+
# enable flagid scanning
27+
FLAGID_SCAN=1
1428
# Flag Lifetime in Ticks (-1 for no check, pls don't use outside testing)
1529
FLAG_LIFETIME=-1
1630
# Flagid endpoint currently Testendpoint in docker compose
1731
FLAGID_ENDPOINT="http://flagidendpoint:8000/flagids.json"
1832
# VM IP (inside gamenet)
19-
VM_IP="10.10.3.1"
2033
TEAM_ID="10.10.3.1"
21-
22-
#PCAP_OVER_IP="host.docker.internal:1337"
23-
#For multiple PCAP_OVER_IP you can comma separate
24-
#PCAP_OVER_IP="host.docker.internal:1337,otherhost.com:5050"

‎docker-compose-flagid.yml

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
version: "3.2"
2+
services:
3+
mongo:
4+
image: mongo:5
5+
networks:
6+
- internal
7+
restart: always
8+
ports:
9+
- "27017:27017"
10+
11+
frontend:
12+
build:
13+
context: frontend
14+
dockerfile: Dockerfile-frontend
15+
image: tulip-frontend:latest
16+
restart: unless-stopped
17+
ports:
18+
- "3000:3000"
19+
depends_on:
20+
- mongo
21+
- api
22+
networks:
23+
- internal
24+
environment:
25+
API_SERVER_ENDPOINT: http://api:5000/
26+
27+
api:
28+
build:
29+
context: services/api
30+
dockerfile: Dockerfile-api
31+
image: tulip-api:latest
32+
restart: unless-stopped
33+
depends_on:
34+
- mongo
35+
networks:
36+
- internal
37+
volumes:
38+
- ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
39+
environment:
40+
TULIP_MONGO: ${TULIP_MONGO}
41+
TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER}
42+
FLAG_REGEX: ${FLAG_REGEX}
43+
TICK_START: ${TICK_START}
44+
TICK_LENGTH: ${TICK_LENGTH}
45+
VM_IP: ${VM_IP}
46+
47+
flagids:
48+
restart: unless-stopped
49+
build:
50+
context: services/flagids
51+
image: tulip-flagids:latest
52+
depends_on:
53+
- mongo
54+
networks:
55+
- internal
56+
environment:
57+
TULIP_MONGO: ${TULIP_MONGO}
58+
TICK_START: ${TICK_START}
59+
TICK_LENGTH: ${TICK_LENGTH}
60+
TEAM_ID: ${TEAM_ID}
61+
FLAGID_ENDPOINT: ${FLAGID_ENDPOINT}
62+
63+
assembler:
64+
build:
65+
context: services/go-importer
66+
dockerfile: Dockerfile-assembler
67+
image: tulip-assembler:latest
68+
restart: unless-stopped
69+
depends_on:
70+
- mongo
71+
networks:
72+
- internal
73+
volumes:
74+
- ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
75+
command: "./assembler -dir ${TRAFFIC_DIR_DOCKER}"
76+
environment:
77+
TULIP_MONGO: ${TULIP_MONGO}
78+
FLAG_REGEX: ${FLAG_REGEX}
79+
TICK_LENGTH: ${TICK_LENGTH}
80+
FLAGID_SCAN: ${FLAGID_SCAN}
81+
FLAG_LIFETIME: ${FLAG_LIFETIME}
82+
PCAP_OVER_IP: ${PCAP_OVER_IP}
83+
extra_hosts:
84+
- "host.docker.internal:host-gateway"
85+
86+
87+
enricher:
88+
build:
89+
context: services/go-importer
90+
dockerfile: Dockerfile-enricher
91+
image: tulip-enricher:latest
92+
restart: unless-stopped
93+
depends_on:
94+
- mongo
95+
networks:
96+
- internal
97+
volumes:
98+
- ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
99+
command: "./enricher -eve ${TRAFFIC_DIR_DOCKER}/eve.json"
100+
environment:
101+
TULIP_MONGO: ${TULIP_MONGO}
102+
103+
networks:
104+
internal:

‎docker-compose-test.yml

+3
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ services:
6464
image: tulip-flagids:latest
6565
depends_on:
6666
- mongo
67+
- flagidendpoint
6768
networks:
6869
- internal
6970
environment:
@@ -87,9 +88,11 @@ services:
8788
- ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro
8889
command: "./assembler -dir ${TRAFFIC_DIR_DOCKER}"
8990
environment:
91+
DELAY: 5
9092
TULIP_MONGO: ${TULIP_MONGO}
9193
FLAG_REGEX: ${FLAG_REGEX}
9294
TICK_LENGTH: ${TICK_LENGTH}
95+
FLAGID_SCAN: ${FLAGID_SCAN}
9396
FLAG_LIFETIME: ${FLAG_LIFETIME}
9497
PCAP_OVER_IP: ${PCAP_OVER_IP}
9598
extra_hosts:

‎docker-compose.yml

+1-16
Original file line numberDiff line numberDiff line change
@@ -44,22 +44,6 @@ services:
4444
TICK_LENGTH: ${TICK_LENGTH}
4545
VM_IP: ${VM_IP}
4646

47-
flagids:
48-
restart: unless-stopped
49-
build:
50-
context: services/flagids
51-
image: tulip-flagids:latest
52-
depends_on:
53-
- mongo
54-
networks:
55-
- internal
56-
environment:
57-
TULIP_MONGO: ${TULIP_MONGO}
58-
TICK_START: ${TICK_START}
59-
TICK_LENGTH: ${TICK_LENGTH}
60-
TEAM_ID: ${TEAM_ID}
61-
FLAGID_ENDPOINT: ${FLAGID_ENDPOINT}
62-
6347
assembler:
6448
build:
6549
context: services/go-importer
@@ -77,6 +61,7 @@ services:
7761
TULIP_MONGO: ${TULIP_MONGO}
7862
FLAG_REGEX: ${FLAG_REGEX}
7963
TICK_LENGTH: ${TICK_LENGTH}
64+
FLAGID_SCAN: ${FLAGID_SCAN}
8065
FLAG_LIFETIME: ${FLAG_LIFETIME}
8166
PCAP_OVER_IP: ${PCAP_OVER_IP}
8267
extra_hosts:

‎services/go-importer/cmd/assembler/main.go

+24-14
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,15 @@ package main
33
import (
44
"go-importer/internal/pkg/db"
55

6-
"fmt"
7-
"net"
8-
"strconv"
9-
106
"flag"
7+
"fmt"
118
"io/ioutil"
129
"log"
10+
"net"
1311
"os"
1412
"os/signal"
1513
"path/filepath"
14+
"strconv"
1615
"strings"
1716
"sync"
1817
"time"
@@ -32,8 +31,6 @@ var checksum = false
3231
var nohttp = true
3332

3433
var snaplen = 65536
35-
var ticklength = -1
36-
var flaglifetime = -1
3734
var tstype = ""
3835
var promisc = true
3936

@@ -44,6 +41,11 @@ var pcap_over_ip = flag.String("pcap-over-ip", "", "PCAP-over-IP host + port (e.
4441
var bpf = flag.String("bpf", "", "BPF filter")
4542
var nonstrict = flag.Bool("nonstrict", false, "Do not check strict TCP / FSM flags")
4643
var experimental = flag.Bool("experimental", false, "Enable experimental features.")
44+
45+
var flagid = flag.Bool("flagid", false, "Check for flagids in traffic (must be present in mong)")
46+
var ticklength = *flag.Int("tick length", -1, "the length (in seconds) of a tick")
47+
var flaglifetime = *flag.Int("flag lifetime", -1, "the lifetime of a flag in ticks")
48+
4749
var flushAfter = flag.String("flush-after", "30s", `(TCP) Connections which have buffered packets (they've gotten packets out of order and
4850
are waiting for old packets to fill the gaps) can be flushed after they're this old
4951
(their oldest gap is skipped). This is particularly useful for pcap-over-ip captures.
@@ -74,14 +76,15 @@ func reassemblyCallback(entry db.FlowEntry) {
7476
ApplyFlagTags(&entry, flag_regex)
7577
}
7678

77-
//Apply flagid
78-
flagids, err := g_db.GetFlagids(flaglifetime)
79-
if err != nil {
80-
log.Fatal(err)
79+
//Apply flagid in / out
80+
if *flagid {
81+
flagids, err := g_db.GetFlagids(flaglifetime)
82+
if err != nil {
83+
log.Fatal(err)
84+
}
85+
ApplyFlagids(&entry, flagids)
8186
}
8287

83-
ApplyFlagids(&entry, flagids)
84-
8588
// Finally, insert the new entry
8689
g_db.InsertFlow(entry)
8790
}
@@ -159,7 +162,7 @@ func main() {
159162

160163
// get TICK_LENGTH
161164
strticklength := os.Getenv("TICK_LENGTH")
162-
if strticklength != "" {
165+
if ticklength == -1 && strticklength != "" {
163166
zwi, err := strconv.ParseInt(strticklength, 10, 64)
164167
if err != nil {
165168
log.Println("Error: ", err)
@@ -170,7 +173,7 @@ func main() {
170173

171174
// get Flag_LIFETIME
172175
strflaglifetime := os.Getenv("FLAG_LIFETIME")
173-
if strticklength != "" {
176+
if flaglifetime == -1 && strticklength != "" {
174177
zwi, err := strconv.Atoi(strflaglifetime)
175178
if err != nil {
176179
log.Println("Error: ", err)
@@ -201,6 +204,13 @@ func main() {
201204
*pcap_over_ip = os.Getenv("PCAP_OVER_IP")
202205
}
203206

207+
// if flagid scans should be done
208+
if !*flagid {
209+
flagid_val := os.Getenv("FLAGID_SCAN")
210+
*flagid = flagid_val != "" && flagid_val != "0" && !strings.EqualFold(flagid_val, "false")
211+
212+
}
213+
204214
if *bpf == "" {
205215
*bpf = os.Getenv("BPF")
206216
}

‎services/go-importer/cmd/assembler/tags.go

+32-20
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@ package main
22

33
import (
44
"go-importer/internal/pkg/db"
5+
56
"log"
67
"regexp"
7-
"strings"
8+
9+
"github.com/cloudflare/ahocorasick"
810
)
911

1012
var flagRegex *regexp.Regexp
@@ -71,31 +73,41 @@ func ApplyFlagTags(flow *db.FlowEntry, reg *string) {
7173

7274
// Apply flagids to the entire flow.
7375
// This assumes the `Data` part of the flowItem is already pre-processed, s.t.
74-
func ApplyFlagids(flow *db.FlowEntry, flagids []db.Flagid) {
76+
func ApplyFlagids(flow *db.FlowEntry, flagidsDb []db.Flagid) {
77+
78+
var flagids []string
79+
var matches = make(map[int]int)
80+
81+
for _, flagid := range flagidsDb {
82+
flagids = append(flagids, flagid.ID)
83+
}
7584

85+
matcher := ahocorasick.NewStringMatcher(flagids)
7686
for idx := 0; idx < len(flow.Flow); idx++ {
7787
flowItem := &flow.Flow[idx]
78-
data := flowItem.Data
79-
for _, flagid := range flagids {
80-
flagidstr := flagid.ID
81-
if strings.Contains(data, flagidstr) {
82-
var tag string
83-
if flowItem.From == "c" {
84-
tag = "flagid-in"
85-
} else {
86-
87-
tag = "flagid-out"
88-
}
88+
found := matcher.Match([]byte(flowItem.Data))
8989

90-
if !contains(flow.Flagids, flagidstr) {
91-
flow.Flagids = append(flow.Flagids, flagidstr)
92-
}
90+
if len(found) > 0 {
91+
var tag string
9392

94-
// Add the tag if it doesn't already exist
95-
if !contains(flow.Tags, tag) {
96-
flow.Tags = append(flow.Tags, tag)
97-
}
93+
if flowItem.From == "c" {
94+
tag = "flagid-in"
95+
} else {
96+
tag = "flagid-out"
97+
}
98+
99+
// Add the tag if it doesn't already exist
100+
if !contains(flow.Tags, tag) {
101+
flow.Tags = append(flow.Tags, tag)
102+
}
103+
104+
for _, match := range found {
105+
matches[match] = 1
98106
}
99107
}
100108
}
109+
110+
for match, _ := range matches {
111+
flow.Flagids = append(flow.Flagids, flagids[match])
112+
}
101113
}

‎services/go-importer/go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,5 @@ require (
88
github.com/google/gopacket v1.1.19
99
github.com/tidwall/gjson v1.14.1
1010
go.mongodb.org/mongo-driver v1.9.1
11+
github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184
1112
)

‎services/go-importer/go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
22
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
3+
github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184 h1:8yL+85JpbwrIc6m+7N1iYrjn/22z68jwrTIBOJHNe4k=
4+
github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184/go.mod h1:tGWUZLZp9ajsxUOnHmFFLnqnlKXsCn6GReG4jAD59H0=
35
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
46
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
57
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

‎services/go-importer/internal/pkg/db/db.go

+3-4
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,6 @@ func (db Database) ConfigureDatabase() {
7676
db.InsertTag("flagid-out")
7777
db.InsertTag("tcp")
7878
db.InsertTag("udp")
79-
db.InsertTag("flagid")
8079
db.ConfigureIndexes()
8180
}
8281

@@ -183,7 +182,7 @@ func (db Database) InsertFlow(flow FlowEntry) {
183182

184183
type PcapFile struct {
185184
FileName string `bson:"file_name"`
186-
Position int64 `bson:"position"`
185+
Position int64 `bson:"position"`
187186
}
188187

189188
// Insert a new pcap uri, returns true if the pcap was not present yet,
@@ -192,9 +191,9 @@ func (db Database) InsertPcap(uri string, position int64) bool {
192191
files := db.client.Database("pcap").Collection("filesImported")
193192
exists, _ := db.GetPcap(uri)
194193
if !exists {
195-
files.InsertOne(context.TODO(), bson.M{"file_name": uri,"position": position})
194+
files.InsertOne(context.TODO(), bson.M{"file_name": uri, "position": position})
196195
} else {
197-
files.UpdateOne(context.TODO(), bson.M{"file_name": uri}, bson.M{"$set":bson.M{"position": position}})
196+
files.UpdateOne(context.TODO(), bson.M{"file_name": uri}, bson.M{"$set": bson.M{"position": position}})
198197
}
199198
return !exists
200199
}

0 commit comments

Comments
 (0)