Skip to content

Commit f5c86e4

Browse files
nbhgytzhengnbhgytzheng
authored
Merge pull request #183 from nttcom/bug-#175
#175 ごく稀にログが欠損する可能性を排除できない問題を修正
2 parents 19a9bc0 + cfe4216 commit f5c86e4

File tree

28 files changed

+743
-32
lines changed

28 files changed

+743
-32
lines changed

‎osect_sensor/Infrastructure/edge_cron/work/ot_tools/bro.sh

+24-23
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
#!/bin/bash
22

3-
merge_log () {
4-
cat $(find /usr/local/zeek/logs -name ${1}) > $2
3+
merge_and_remove_log () {
4+
files=$(find /usr/local/zeek/logs -name "$1")
5+
cat ${files} > $2
56
sed -i '/^#/d' $2
67
sed -i '1i #\n#\n#\n#\n#\n#\n#\n#' $2
78
sed -i '$a #close' $2
9+
rm ${files}
810
}
911

1012
reformat_log () {
@@ -14,40 +16,39 @@ reformat_log () {
1416

1517
cd $1/$2
1618
# conn_long.logと重複するためconn.logに出力されるduration>60を除外
17-
awk '$9<60{print}' $(find /usr/local/zeek/logs -name "conn.*.log") > "/usr/local/zeek/logs/conn_replace.log"
18-
rm $(find /usr/local/zeek/logs -name "conn.*.log")
19+
tmp_files=$(find /usr/local/zeek/logs -name "conn.*.log")
20+
awk '$9<60{print}' ${tmp_files} > "/usr/local/zeek/logs/conn_replace.log"
21+
rm ${tmp_files}
1922
# conn.logとconn_long.logの両方を回収
20-
merge_log "conn*.log" "conn.log"
21-
merge_log "arp.*.log" "arp.log"
22-
merge_log "ns.*.log" "ns.log"
23-
merge_log "dns.*.log" "dns.log"
24-
merge_log "http.*.log" "http.log"
25-
merge_log "cifs.*.log" "mswin-browser.log"
23+
merge_and_remove_log "conn*.log" "conn.log"
24+
merge_and_remove_log "arp.*.log" "arp.log"
25+
merge_and_remove_log "ns.*.log" "ns.log"
26+
merge_and_remove_log "dns.*.log" "dns.log"
27+
merge_and_remove_log "http.*.log" "http.log"
28+
merge_and_remove_log "cifs.*.log" "mswin-browser.log"
2629
reformat_log "mswin-browser.log"
27-
merge_log "mydhcp.*.log" "dhcp2.log"
30+
merge_and_remove_log "mydhcp.*.log" "dhcp2.log"
2831
reformat_log "dhcp2.log"
29-
merge_log "dhcpv6.*.log" "dhcpv6.log"
32+
merge_and_remove_log "dhcpv6.*.log" "dhcpv6.log"
3033
reformat_log "dhcpv6.log"
31-
merge_log "nbns.*.log" "netbios-ns.log"
34+
merge_and_remove_log "nbns.*.log" "netbios-ns.log"
3235
reformat_log "netbios-ns.log"
33-
merge_log "ssdp.*.log" "ssdp.log"
36+
merge_and_remove_log "ssdp.*.log" "ssdp.log"
3437
reformat_log "ssdp.log"
3538
# OTプロトコル: CC-Link
36-
merge_log "cclink-ief-basic.*.log" "cclink-ief-basic.log"
37-
merge_log "cclink-ie.*.log" "cclink-ie.log"
38-
merge_log "cclink-ie-tsn.*.log" "cclink-ie-tsn.log"
39-
merge_log "cclink-ie-tsn-slmp.*.log" "cclink-ie-tsn-slmp.log"
40-
merge_log "cclink-ie-tsn-ptp.*.log" "cclink-ie-tsn-ptp.log"
39+
merge_and_remove_log "cclink-ief-basic.*.log" "cclink-ief-basic.log"
40+
merge_and_remove_log "cclink-ie.*.log" "cclink-ie.log"
41+
merge_and_remove_log "cclink-ie-tsn.*.log" "cclink-ie-tsn.log"
42+
merge_and_remove_log "cclink-ie-tsn-slmp.*.log" "cclink-ie-tsn-slmp.log"
43+
merge_and_remove_log "cclink-ie-tsn-ptp.*.log" "cclink-ie-tsn-ptp.log"
4144

4245
if [ $4 = "True" ]; then
4346
# tsharkでの出力と同じにするため
44-
merge_log "bacnet_service.*.log" "bacnet_service.log"
47+
merge_and_remove_log "bacnet_service.*.log" "bacnet_service.log"
4548
sed -i '/^#/d' bacnet_service.log
4649
sed -i '1i #' bacnet_service.log
4750
fi
4851

4952
if [ $5 = "True" ]; then
50-
merge_log "modbus_detailed.*.log" "modbus_detailed.log"
53+
merge_and_remove_log "modbus_detailed.*.log" "modbus_detailed.log"
5154
fi
52-
53-
find /usr/local/zeek/logs -name "*.log" -print0 | xargs -0 rm

‎osect_sensor/Infrastructure/edge_cron/work/ot_tools/p0f.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22

33
cd $1/$2
4-
cat /var/log/p0f-k.log.* > p0f-k.log
4+
tmp_files=$(ls /var/log/p0f-k.log.*)
5+
cat ${tmp_files} > p0f-k.log
56
chmod 644 p0f-k.log
6-
rm /var/log/p0f-k.log.*
7-
7+
rm ${tmp_files}

‎osect_sensor/Infrastructure/edge_cron/work/ot_tools/suricata.sh

+3-2
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22

33
# suricata -c $1 -r $2$3 -l $4
44

5-
cat /var/log/suricata/eve?*.json > $4/eve.json
6-
rm /var/log/suricata/eve?*.json
5+
tmp_files=$(ls /var/log/suricata/eve?*.json)
6+
cat ${tmp_files} > $4/eve.json
7+
rm ${tmp_files}
78

89
# rm $4/fast.log
910
# rm $4/stats.log

‎osect_sensor/Infrastructure/edge_cron/work/ot_tools/yaf.sh

+5-4
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
#!/bin/bash
22

3-
merge_log () {
4-
cat $1 > $2
3+
merge_and_remove_log () {
4+
files=$(ls $1)
5+
cat ${files} > $2
56
sed -i '/^#/d' $2
67
sed -i '1i #ts start-time end-time duration rtt proto sip sp dip dp srcMacAddress destMacAddress iflags uflags riflags ruflags isn risn tag rtag pktoct rpkt roct end-reason' $2
8+
rm ${files}
79
}
810

911
cd $1/$2 || exit
@@ -15,5 +17,4 @@ for flowfile in $flow; do
1517
rm "$flowfile" flow.csv
1618
done
1719

18-
merge_log "/var/log/yaf/flow*.log" "yaf_flow.log"
19-
rm /var/log/yaf/flow*.log
20+
merge_and_remove_log "/var/log/yaf/flow*.log" "yaf_flow.log"

‎osect_sensor/docker-compose.sh_test.yml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
version: '3'
2+
services:
3+
sh_test_env:
4+
container_name: osect_sensor_sh_test_env
5+
image: cron:revxxx
6+
build: "./Infrastructure/edge_cron/"
7+
volumes:
8+
- ./Application/edge_cron:/opt/edge_cron
9+
- ./sh_tests:/home/work/sh_tests
10+
- ./Infrastructure/edge_cron/work/ot_tools/:/opt/ot_tools/:ro
11+
environment:
12+
- DEBUG=False
13+
command: bash -c 'if [[ "$DEBUG" == "True" ]]; then sleep infinity; else bash /home/work/sh_tests/test_all.sh; fi'

‎osect_sensor/run_sh_test.sh

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
compose_file=docker-compose.sh_test.yml
4+
container_name=$(cat ${compose_file} | grep -E '^\s+container_name:\s*.+' | sed -r 's/^\s+container_name:\s*//g')
5+
docker compose -f ${compose_file} down
6+
docker compose -f ${compose_file} up
7+
exit $(docker inspect --format='{{.State.ExitCode}}' ${container_name})

‎osect_sensor/sh_tests/README.md

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# sh_tests/ot_tools
2+
`Infrastructure/edge_cron/work/ot_tools`配下のスクリプトをテストするためのコードを格納するディレクトリ。
3+
4+
本ディレクトリ内のファイルは、全てDocker container内で実行する前提。
5+
6+
## ディレクトリの基本構成
7+
```
8+
OsecT/osect_sensor/sh_tests$ tree .
9+
.
10+
├── common.sh # テスト用の関数や変数を記述
11+
├── ot_tools
12+
│ ├── bro
13+
│ │ ├── bro_unfixed.sh # 修正前のコード
14+
│ │ ├── data # テスト用データの生成、保管先ディレクトリ
15+
│ │ │ ├── expected
16+
│ │ │ └── init
17+
│ │ └── test.sh　　　　　　　 # 実際のテストコードを記述したスクリプト
18+
│ ├── p0f
19+
│ │ ├── data
20+
│ │ │ ├── expected
21+
│ │ │ └── init
22+
│ │ ├── p0f_unfixed.sh
23+
│ │ └── test.sh
24+
│ ├── suricata
25+
│ │ ├── data
26+
│ │ │ ├── expected
27+
│ │ │ └── init
28+
│ │ ├── suricata_unfixed.sh
29+
│ │ └── test.sh
30+
│ └── yaf
31+
│ ├── data
32+
│ │ ├── expected
33+
│ │ ├── gen_pcap.sh # テスト用のデータ(pcap)を生成するスクリプト
34+
│ │ └── init
35+
│ │ └── udplite.pcap # テスト用のオリジナルデータ(IPアドレスを書き換えて利用)
36+
│ ├── test.sh
37+
│ └── yaf_unfixed.sh
38+
└── test_all.sh # 各テストコードを呼び出すスクリプト
39+
```

‎osect_sensor/sh_tests/common.sh

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#!/bin/bash
2+
3+
if [ ! -e /.dockerenv ]; then
4+
echo '[ERROR] This script must be run in a Docker container'
5+
exit 1
6+
fi
7+
8+
DATA_DIR="data/"
9+
INIT_DIR="${DATA_DIR}init/"
10+
EXPECTED_DIR="${DATA_DIR}expected/"
11+
WORK_DIR="work/"
12+
ACTUAL_DIR="${WORK_DIR}actual/"
13+
test_init() {
14+
local name=$1
15+
echo ""
16+
echo "[TEST_NAME] ${name}"
17+
rm -rf ${WORK_DIR}
18+
mkdir -p ${WORK_DIR}
19+
cp -rp ${INIT_DIR} ${ACTUAL_DIR}
20+
}
21+
22+
test_line_num() {
23+
local cmd="test \$(cat ${1} | wc -l) -eq ${2}"
24+
echo "[TEST_CMD] ${cmd}"
25+
bash -c "${cmd}"
26+
sts="$?"
27+
if [ $sts -eq 0 ]; then
28+
echo "<<< OK >>> ${FUNCNAME[0]}"
29+
echo "----------"
30+
else
31+
echo "!!! NG !!! ${FUNCNAME[0]}"
32+
echo "actual: $(cat ${1} | wc -l)"
33+
exit 1
34+
fi
35+
}
36+
37+
test_fullmatch_dir() {
38+
local cmd="diff -r ${ACTUAL_DIR} ${EXPECTED_DIR} $*"
39+
echo "[TEST_CMD] ${cmd}"
40+
bash -c "${cmd}"
41+
sts="$?"
42+
if [ $sts -eq 0 ]; then
43+
echo "<<< OK >>> ${FUNCNAME[0]}"
44+
echo "----------"
45+
else
46+
echo "!!! NG !!! ${FUNCNAME[0]}"
47+
exit 1
48+
fi
49+
}
50+
51+
test_fullmatch_file() {
52+
local cmd="diff -r $*"
53+
echo "[TEST_CMD] ${cmd}"
54+
bash -c "${cmd}"
55+
sts="$?"
56+
if [ $sts -eq 0 ]; then
57+
echo "<<< OK >>> ${FUNCNAME[0]}"
58+
echo "----------"
59+
else
60+
echo "!!! NG !!! ${FUNCNAME[0]}"
61+
exit 1
62+
fi
63+
}
64+
65+
test_unmatch_file() {
66+
local cmd="diff $* > /dev/null"
67+
echo "[TEST_CMD] ${cmd}"
68+
bash -c "${cmd}"
69+
sts="$?"
70+
if [ $sts -ne 0 ]; then
71+
echo "<<< OK >>> ${FUNCNAME[0]}"
72+
echo "----------"
73+
return 0
74+
fi
75+
echo "!!! NG !!! ${FUNCNAME[0]}"
76+
exit 1
77+
}
78+
79+
test_no_empty_files() {
80+
local cmd="ls $* | xargs -I{} bash -c 'cat {} | grep -cE ^.+$'"
81+
echo "[TEST_CMD] ${cmd}"
82+
for c in $(bash -c "${cmd}"); do
83+
if [ $c -eq 0 ]; then
84+
echo "<<< NG >>> ${FUNCNAME[0]}"
85+
ls $* | xargs -I{} bash -c 'if [ $(cat {} | grep -cE ^.+$) -eq 0 ]; then echo "> empty file: {}"; fi'
86+
exit 1
87+
fi
88+
done
89+
echo "<<< OK >>> ${FUNCNAME[0]}"
90+
echo "----------"
91+
return 0
92+
}
93+
94+
test_files_contain_digits() {
95+
local cmd="ls $* | xargs -I{} bash -c 'cat {} | grep -cE ^[0-9]+$'"
96+
echo "[TEST_CMD] ${cmd}"
97+
for c in $(bash -c "${cmd}"); do
98+
if [ $c -eq 0 ]; then
99+
echo "<<< NG >>> ${FUNCNAME[0]}"
100+
ls $* | xargs -I{} bash -c 'if [ $(cat {} | grep -cE ^[0-9]+$) -eq 0 ]; then echo "> empty file: {}"; fi'
101+
exit 1
102+
fi
103+
done
104+
echo "<<< OK >>> ${FUNCNAME[0]}"
105+
echo "----------"
106+
return 0
107+
}

‎osect_sensor/sh_tests/ot_tools/.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
work
2+
*.pcap
3+
!yaf/data/init/udplite.pcap

‎osect_sensor/sh_tests/ot_tools/bro/bro_unfixed.sh

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
3+
# ログファイルの欠損が生じる可能性があるスクリプト
4+
# Commit ID: 71baeab531d8170456bb20920caa62be51be8be3
5+
6+
merge_log () {
7+
# NOTE: テスト環境の都合上${1}をダブルクォートで囲んだ(Commit ID: 71baeab531d8170456bb20920caa62be51be8be3 との差分)
8+
cat $(find /usr/local/zeek/logs -name "${1}") > $2
9+
sed -i '/^#/d' $2
10+
sed -i '1i #\n#\n#\n#\n#\n#\n#\n#' $2
11+
sed -i '$a #close' $2
12+
}
13+
14+
reformat_log () {
15+
sed -i '/^#/d' $1
16+
sed -i '1i #' $1
17+
}
18+
19+
cd $1/$2
20+
# conn_long.logと重複するためconn.logに出力されるduration>60を除外
21+
awk '$9<60{print}' $(find /usr/local/zeek/logs -name "conn.*.log") > "/usr/local/zeek/logs/conn_replace.log"
22+
rm $(find /usr/local/zeek/logs -name "conn.*.log")
23+
# conn.logとconn_long.logの両方を回収
24+
merge_log "conn*.log" "conn.log"
25+
merge_log "arp.*.log" "arp.log"
26+
merge_log "ns.*.log" "ns.log"
27+
merge_log "dns.*.log" "dns.log"
28+
merge_log "http.*.log" "http.log"
29+
merge_log "cifs.*.log" "mswin-browser.log"
30+
reformat_log "mswin-browser.log"
31+
merge_log "mydhcp.*.log" "dhcp2.log"
32+
reformat_log "dhcp2.log"
33+
merge_log "dhcpv6.*.log" "dhcpv6.log"
34+
reformat_log "dhcpv6.log"
35+
merge_log "nbns.*.log" "netbios-ns.log"
36+
reformat_log "netbios-ns.log"
37+
merge_log "ssdp.*.log" "ssdp.log"
38+
reformat_log "ssdp.log"
39+
# OTプロトコル: CC-Link
40+
merge_log "cclink-ief-basic.*.log" "cclink-ief-basic.log"
41+
merge_log "cclink-ie.*.log" "cclink-ie.log"
42+
merge_log "cclink-ie-tsn.*.log" "cclink-ie-tsn.log"
43+
merge_log "cclink-ie-tsn-slmp.*.log" "cclink-ie-tsn-slmp.log"
44+
merge_log "cclink-ie-tsn-ptp.*.log" "cclink-ie-tsn-ptp.log"
45+
46+
if [ $4 = "True" ]; then
47+
# tsharkでの出力と同じにするため
48+
merge_log "bacnet_service.*.log" "bacnet_service.log"
49+
sed -i '/^#/d' bacnet_service.log
50+
sed -i '1i #' bacnet_service.log
51+
fi
52+
53+
if [ $5 = "True" ]; then
54+
merge_log "modbus_detailed.*.log" "modbus_detailed.log"
55+
fi
56+
57+
find /usr/local/zeek/logs -name "*.log" -print0 | xargs -0 rm

‎osect_sensor/sh_tests/ot_tools/bro/data/expected/.gitkeep

Whitespace-only changes.

‎osect_sensor/sh_tests/ot_tools/bro/data/init/.gitkeep

Whitespace-only changes.

0 commit comments

Comments
 (0)