Skip to content

Commit 3b33e20

Browse files
JackWoo0831JackWoo0831
committed
Add TrackEval
1 parent b994877 commit 3b33e20

File tree

147 files changed

+11649
-27
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

147 files changed

+11649
-27
lines changed

‎tracker/config_files/mot.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Config file of MOT17 dataset
2+
3+
DATASET_ROOT: '/data/wujiapeng/datasets/' # your dataset root
4+
CATEGORY_NAMES: # category names to show
5+
- 'pedestrian'
6+
7+
CATEGORY_DICT:
8+
0: 'pedestrian'
9+
10+
CERTAIN_SEQS:
11+
-
12+
IGNORE_SEQS: # Seqs you want to ignore
13+
-
14+
15+
YAML_DICT: '' # NOTE: ONLY for yolo v5 model loader(func DetectMultiBackend)
16+
17+
TRACK_EVAL: # If use TrackEval to evaluate, use these configs
18+
'DISPLAY_LESS_PROGRESS': False
19+
'GT_FOLDER': '/data/wujiapeng/datasets/MOT17/train'
20+
'TRACKERS_FOLDER': './tracker/results'
21+
'SKIP_SPLIT_FOL': True
22+
'TRACKER_SUB_FOLDER': ''
23+
'SEQ_INFO':
24+
'MOT17-02-SDP': -
25+
'MOT17-04-SDP': -
26+
'MOT17-05-SDP': -
27+
'MOT17-09-SDP': -
28+
'MOT17-10-SDP': -
29+
'MOT17-11-SDP': -
30+
'MOT17-13-SDP': -
31+
'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt'

‎tracker/config_files/uavdt.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Config file of UAVDT dataset
2+
3+
DATASET_ROOT: '/data/wujiapeng/datasets/' # your dataset root
4+
CATEGORY_NAMES: # category names to show
5+
- 'car'
6+
7+
CATEGORY_DICT:
8+
0: 'car'
9+
10+
CERTAIN_SEQS:
11+
- 'M0101'
12+
IGNORE_SEQS: # Seqs you want to ignore
13+
-
14+
15+
YAML_DICT: './data/UAVDT.yaml' # NOTE: ONLY for yolo v5 model loader(func DetectMultiBackend)
16+
17+
TRACK_EVAL: # If use TrackEval to evaluate, use these configs
18+
'DISPLAY_LESS_PROGRESS': False
19+
'GT_FOLDER': '/data/wujiapeng/datasets/UAVDT/UAV-benchmark-M'
20+
'TRACKERS_FOLDER': './tracker/results'
21+
'SKIP_SPLIT_FOL': True
22+
'TRACKER_SUB_FOLDER': ''
23+
'SEQ_INFO':
24+
'M0101': 407
25+
'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt'

‎tracker/config_files/visdrone.yaml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Config file of VisDrone dataset
2+
3+
DATASET_ROOT: '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019'
4+
CATEGORY_NAMES:
5+
- 'pedestrain'
6+
- 'people'
7+
- 'bicycle'
8+
- 'car'
9+
- 'van'
10+
- 'truck'
11+
- 'tricycle'
12+
- 'awning-tricycle'
13+
- 'bus'
14+
- 'motor'
15+
16+
CATEGORY_DICT:
17+
0: 'pedestrain'
18+
1: 'people'
19+
2: 'bicycle'
20+
3: 'car'
21+
4: 'van'
22+
5: 'truck'
23+
6: 'tricycle'
24+
7: 'awning-tricycle'
25+
8: 'bus'
26+
9: 'motor'
27+
28+
CERTAIN_SEQS:
29+
-
30+
31+
IGNORE_SEQS: # Seqs you want to ignore
32+
-
33+
34+
YAML_DICT: './data/Visdrone_all.yaml' # NOTE: ONLY for yolo v5 model loader(func DetectMultiBackend)
35+
36+
TRACK_EVAL: # If use TrackEval to evaluate, use these configs
37+
'DISPLAY_LESS_PROGRESS': False
38+
'GT_FOLDER': '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019/VisDrone2019-MOT-test-dev/annotations'
39+
'TRACKERS_FOLDER': './tracker/results'
40+
'SKIP_SPLIT_FOL': True
41+
'TRACKER_SUB_FOLDER': ''
42+
'SEQ_INFO':
43+
'uav0000009_03358_v': 219
44+
'uav0000073_00600_v': 328
45+
'uav0000073_04464_v': 312
46+
'uav0000077_00720_v': 780
47+
'uav0000088_00290_v': 296
48+
'uav0000119_02301_v': 179
49+
'uav0000120_04775_v': 1000
50+
'uav0000161_00000_v': 308
51+
'uav0000188_00000_v': 260
52+
'uav0000201_00000_v': 677
53+
'uav0000249_00001_v': 360
54+
'uav0000249_02688_v': 244
55+
'uav0000297_00000_v': 146
56+
'uav0000297_02761_v': 373
57+
'uav0000306_00230_v': 420
58+
'uav0000355_00001_v': 468
59+
'uav0000370_00001_v': 265
60+
'GT_LOC_FORMAT': '{gt_folder}/{seq}.txt'

‎tracker/config_files/visdrone_car.yaml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Config file of VisDrone dataset
2+
3+
DATASET_ROOT: '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019'
4+
CATEGORY_NAMES:
5+
- 'car'
6+
- 'van'
7+
- 'truck'
8+
- 'bus'
9+
10+
CATEGORY_DICT:
11+
0: 'car'
12+
1: 'van'
13+
2: 'truck'
14+
3: 'bus'
15+
16+
CERTAIN_SEQS:
17+
-
18+
19+
IGNORE_SEQS: # Seqs you want to ignore
20+
- 'uav0000073_00600_v'
21+
- 'uav0000088_00290_v'
22+
- 'uav0000073_04464_v'
23+
24+
YAML_DICT: './data/Visdrone_car.yaml' # NOTE: ONLY for yolo v5 model loader(func DetectMultiBackend)
25+
26+
TRACK_EVAL: # If use TrackEval to evaluate, use these configs
27+
'DISPLAY_LESS_PROGRESS': False
28+
'GT_FOLDER': '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019/VisDrone2019-MOT-test-dev/annotations'
29+
'TRACKERS_FOLDER': './tracker/results'
30+
'CLASSES_TO_EVAL':
31+
- 'car'
32+
- 'van'
33+
- 'truck'
34+
- 'bus'
35+
'SKIP_SPLIT_FOL': True
36+
'TRACKER_SUB_FOLDER': ''
37+
'SEQ_INFO':
38+
'uav0000009_03358_v': 219
39+
'uav0000077_00720_v': 780
40+
'uav0000119_02301_v': 179
41+
'uav0000120_04775_v': 1000
42+
'uav0000161_00000_v': 308
43+
'uav0000188_00000_v': 260
44+
'uav0000201_00000_v': 677
45+
'uav0000249_00001_v': 360
46+
'uav0000249_02688_v': 244
47+
'uav0000297_00000_v': 146
48+
'uav0000297_02761_v': 373
49+
'uav0000306_00230_v': 420
50+
'uav0000355_00001_v': 468
51+
'uav0000370_00001_v': 265
52+
'GT_LOC_FORMAT': '{gt_folder}/{seq}.txt'

‎tracker/track.py‎

Lines changed: 45 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import os
1212
from time import gmtime, strftime
1313
from timer import Timer
14+
import yaml
1415

1516
from basetrack import BaseTracker # for framework
1617
from deepsort import DeepSORT
@@ -33,19 +34,22 @@
3334
pass
3435

3536
import tracker_dataloader
37+
import trackeval
3638

37-
DATASET_ROOT = '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019' # your dataset root
39+
def set_basic_params(cfgs):
40+
global CATEGORY_DICT, DATASET_ROOT, CERTAIN_SEQS, IGNORE_SEQS, YAML_DICT
41+
CATEGORY_DICT = cfgs['CATEGORY_DICT']
42+
DATASET_ROOT = cfgs['DATASET_ROOT']
43+
CERTAIN_SEQS = cfgs['CERTAIN_SEQS']
44+
IGNORE_SEQS = cfgs['IGNORE_SEQS']
45+
YAML_DICT = cfgs['YAML_DICT']
3846

39-
CATEGORY_NAMES = ['car', 'van', 'truck', 'bus']
40-
# CATEGORY_NAMES = ['pedestrain', 'people', 'bicycle', 'car', 'van', 'truck', 'tricycle', 'awning-tricycle', 'bus', 'motor']
41-
CATEGORY_DICT = {i: CATEGORY_NAMES[i] for i in range(len(CATEGORY_NAMES))} # show class
42-
43-
# IGNORE_SEQS = []
44-
IGNORE_SEQS = ['uav0000073_00600_v', 'uav0000088_00290_v'] # ignore seqs
4547

4648
timer = Timer()
4749
seq_fps = [] # list to store time used for every seq
48-
def main(opts):
50+
def main(opts, cfgs):
51+
set_basic_params(cfgs) # NOTE: set basic path and seqs params first
52+
4953
TRACKER_DICT = {
5054
'sort': BaseTracker,
5155
'deepsort': DeepSORT,
@@ -195,9 +199,33 @@ def main(opts):
195199
3. evaluate results
196200
"""
197201
print(f'average fps: {np.mean(seq_fps)}')
198-
evaluate(sorted(os.listdir(f'./tracker/results/{folder_name}')),
199-
sorted([seq + '.txt' for seq in seqs]), data_type='visdrone', result_folder=folder_name)
200-
202+
if opts.track_eval:
203+
default_eval_config = trackeval.Evaluator.get_default_eval_config()
204+
default_dataset_config = trackeval.datasets.MotChallenge2DBox.get_default_dataset_config()
205+
yaml_dataset_config = cfgs['TRACK_EVAL'] # read yaml file to read TrackEval configs
206+
for k in default_dataset_config.keys():
207+
if k in yaml_dataset_config.keys(): # if the key need to be modified
208+
default_dataset_config[k] = yaml_dataset_config[k]
209+
210+
default_metrics_config = {'METRICS': ['HOTA', 'CLEAR', 'Identity'], 'THRESHOLD': 0.5}
211+
config = {**default_eval_config, **default_dataset_config, **default_metrics_config} # Merge default configs
212+
eval_config = {k: v for k, v in config.items() if k in default_eval_config.keys()}
213+
dataset_config = {k: v for k, v in config.items() if k in default_dataset_config.keys()}
214+
metrics_config = {k: v for k, v in config.items() if k in default_metrics_config.keys()}
215+
216+
# Run code
217+
evaluator = trackeval.Evaluator(eval_config)
218+
dataset_list = [trackeval.datasets.MotChallenge2DBox(dataset_config)] if opts.datasets in ['mot', 'uavdt'] else [trackeval.datasets.VisDrone2DBox(dataset_config)]
219+
metrics_list = []
220+
for metric in [trackeval.metrics.HOTA, trackeval.metrics.CLEAR, trackeval.metrics.Identity, trackeval.metrics.VACE]:
221+
if metric.get_name() in metrics_config['METRICS']:
222+
metrics_list.append(metric(metrics_config))
223+
if len(metrics_list) == 0:
224+
raise Exception('No metrics selected for evaluation')
225+
evaluator.evaluate(dataset_list, metrics_list)
226+
else:
227+
evaluate(sorted(os.listdir(f'./tracker/results/{folder_name}')),
228+
sorted([seq + '.txt' for seq in seqs]), data_type='visdrone', result_folder=folder_name)
201229

202230
def save_results(folder_name, seq_name, results, data_type='default'):
203231
"""
@@ -330,10 +358,11 @@ def get_color(idx):
330358
# detect per several frames
331359
parser.add_argument('--detect_per_frame', type=int, default=1, help='choose how many frames per detect')
332360

361+
parser.add_argument('--track_eval', type=bool, default=True, help='Use TrackEval to evaluate')
333362

334363
opts = parser.parse_args()
335-
336-
# for debug
337-
# evaluate(sorted(os.listdir('./tracker/results/deepmot_17_08_02_38')),
338-
# sorted(os.listdir('./tracker/results/deepmot_17_08_02_38')), data_type='visdrone', result_folder='deepmot_17_08_02_38')
339-
main(opts)
364+
365+
# NOTE: read path of datasets, sequences and TrackEval configs
366+
with open(f'./tracker/config_files/{opts.dataset}.yaml', 'r') as f:
367+
cfgs = yaml.load(f, Loader=yaml.FullLoader)
368+
main(opts, cfgs)

‎tracker/track_yolov5.py‎

Lines changed: 50 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import os
1212
from time import gmtime, strftime
1313
from timer import Timer
14+
import yaml
1415

1516
from basetrack import BaseTracker # for framework
1617
from deepsort import DeepSORT
@@ -32,7 +33,9 @@
3233
pass
3334

3435
import tracker_dataloader
36+
import trackeval
3537

38+
"""
3639
DATASET_ROOT = '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019' # your dataset root
3740
# DATASET_ROOT = '/data/wujiapeng/datasets/'
3841
@@ -47,10 +50,21 @@
4750
# NOTE: ONLY for yolo v5 model loader(func DetectMultiBackend)
4851
YAML_DICT = {'visdrone': './data/Visdrone_car.yaml',
4952
'uavdt': './data/UAVDT.yaml'}
53+
"""
54+
55+
def set_basic_params(cfgs):
56+
global CATEGORY_DICT, DATASET_ROOT, CERTAIN_SEQS, IGNORE_SEQS, YAML_DICT
57+
CATEGORY_DICT = cfgs['CATEGORY_DICT']
58+
DATASET_ROOT = cfgs['DATASET_ROOT']
59+
CERTAIN_SEQS = cfgs['CERTAIN_SEQS']
60+
IGNORE_SEQS = cfgs['IGNORE_SEQS']
61+
YAML_DICT = cfgs['YAML_DICT']
62+
5063

5164
timer = Timer()
5265
seq_fps = [] # list to store time used for every seq
53-
def main(opts):
66+
def main(opts, cfgs):
67+
set_basic_params(cfgs) # NOTE: set basic path and seqs params first
5468

5569
TRACKER_DICT = {
5670
'sort': BaseTracker,
@@ -76,7 +90,7 @@ def main(opts):
7690
1. load model
7791
"""
7892
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
79-
model = DetectMultiBackend(opts.model_path, device=device, dnn=False, data=YAML_DICT[opts.dataset], fp16=False)
93+
model = DetectMultiBackend(opts.model_path, device=device, dnn=False, data=YAML_DICT, fp16=False)
8094
model.eval()
8195
# warm up
8296
model.warmup(imgsz=(1, 3, 640, 640))
@@ -196,9 +210,33 @@ def main(opts):
196210
3. evaluate results
197211
"""
198212
print(f'average fps: {np.mean(seq_fps)}')
199-
evaluate(sorted(os.listdir(f'./tracker/results/{folder_name}')),
200-
sorted([seq + '.txt' for seq in seqs]), data_type='visdrone', result_folder=folder_name)
201-
213+
if opts.track_eval:
214+
default_eval_config = trackeval.Evaluator.get_default_eval_config()
215+
default_dataset_config = trackeval.datasets.MotChallenge2DBox.get_default_dataset_config()
216+
yaml_dataset_config = cfgs['TRACK_EVAL'] # read yaml file to read TrackEval configs
217+
for k in default_dataset_config.keys():
218+
if k in yaml_dataset_config.keys(): # if the key need to be modified
219+
default_dataset_config[k] = yaml_dataset_config[k]
220+
221+
default_metrics_config = {'METRICS': ['HOTA', 'CLEAR', 'Identity'], 'THRESHOLD': 0.5}
222+
config = {**default_eval_config, **default_dataset_config, **default_metrics_config} # Merge default configs
223+
eval_config = {k: v for k, v in config.items() if k in default_eval_config.keys()}
224+
dataset_config = {k: v for k, v in config.items() if k in default_dataset_config.keys()}
225+
metrics_config = {k: v for k, v in config.items() if k in default_metrics_config.keys()}
226+
227+
# Run code
228+
evaluator = trackeval.Evaluator(eval_config)
229+
dataset_list = [trackeval.datasets.MotChallenge2DBox(dataset_config)] if opts.datasets in ['mot', 'uavdt'] else [trackeval.datasets.VisDrone2DBox(dataset_config)]
230+
metrics_list = []
231+
for metric in [trackeval.metrics.HOTA, trackeval.metrics.CLEAR, trackeval.metrics.Identity, trackeval.metrics.VACE]:
232+
if metric.get_name() in metrics_config['METRICS']:
233+
metrics_list.append(metric(metrics_config))
234+
if len(metrics_list) == 0:
235+
raise Exception('No metrics selected for evaluation')
236+
evaluator.evaluate(dataset_list, metrics_list)
237+
else:
238+
evaluate(sorted(os.listdir(f'./tracker/results/{folder_name}')),
239+
sorted([seq + '.txt' for seq in seqs]), data_type='visdrone', result_folder=folder_name)
202240

203241

204242
def save_results(folder_name, seq_name, results, data_type='default'):
@@ -299,7 +337,7 @@ def get_color(idx):
299337
if __name__ == '__main__':
300338
parser = argparse.ArgumentParser()
301339

302-
parser.add_argument('--dataset', type=str, default='visdrone', help='visdrone, or mot')
340+
parser.add_argument('--dataset', type=str, default='visdrone', help='visdrone, visdrone_car, uavdt or mot')
303341
parser.add_argument('--data_format', type=str, default='origin', help='format of reading dataset')
304342
parser.add_argument('--det_output_format', type=str, default='yolo', help='data format of output of detector, yolo or other')
305343

@@ -331,10 +369,11 @@ def get_color(idx):
331369
# detect per several frames
332370
parser.add_argument('--detect_per_frame', type=int, default=1, help='choose how many frames per detect')
333371

372+
parser.add_argument('--track_eval', type=bool, default=True, help='Use TrackEval to evaluate')
334373

335374
opts = parser.parse_args()
336-
337-
# for debug
338-
# evaluate(sorted(os.listdir('./tracker/results/deepmot_17_08_02_38')),
339-
# sorted(os.listdir('./tracker/results/deepmot_17_08_02_38')), data_type='visdrone', result_folder='deepmot_17_08_02_38')
340-
main(opts)
375+
376+
# NOTE: read path of datasets, sequences and TrackEval configs
377+
with open(f'./tracker/config_files/{opts.dataset}.yaml', 'r') as f:
378+
cfgs = yaml.load(f, Loader=yaml.FullLoader)
379+
main(opts, cfgs)

‎tracker/trackeval/__init__.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
from .eval import Evaluator
2+
from . import datasets
3+
from . import metrics
4+
from . import plotting
5+
from . import utils

‎tracker/trackeval/__pycache__/__init__.cpython-36.pyc‎

291 Bytes
Binary file not shown.

‎tracker/trackeval/__pycache__/__init__.cpython-37.pyc‎

305 Bytes
Binary file not shown.

‎tracker/trackeval/__pycache__/__init__.cpython-38.pyc‎

309 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)