onepanelio · javoweb · Jan 11, 2021 · Jan 11, 2021 · Jan 11, 2021 · Jan 11, 2021
diff --git a/workflows/tf-object-detection-training-tf2/defaults.json b/workflows/tf-object-detection-training-tf2/defaults.json
@@ -0,0 +1,295 @@
+{
+    "frcnn-res50-coco": {
+        "num_clones": 1,
+        "batch_size": 1,
+        "num_steps": 10000,
+        "num_classes": 90,
+        "initial_learning_rate": 0.0003,
+        "warmup_learning_rate": 0.0,
+        "warmup_steps": 5000,
+        "momentum_optimizer_value": 0.9,
+        "min_dimension": 800,
+        "max_dimension": 1333,
+        "height_stride": 16,
+        "width_stride": 16,
+        "first_stage_regularizer_weight": 0.0,
+        "first_stage_initializer_stddev": 0.01,
+        "first_stage_nms_score_threshold": 0.0,
+        "first_stage_nms_iou_threshold": 0.7,
+        "first_stage_max_proposals": 300,
+        "first_stage_localization_loss_weight": 2.0,
+        "first_stage_objectness_loss_weight": 1.0,
+        "initial_crop_size": 14,
+        "maxpool_kernel_size": 2,
+        "maxpool_stride": 2,
+        "second_stage_regularizer_weight": 0.0,
+        "second_stage_initializer_factor": 1.0,
+        "second_stage_initializer_mode": 2,
+        "second_stage_use_dropout": false,
+        "second_stage_dropout_keep_probability": 1.0,
+        "second_stage_nms_score_threshold": 0.0,
+        "second_stage_nms_iou_threshold": 0.6,
+        "second_stage_max_detections_per_class": 100,
+        "second_stage_max_detections_max_total_detections": 100,
+        "second_stage_localization_loss_weight": 2.0,
+        "second_stage_classification_loss_weight": 1.0
+    },
+    "frcnn-res101-coco": {
+        "num_clones": 1,
+        "batch_size": 1,
+        "num_steps": 10000,
+        "num_classes": 90,
+        "initial_learning_rate": 0.0003,
+        "warmup_learning_rate": 0.0,
+        "warmup_steps": 5000,
+        "momentum_optimizer_value": 0.9,
+        "min_dimension": 800,
+        "max_dimension": 1333,
+        "first_stage_features_stride": 16,
+        "height_stride": 16,
+        "width_stride": 16,
+        "first_stage_regularizer_weight": 0.0,
+        "first_stage_initializer_stddev": 0.01,
+        "first_stage_nms_score_threshold": 0.0,
+        "first_stage_nms_iou_threshold": 0.7,
+        "first_stage_max_proposals": 20,
+        "first_stage_localization_loss_weight": 2.0,
+        "first_stage_objectness_loss_weight": 1.0,
+        "initial_crop_size": 14,
+        "maxpool_kernel_size": 2,
+        "maxpool_stride": 2,
+        "second_stage_regularizer_weight": 0.0,
+        "second_stage_initializer_factor": 1.0,
+        "second_stage_initializer_mode": 2,
+        "second_stage_use_dropout": false,
+        "second_stage_dropout_keep_probability": 1.0,
+        "second_stage_nms_score_threshold": 0.3,
+        "second_stage_nms_iou_threshold": 0.6,
+        "second_stage_max_detections_per_class": 100,
+        "second_stage_max_detections_max_total_detections": 100,
+        "second_stage_localization_loss_weight": 2.0,
+        "second_stage_classification_loss_weight": 1.0
+    },
+    "frcnn-res101-low": {
+        "num_clones": 1,
+        "batch_size": 1,
+        "num_steps": 10,
+        "num_classes": 90,
+        "initial_learning_rate": 0.0003,
+        "warmup_learning_rate": 0.0,
+        "warmup_steps": 5000,
+        "momentum_optimizer_value": 0.9,
+        "min_dimension": 600,
+        "max_dimension": 1024,
+        "first_stage_features_stride": 16,
+        "height_stride": 16,
+        "width_stride": 16,
+        "first_stage_regularizer_weight": 0.0,
+        "first_stage_initializer_stddev": 0.01,
+        "first_stage_nms_score_threshold": 0.0,
+        "first_stage_nms_iou_threshold": 0.7,
+        "first_stage_max_proposals": 20,
+        "first_stage_localization_loss_weight": 2.0,
+        "first_stage_objectness_loss_weight": 1.0,
+        "initial_crop_size": 14,
+        "maxpool_kernel_size": 2,
+        "maxpool_stride": 2,
+        "second_stage_regularizer_weight": 0.0,
+        "second_stage_initializer_factor": 1.0,
+        "second_stage_initializer_mode": 2,
+        "second_stage_use_dropout": false,
+        "second_stage_dropout_keep_probability": 1.0,
+        "second_stage_nms_score_threshold": 0.3,
+        "second_stage_nms_iou_threshold": 0.6,
+        "second_stage_max_detections_per_class": 20,
+        "second_stage_max_detections_max_total_detections": 20,
+        "second_stage_localization_loss_weight": 2.0,
+        "second_stage_classification_loss_weight": 1.0
+    },
+    "frcnn-nas-coco": {
+        "num_clones": 1,
+        "batch_size": 1,
+        "num_steps": 10000,
+        "num_classes": 90,
+        "initial_learning_rate": 0.0003,
+        "warmup_learning_rate": 0.0,
+        "warmup_steps": 5000,
+        "momentum_optimizer_value": 0.9,
+        "min_dimension": 600,
+        "max_dimension": 1024,
+        "first_stage_features_stride": 16,
+        "height_stride": 16,
+        "width_stride": 16,
+        "first_stage_regularizer_weight": 0.0,
+        "first_stage_initializer_stddev": 0.01,
+        "first_stage_nms_score_threshold": 0.0,
+        "first_stage_nms_iou_threshold": 0.7,
+        "first_stage_max_proposals": 20,
+        "first_stage_localization_loss_weight": 2.0,
+        "first_stage_objectness_loss_weight": 1.0,
+        "initial_crop_size": 14,
+        "maxpool_kernel_size": 2,
+        "maxpool_stride": 2,
+        "second_stage_regularizer_weight": 0.0,
+        "second_stage_initializer_factor": 1.0,
+        "second_stage_initializer_mode": 2,
+        "second_stage_use_dropout": false,
+        "second_stage_dropout_keep_probability": 1.0,
+        "second_stage_nms_score_threshold": 0.3,
+        "second_stage_nms_iou_threshold": 0.6,
+        "second_stage_max_detections_per_class": 20,
+        "second_stage_max_detections_max_total_detections": 20,
+        "second_stage_localization_loss_weight": 2.0,
+        "second_stage_classification_loss_weight": 1.0
+    },
+    "ssd-mobilenet-v1-coco2": {
+        "num_clones": 1,
+        "batch_size": 24,
+        "num_steps": 15000,
+        "num_classes": 90,
+        "initial_learning_rate": 0.004,
+        "decay_steps": 800720,
+        "decay_factor": 0.95,
+        "momentum_decay": 0.9,
+        "momentum_epsilon": 1.0,
+        "momentum_optimizer_value": 0.9,
+        "image_height": 300,
+        "image_width": 300,
+        "depth_multiplier": 1.0,
+        "min_depth": 16,
+        "first_stage_regularizer_weight": 0.00004,
+        "first_stage_initializer_mean": 0.0,
+        "first_stage_initializer_stddev": 0.03,
+        "first_stage_activation": 2,
+        "first_stage_batchnorm_decay": 0.9997,
+        "first_stage_batchnorm_epsilon": 0.001,
+        "boxcoder_y_scale": 10.0,
+        "boxcoder_x_scale": 10.0,
+        "boxcoder_height_scale": 5.0,
+        "boxcoder_width_scale": 5.0,
+        "matched_threshold": 0.5,
+        "unmatched_threshold": 0.5,
+        "second_stage_regularizer_weight": 0.00004,
+        "second_stage_initializer_mean": 0.0,
+        "second_stage_initializer_stddev": 0.03,
+        "second_stage_activation": 2,
+        "second_stage_batchnorm_decay": 0.9997,
+        "second_stage_batchnorm_epsilon": 0.001,
+        "second_stage_min_depth": 0,
+        "second_stage_max_depth": 0,
+        "second_stage_num_layers_before_predictor": 0,
+        "second_stage_use_dropout": false,
+        "second_stage_dropout_keep_probability": 0.8,
+        "second_stage_dropout_kernel_size": 3,
+        "second_stage_dropout_box_code_size": 4,
+        "anchor_generator_num_layers": 6,
+        "anchor_generator_min_scale": 0.2,
+        "anchor_generator_max_scale": 0.95,
+        "second_stage_nms_score_threshold": 0.3,
+        "second_stage_nms_iou_threshold": 0.6,
+        "second_stage_max_detections_per_class": 100,
+        "second_stage_max_detections_max_total_detections": 100,
+        "second_stage_localization_loss_weight": 1.0,
+        "second_stage_classification_loss_weight": 1.0
+    },
+    "ssd-mobilenet-v2-coco": {
+        "num_clones": 1,
+        "batch_size": 24,
+        "num_steps": 15000,
+        "num_classes": 90,
+        "initial_learning_rate": 0.004,
+        "decay_steps": 800720,
+        "decay_factor": 0.95,
+        "momentum_decay": 0.9,
+        "momentum_epsilon": 1.0,
+        "momentum_optimizer_value": 0.9,
+        "image_height": 300,
+        "image_width": 300,
+        "depth_multiplier": 1.0,
+        "min_depth": 16,
+        "first_stage_regularizer_weight": 0.00004,
+        "first_stage_initializer_mean": 0.0,
+        "first_stage_initializer_stddev": 0.03,
+        "first_stage_activation": 2,
+        "first_stage_batchnorm_decay": 0.9997,
+        "first_stage_batchnorm_epsilon": 0.001,
+        "boxcoder_y_scale": 10.0,
+        "boxcoder_x_scale": 10.0,
+        "boxcoder_height_scale": 5.0,
+        "boxcoder_width_scale": 5.0,
+        "matched_threshold": 0.5,
+        "unmatched_threshold": 0.5,
+        "second_stage_regularizer_weight": 0.00004,
+        "second_stage_initializer_mean": 0.0,
+        "second_stage_initializer_stddev": 0.03,
+        "second_stage_activation": 2,
+        "second_stage_batchnorm_decay": 0.9997,
+        "second_stage_batchnorm_epsilon": 0.001,
+        "second_stage_min_depth": 0,
+        "second_stage_max_depth": 0,
+        "second_stage_num_layers_before_predictor": 0,
+        "second_stage_use_dropout": false,
+        "second_stage_dropout_keep_probability": 0.8,
+        "second_stage_dropout_kernel_size": 3,
+        "second_stage_dropout_box_code_size": 4,
+        "anchor_generator_num_layers": 6,
+        "anchor_generator_min_scale": 0.2,
+        "anchor_generator_max_scale": 0.95,
+        "second_stage_nms_score_threshold": 0.3,
+        "second_stage_nms_iou_threshold": 0.6,
+        "second_stage_max_detections_per_class": 100,
+        "second_stage_max_detections_max_total_detections": 100,
+        "second_stage_localization_loss_weight": 1.0,
+        "second_stage_classification_loss_weight": 1.0
+    },
+    "ssdlite-mobilenet-coco": {
+        "num_clones": 1,
+        "batch_size": 24,
+        "num_steps": 1000,
+        "num_classes": 90,
+        "initial_learning_rate": 0.004,
+        "decay_steps": 800720,
+        "decay_factor": 0.95,
+        "momentum_decay": 0.9,
+        "momentum_epsilon": 0.000003,
+        "momentum_optimizer_value": 1.0,
+        "image_height": 300,
+        "image_width": 300,
+        "depth_multiplier": 1.0,
+        "min_depth": 16,
+        "first_stage_regularizer_weight": 0.00004,
+        "first_stage_initializer_mean": 0.0,
+        "first_stage_initializer_stddev": 0.03,
+        "first_stage_activation": 2,
+        "first_stage_batchnorm_decay": 0.9997,
+        "first_stage_batchnorm_epsilon": 0.001,
+        "boxcoder_y_scale": 10.0,
+        "boxcoder_x_scale": 10.0,
+        "boxcoder_height_scale": 5.0,
+        "boxcoder_width_scale": 5.0,
+        "matched_threshold": 0.5,
+        "unmatched_threshold": 0.5,
+        "second_stage_regularizer_weight": 0.00004,
+        "second_stage_initializer_mean": 0.0,
+        "second_stage_initializer_stddev": 0.03,
+        "second_stage_activation": 2,
+        "second_stage_batchnorm_decay": 0.9997,
+        "second_stage_batchnorm_epsilon": 0.001,
+        "second_stage_min_depth": 0,
+        "second_stage_max_depth": 0,
+        "second_stage_num_layers_before_predictor": 0,
+        "second_stage_use_dropout": false,
+        "second_stage_dropout_keep_probability": 0.8,
+        "second_stage_dropout_kernel_size": 3,
+        "second_stage_dropout_box_code_size": 4,
+        "anchor_generator_num_layers": 6,
+        "anchor_generator_min_scale": 0.2,
+        "anchor_generator_max_scale": 0.95,
+        "second_stage_nms_score_threshold": 0.3,
+        "second_stage_nms_iou_threshold": 0.6,
+        "second_stage_max_detections_per_class": 100,
+        "second_stage_max_detections_max_total_detections": 100,
+        "second_stage_localization_loss_weight": 1.0,
+        "second_stage_classification_loss_weight": 1.0
+    }
+  }
diff --git a/workflows/tf-object-detection-training-tf2/main.py b/workflows/tf-object-detection-training-tf2/main.py
@@ -0,0 +1,86 @@
+import sys
+import os
+import subprocess
+import shutil
+import urllib.request
+import tarfile
+import argparse
+
+from utils import convert_labels_to_csv, create_pipeline
+
+
+def main(params):
+    if not os.path.isdir('/mnt/data/models'):
+        try:
+            os.remove('/mnt/data/models')
+        except:
+            pass
+        os.makedirs('/mnt/data/models/')
+
+	#check if base model exists, if not then download
+    if params['sys_finetune_checkpoint'] == '':
+        print('base model does not exist, downloading...')
+        #urllib.request.urlretrieve('https://github.com/onepanelio/templates/releases/download/v0.2.0/{}.tar'.format(params['model']), '/mnt/data/models/model.tar')
+        urllib.request.urlretrieve('http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet50_v1_800x1333_coco17_gpu-8.tar.gz'.format(params['model']), '/mnt/data/models/model.tar')
+        model_files = tarfile.open('/mnt/data/models/model.tar')
+        model_files.extractall('/mnt/data/models')
+        model_files.close()
+        #model_dir = '/mnt/data/models/'+params['model']
+        model_dir = '/mnt/data/models/'+'faster_rcnn_resnet50_v1_800x1333_coco17_gpu-8'
+        files = os.listdir(model_dir)
+        for f in files:
+            shutil.move(model_dir+'/'+f,'/mnt/data/models')
+
+    params = create_pipeline('/mnt/data/models/pipeline.config',
+        '/mnt/data/models/checkpoint/ckpt-0',
+        params['dataset']+'/label_map.pbtxt',
+        params['dataset']+'/*.tfrecord',
+        params['dataset']+'/default.tfrecord',
+        '/mnt/output/pipeline.config',
+        params)
+
+    os.chdir('/mnt/output')
+    os.mkdir('eval/')
+    return_code = subprocess.call(['python',
+        '/mnt/src/tf/research/object_detection/model_main_tf2.py',
+        '--pipeline_config_path=/mnt/output/pipeline.config',
+        '--model_dir=/mnt/output/',
+        '--num_train_steps={}'.format(params['epochs']),
+        '--alsologtostderr'
+    ])
+    if return_code != 0:
+        print(return_code)
+        raise RuntimeError('Training process failed')
+    return_code = subprocess.call(['python',
+        '/mnt/src/tf/research/object_detection/exporter_main_v2.py',
+        '--input_type=image_tensor',
+        '--pipeline_config_path=/mnt/output/pipeline.config',
+        '--trained_checkpoint_dir=/mnt/output/',
+        '--output_directory=/mnt/output/model/'
+    ])
+    if return_code != 0:
+        print(return_code)
+        raise RuntimeError('Model export process failed')
+
+    # generate lable map
+    convert_labels_to_csv(params['dataset'])
+    print('Training complete and output saved')
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Train TFOD.')
+    parser.add_argument('--dataset', default='/mnt/data/datasets')
+    parser.add_argument('--extras', help='hyperparameters or other configs')
+    parser.add_argument('--sys_finetune_checkpoint', default=' ', help='path to checkpoint')
+    parser.add_argument('--model', default='frcnn-res50-coco', help='which model to train')
+    parser.add_argument('--num_classes', default=81, type=int, help='number of classes')
+    args = parser.parse_args()
+    # parse parameters
+    # sample: epochs=100;num_classes=1
+    print('Arguments: ', args)
+    extras = args.extras.split('\n')
+    extras_processed = [i.split('#')[0].replace(' ','') for i in extras if i]
+    params = {i.split('=')[0]:i.split('=')[1] for i in extras_processed}
+    params.update(vars(args))
+    print('Processed parameters: ', params)
+    main(params)
+