274 lines
12 KiB
Python
274 lines
12 KiB
Python
# copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import traceback
|
|
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--filename", type=str, help="The name of log which need to analysis.")
|
|
parser.add_argument(
|
|
"--log_with_profiler", type=str, help="The path of train log with profiler")
|
|
parser.add_argument(
|
|
"--profiler_path", type=str, help="The path of profiler timeline log.")
|
|
parser.add_argument(
|
|
"--keyword", type=str, help="Keyword to specify analysis data")
|
|
parser.add_argument(
|
|
"--separator", type=str, default=None, help="Separator of different field in log")
|
|
parser.add_argument(
|
|
'--position', type=int, default=None, help='The position of data field')
|
|
parser.add_argument(
|
|
'--range', type=str, default="", help='The range of data field to intercept')
|
|
parser.add_argument(
|
|
'--base_batch_size', type=int, help='base_batch size on gpu')
|
|
parser.add_argument(
|
|
'--skip_steps', type=int, default=0, help='The number of steps to be skipped')
|
|
parser.add_argument(
|
|
'--model_mode', type=int, default=-1, help='Analysis mode, default value is -1')
|
|
parser.add_argument(
|
|
'--ips_unit', type=str, default=None, help='IPS unit')
|
|
parser.add_argument(
|
|
'--model_name', type=str, default=0, help='training model_name, transformer_base')
|
|
parser.add_argument(
|
|
'--mission_name', type=str, default=0, help='training mission name')
|
|
parser.add_argument(
|
|
'--direction_id', type=int, default=0, help='training direction_id')
|
|
parser.add_argument(
|
|
'--run_mode', type=str, default="sp", help='multi process or single process')
|
|
parser.add_argument(
|
|
'--index', type=int, default=1, help='{1: speed, 2:mem, 3:profiler, 6:max_batch_size}')
|
|
parser.add_argument(
|
|
'--gpu_num', type=int, default=1, help='nums of training gpus')
|
|
args = parser.parse_args()
|
|
args.separator = None if args.separator == "None" else args.separator
|
|
return args
|
|
|
|
|
|
def _is_number(num):
|
|
pattern = re.compile(r'^[-+]?[-0-9]\d*\.\d*|[-+]?\.?[0-9]\d*$')
|
|
result = pattern.match(num)
|
|
if result:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
|
|
class TimeAnalyzer(object):
|
|
def __init__(self, filename, keyword=None, separator=None, position=None, range="-1"):
|
|
if filename is None:
|
|
raise Exception("Please specify the filename!")
|
|
|
|
if keyword is None:
|
|
raise Exception("Please specify the keyword!")
|
|
|
|
self.filename = filename
|
|
self.keyword = keyword
|
|
self.separator = separator
|
|
self.position = position
|
|
self.range = range
|
|
self.records = None
|
|
self._distil()
|
|
|
|
def _distil(self):
|
|
self.records = []
|
|
with open(self.filename, "r") as f_object:
|
|
lines = f_object.readlines()
|
|
for line in lines:
|
|
if self.keyword not in line:
|
|
continue
|
|
try:
|
|
result = None
|
|
|
|
# Distil the string from a line.
|
|
line = line.strip()
|
|
line_words = line.split(self.separator) if self.separator else line.split()
|
|
if args.position:
|
|
result = line_words[self.position]
|
|
else:
|
|
# Distil the string following the keyword.
|
|
for i in range(len(line_words) - 1):
|
|
if line_words[i] == self.keyword:
|
|
result = line_words[i + 1]
|
|
break
|
|
|
|
# Distil the result from the picked string.
|
|
if not self.range:
|
|
result = result[0:]
|
|
elif _is_number(self.range):
|
|
result = result[0: int(self.range)]
|
|
else:
|
|
result = result[int(self.range.split(":")[0]): int(self.range.split(":")[1])]
|
|
self.records.append(float(result))
|
|
except Exception as exc:
|
|
print("line is: {}; separator={}; position={}".format(line, self.separator, self.position))
|
|
|
|
print("Extract {} records: separator={}; position={}".format(len(self.records), self.separator, self.position))
|
|
|
|
def _get_fps(self, mode, batch_size, gpu_num, avg_of_records, run_mode, unit=None):
|
|
if mode == -1 and run_mode == 'sp':
|
|
assert unit, "Please set the unit when mode is -1."
|
|
fps = gpu_num * avg_of_records
|
|
elif mode == -1 and run_mode == 'mp':
|
|
assert unit, "Please set the unit when mode is -1."
|
|
fps = gpu_num * avg_of_records #temporarily, not used now
|
|
print("------------this is mp")
|
|
elif mode == 0:
|
|
# s/step -> samples/s
|
|
fps = (batch_size * gpu_num) / avg_of_records
|
|
unit = "samples/s"
|
|
elif mode == 1:
|
|
# steps/s -> steps/s
|
|
fps = avg_of_records
|
|
unit = "steps/s"
|
|
elif mode == 2:
|
|
# s/step -> steps/s
|
|
fps = 1 / avg_of_records
|
|
unit = "steps/s"
|
|
elif mode == 3:
|
|
# steps/s -> samples/s
|
|
fps = batch_size * gpu_num * avg_of_records
|
|
unit = "samples/s"
|
|
elif mode == 4:
|
|
# s/epoch -> s/epoch
|
|
fps = avg_of_records
|
|
unit = "s/epoch"
|
|
else:
|
|
ValueError("Unsupported analysis mode.")
|
|
|
|
return fps, unit
|
|
|
|
def analysis(self, batch_size, gpu_num=1, skip_steps=0, mode=-1, run_mode='sp', unit=None):
|
|
if batch_size <= 0:
|
|
print("base_batch_size should larger than 0.")
|
|
return 0, ''
|
|
|
|
if len(self.records) <= skip_steps: # to address the condition which item of log equals to skip_steps
|
|
print("no records")
|
|
return 0, ''
|
|
|
|
sum_of_records = 0
|
|
sum_of_records_skipped = 0
|
|
skip_min = self.records[skip_steps]
|
|
skip_max = self.records[skip_steps]
|
|
|
|
count = len(self.records)
|
|
for i in range(count):
|
|
sum_of_records += self.records[i]
|
|
if i >= skip_steps:
|
|
sum_of_records_skipped += self.records[i]
|
|
if self.records[i] < skip_min:
|
|
skip_min = self.records[i]
|
|
if self.records[i] > skip_max:
|
|
skip_max = self.records[i]
|
|
|
|
avg_of_records = sum_of_records / float(count)
|
|
avg_of_records_skipped = sum_of_records_skipped / float(count - skip_steps)
|
|
|
|
fps, fps_unit = self._get_fps(mode, batch_size, gpu_num, avg_of_records, run_mode, unit)
|
|
fps_skipped, _ = self._get_fps(mode, batch_size, gpu_num, avg_of_records_skipped, run_mode, unit)
|
|
if mode == -1:
|
|
print("average ips of %d steps, skip 0 step:" % count)
|
|
print("\tAvg: %.3f %s" % (avg_of_records, fps_unit))
|
|
print("\tFPS: %.3f %s" % (fps, fps_unit))
|
|
if skip_steps > 0:
|
|
print("average ips of %d steps, skip %d steps:" % (count, skip_steps))
|
|
print("\tAvg: %.3f %s" % (avg_of_records_skipped, fps_unit))
|
|
print("\tMin: %.3f %s" % (skip_min, fps_unit))
|
|
print("\tMax: %.3f %s" % (skip_max, fps_unit))
|
|
print("\tFPS: %.3f %s" % (fps_skipped, fps_unit))
|
|
elif mode == 1 or mode == 3:
|
|
print("average latency of %d steps, skip 0 step:" % count)
|
|
print("\tAvg: %.3f steps/s" % avg_of_records)
|
|
print("\tFPS: %.3f %s" % (fps, fps_unit))
|
|
if skip_steps > 0:
|
|
print("average latency of %d steps, skip %d steps:" % (count, skip_steps))
|
|
print("\tAvg: %.3f steps/s" % avg_of_records_skipped)
|
|
print("\tMin: %.3f steps/s" % skip_min)
|
|
print("\tMax: %.3f steps/s" % skip_max)
|
|
print("\tFPS: %.3f %s" % (fps_skipped, fps_unit))
|
|
elif mode == 0 or mode == 2:
|
|
print("average latency of %d steps, skip 0 step:" % count)
|
|
print("\tAvg: %.3f s/step" % avg_of_records)
|
|
print("\tFPS: %.3f %s" % (fps, fps_unit))
|
|
if skip_steps > 0:
|
|
print("average latency of %d steps, skip %d steps:" % (count, skip_steps))
|
|
print("\tAvg: %.3f s/step" % avg_of_records_skipped)
|
|
print("\tMin: %.3f s/step" % skip_min)
|
|
print("\tMax: %.3f s/step" % skip_max)
|
|
print("\tFPS: %.3f %s" % (fps_skipped, fps_unit))
|
|
|
|
return round(fps_skipped, 3), fps_unit
|
|
|
|
|
|
if __name__ == "__main__":
|
|
args = parse_args()
|
|
run_info = dict()
|
|
run_info["log_file"] = args.filename
|
|
run_info["model_name"] = args.model_name
|
|
run_info["mission_name"] = args.mission_name
|
|
run_info["direction_id"] = args.direction_id
|
|
run_info["run_mode"] = args.run_mode
|
|
run_info["index"] = args.index
|
|
run_info["gpu_num"] = args.gpu_num
|
|
run_info["FINAL_RESULT"] = 0
|
|
run_info["JOB_FAIL_FLAG"] = 0
|
|
|
|
try:
|
|
if args.index == 1:
|
|
if args.gpu_num == 1:
|
|
run_info["log_with_profiler"] = args.log_with_profiler
|
|
run_info["profiler_path"] = args.profiler_path
|
|
analyzer = TimeAnalyzer(args.filename, args.keyword, args.separator, args.position, args.range)
|
|
run_info["FINAL_RESULT"], run_info["UNIT"] = analyzer.analysis(
|
|
batch_size=args.base_batch_size,
|
|
gpu_num=args.gpu_num,
|
|
skip_steps=args.skip_steps,
|
|
mode=args.model_mode,
|
|
run_mode=args.run_mode,
|
|
unit=args.ips_unit)
|
|
try:
|
|
if int(os.getenv('job_fail_flag')) == 1 or int(run_info["FINAL_RESULT"]) == 0:
|
|
run_info["JOB_FAIL_FLAG"] = 1
|
|
except:
|
|
pass
|
|
elif args.index == 3:
|
|
run_info["FINAL_RESULT"] = {}
|
|
records_fo_total = TimeAnalyzer(args.filename, 'Framework overhead', None, 3, '').records
|
|
records_fo_ratio = TimeAnalyzer(args.filename, 'Framework overhead', None, 5).records
|
|
records_ct_total = TimeAnalyzer(args.filename, 'Computation time', None, 3, '').records
|
|
records_gm_total = TimeAnalyzer(args.filename, 'GpuMemcpy Calls', None, 4, '').records
|
|
records_gm_ratio = TimeAnalyzer(args.filename, 'GpuMemcpy Calls', None, 6).records
|
|
records_gmas_total = TimeAnalyzer(args.filename, 'GpuMemcpyAsync Calls', None, 4, '').records
|
|
records_gms_total = TimeAnalyzer(args.filename, 'GpuMemcpySync Calls', None, 4, '').records
|
|
run_info["FINAL_RESULT"]["Framework_Total"] = records_fo_total[0] if records_fo_total else 0
|
|
run_info["FINAL_RESULT"]["Framework_Ratio"] = records_fo_ratio[0] if records_fo_ratio else 0
|
|
run_info["FINAL_RESULT"]["ComputationTime_Total"] = records_ct_total[0] if records_ct_total else 0
|
|
run_info["FINAL_RESULT"]["GpuMemcpy_Total"] = records_gm_total[0] if records_gm_total else 0
|
|
run_info["FINAL_RESULT"]["GpuMemcpy_Ratio"] = records_gm_ratio[0] if records_gm_ratio else 0
|
|
run_info["FINAL_RESULT"]["GpuMemcpyAsync_Total"] = records_gmas_total[0] if records_gmas_total else 0
|
|
run_info["FINAL_RESULT"]["GpuMemcpySync_Total"] = records_gms_total[0] if records_gms_total else 0
|
|
else:
|
|
print("Not support!")
|
|
except Exception:
|
|
traceback.print_exc()
|
|
print("{}".format(json.dumps(run_info))) # it's required, for the log file path insert to the database
|
|
|