1import os
2import numpy as np
3import tensorflow.compat.v1 as tf
4tf.compat.v1.disable_eager_execution()
5 
6def _wget_demo_tgz():
7    # 下载一个公开的resnet50模型。
8    url = 'http://url/to/your/model/YOUR_MODEL.tar.gz'
9    local_tgz = os.path.basename(url)
10    local_dir = local_tgz.split('.')[0]
11    if not os.path.exists(local_dir):
12        luno.util.wget(url, local_tgz)
13        luno.util.unpack(local_tgz)
14    model_path = os.path.abspath(os.path.join(local_dir, "frozen_inference_graph.pb"))
15    graph_def = tf.GraphDef()
16    with open(model_path, 'rb') as f:
17        graph_def.ParseFromString(f.read())
18    # 以随机数作为测试数据。
19    test_data = np.random.rand(1, 800, 1000, 3)
20    return graph_def, {'image_tensor:0': test_data}
21 
22graph_def, test_data = _wget_demo_tgz()
23 
24input_nodes=['image_tensor']
25output_nodes = ['detection_boxes', 'detection_scores', 'detection_classes', 'num_detections', 'detection_masks']

1import time
2 
3def benchmark(model):
4    tf.reset_default_graph()
5    with tf.Session() as sess:
6        sess.graph.as_default()
7        tf.import_graph_def(model, name="")
8        # Warmup!
9        for i in range(0, 1000):
10            sess.run(['image_tensor:0'], test_data)
11        # Benchmark!
12        num_runs = 1000
13        start = time.time()
14        for i in range(0, num_runs):
15            sess.run(['image_tensor:0'], test_data)
16        elapsed = time.time() - start
17        rt_ms = elapsed / num_runs * 1000.0
18        # Show the result!
19        print("Latency of model: {:.2f} ms.".format(rt_ms))
20 
21# original graph
22print("before compile:")
23benchmark(graph_def)

1import aiak_inference
2optimized_model = aiak_inference.optimize(
3graph_def,
4'gpu',
5outputs=['detection_boxes', 'detection_scores', 'detection_classes', 'num_detections', 'detection_masks']
6)

1# optimized graph
2print("after compile:")
3benchmark(optimized_model)

1import os
2import time
3import torch
4import torchvision.models as models
5
6model = models.resnet50().float().cuda()
7model = torch.jit.script(model).eval()     # 使用jit转为静态图
8dummy = torch.rand(1, 3, 224, 224).cuda()

1@torch.no_grad()
2def benchmark(model, inp):
3    for i in range(100):
4        model(inp)
5    start = time.time()
6    for i in range(200):
7        model(inp)
8    elapsed_ms = (time.time() - start) * 1000
9    print("Latency: {:.2f}".format(elapsed_ms / 200))
10
11# benchmark before optimization
12print("before optimization:")
13benchmark(model, dummy)

1import aiak_inference
2
3optimized_model = aiak_inference.optimize(
4model,
5'gpu',
6test_data=[dummy],
7)

1# benchmark after optimization
2print("after optimization:")
3benchmark(optimized_model, dummy)