create new examples directory

Author: felixhjh

examples/Cpp/PaddleClas/imagenet/README.md

@@ -0,0 +1,41 @@
+## Image Classification
+
+([简体中文](./README_CN.md)|English)
+
+The example uses the ResNet50_vd model to perform the imagenet 1000 classification task.
+
+### Get model config and sample dataset
+```
+sh get_model.sh
+```
+
+### Install preprocess module
+
+```
+pip3 install paddle_serving_app
+```
+
+
+### Inference Service(Support BRPC-Client/GRPC-Client/Http-Client)
+
+launch server side
+```
+python3 -m paddle_serving_server.serve --model ResNet50_vd_model --port 9696 #cpu inference service
+```
+
+```
+python3 -m paddle_serving_server.serve --model ResNet50_vd_model --port 9696 --gpu_ids 0 #gpu inference service
+```
+
+### BRPC-Client
+client send inference request
+```
+python3 resnet50_rpc_client.py ResNet50_vd_client_config/serving_client_conf.prototxt
+```
+*the port of server side in this example is 9696
+
+### GRPC-Client/Http-Client
+client send inference request
+```
+python3 resnet50_http_client.py ResNet50_vd_client_config/serving_client_conf.prototxt
+```

examples/Cpp/PaddleClas/imagenet/README_CN.md

@@ -0,0 +1,41 @@
+## 图像分类示例
+
+(简体中文|[English](./README.md))
+
+示例中采用ResNet50_vd模型执行imagenet 1000分类任务。
+
+### 获取模型配置文件和样例数据
+```
+sh get_model.sh
+```
+
+### 安装数据预处理模块
+
+```
+pip3 install paddle_serving_app
+```
+
+### 启动服务端（支持BRPC-Client、GRPC-Client、Http-Client）
+
+启动server端
+```
+python3 -m paddle_serving_server.serve --model ResNet50_vd_model --port 9696 #cpu预测服务
+```
+
+```
+python3 -m paddle_serving_server.serve --model ResNet50_vd_model --port 9696 --gpu_ids 0 #gpu预测服务
+```
+
+### BRPC-Client预测
+client端进行预测
+```
+python3 resnet50_rpc_client.py ResNet50_vd_client_config/serving_client_conf.prototxt
+```
+*server端示例中服务端口为9696端口
+
+
+### GRPC-Client/Http-Client预测
+client端进行预测
+```
+python3 resnet50_http_client.py ResNet50_vd_client_config/serving_client_conf.prototxt
+```

examples/Cpp/PaddleClas/imagenet/benchmark.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# pylint: disable=doc-string-missing
+
+from __future__ import unicode_literals, absolute_import
+import os
+import sys
+import time
+import requests
+import json
+import base64
+from paddle_serving_client import Client
+from paddle_serving_client.utils import MultiThreadRunner
+from paddle_serving_client.utils import benchmark_args, show_latency
+from paddle_serving_app.reader import Sequential, File2Image, Resize
+from paddle_serving_app.reader import CenterCrop, RGB2BGR, Transpose, Div, Normalize
+
+args = benchmark_args()
+
+seq_preprocess = Sequential([
+    File2Image(), Resize(256), CenterCrop(224), RGB2BGR(), Transpose((2, 0, 1)),
+    Div(255), Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], True)
+])
+
+
+def single_func(idx, resource):
+    file_list = []
+    turns = resource["turns"]
+    latency_flags = False
+    if os.getenv("FLAGS_serving_latency"):
+        latency_flags = True
+        latency_list = []
+    for file_name in os.listdir("./image_data/n01440764"):
+        file_list.append(file_name)
+    img_list = []
+    for i in range(1000):
+        img_list.append("./image_data/n01440764/" + file_list[i])
+    profile_flags = False
+    if "FLAGS_profile_client" in os.environ and os.environ[
+            "FLAGS_profile_client"]:
+        profile_flags = True
+    if args.request == "rpc":
+        fetch = ["score"]
+        client = Client()
+        client.load_client_config(args.model)
+        client.connect([resource["endpoint"][idx % len(resource["endpoint"])]])
+        start = time.time()
+        for i in range(turns):
+            if args.batch_size >= 1:
+                l_start = time.time()
+                feed_batch = []
+                i_start = time.time()
+                for bi in range(args.batch_size):
+                    img = seq_preprocess(img_list[i])
+                    feed_batch.append({"image": img})
+                i_end = time.time()
+                if profile_flags:
+                    print("PROFILE\tpid:{}\timage_pre_0:{} image_pre_1:{}".
+                          format(os.getpid(),
+                                 int(round(i_start * 1000000)),
+                                 int(round(i_end * 1000000))))
+
+                result = client.predict(feed=feed_batch, fetch=fetch)
+                l_end = time.time()
+                if latency_flags:
+                    latency_list.append(l_end * 1000 - l_start * 1000)
+            else:
+                print("unsupport batch size {}".format(args.batch_size))
+
+    elif args.request == "http":
+        py_version = sys.version_info[0]
+        server = "http://" + resource["endpoint"][idx % len(resource[
+            "endpoint"])] + "/image/prediction"
+        start = time.time()
+        for i in range(turns):
+            if py_version == 2:
+                image = base64.b64encode(
+                    open("./image_data/n01440764/" + file_list[i]).read())
+            else:
+                image_path = "./image_data/n01440764/" + file_list[i]
+                image = base64.b64encode(open(image_path, "rb").read()).decode(
+                    "utf-8")
+            req = json.dumps({"feed": [{"image": image}], "fetch": ["score"]})
+            r = requests.post(
+                server, data=req, headers={"Content-Type": "application/json"})
+    end = time.time()
+    if latency_flags:
+        return [[end - start], latency_list]
+    return [[end - start]]
+
+
+if __name__ == '__main__':
+    multi_thread_runner = MultiThreadRunner()
+    endpoint_list = [
+        "127.0.0.1:9292", "127.0.0.1:9293", "127.0.0.1:9294", "127.0.0.1:9295"
+    ]
+    turns = 100
+    start = time.time()
+    result = multi_thread_runner.run(
+        single_func, args.thread, {"endpoint": endpoint_list,
+                                   "turns": turns})
+    #result = single_func(0, {"endpoint": endpoint_list})
+    end = time.time()
+    total_cost = end - start
+    avg_cost = 0
+    for i in range(args.thread):
+        avg_cost += result[0][i]
+    avg_cost = avg_cost / args.thread
+    print("total cost: {}s".format(end - start))
+    print("each thread cost: {}s.".format(avg_cost))
+    print("qps: {}samples/s".format(args.batch_size * args.thread * turns /
+                                    total_cost))
+    if os.getenv("FLAGS_serving_latency"):
+        show_latency(result[1])

examples/Cpp/PaddleClas/imagenet/benchmark.sh

@@ -0,0 +1,50 @@
+rm profile_log*
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+export FLAGS_profile_server=1
+export FLAGS_profile_client=1
+python -m paddle_serving_server.serve --model $1 --port 9292 --thread 4 --gpu_ids 0,1,2,3 --mem_optim --ir_optim  2> elog > stdlog &
+
+sleep 5
+gpu_id=0
+#save cpu and gpu utilization log
+if [ -d utilization ];then
+    rm -rf utilization
+else
+    mkdir utilization
+fi
+
+#warm up
+$PYTHONROOT/bin/python3 benchmark.py --thread 4 --batch_size 1 --model $2/serving_client_conf.prototxt --request rpc > profile 2>&1
+echo -e "import psutil\ncpu_utilization=psutil.cpu_percent(1,False)\nprint('CPU_UTILIZATION:', cpu_utilization)\n" > cpu_utilization.py
+
+for thread_num in 1 4 8 16
+do
+for batch_size in 1 4 16 64
+do
+    job_bt=`date '+%Y%m%d%H%M%S'`
+    nvidia-smi --id=0 --query-compute-apps=used_memory --format=csv -lms 100 > gpu_use.log 2>&1 &
+    nvidia-smi --id=0 --query-gpu=utilization.gpu --format=csv -lms 100 > gpu_utilization.log 2>&1 &
+    gpu_memory_pid=$!
+    $PYTHONROOT/bin/python benchmark.py --thread $thread_num --batch_size $batch_size --model $2/serving_client_conf.prototxt --request rpc > profile 2>&1
+    kill ${gpu_memory_pid}
+    kill `ps -ef|grep used_memory|awk '{print $2}'`
+    echo "model name :" $1
+    echo "thread num :" $thread_num
+    echo "batch size :" $batch_size
+    echo "=================Done===================="
+    echo "model name :$1" >> profile_log
+    echo "batch size :$batch_size" >> profile_log
+    job_et=`date '+%Y%m%d%H%M%S'`
+    awk 'BEGIN {max = 0} {if(NR>1){if ($1 > max) max=$1}} END {print "MAX_GPU_MEMORY:", max}' gpu_use.log >> profile_log_$1
+    awk 'BEGIN {max = 0} {if(NR>1){if ($1 > max) max=$1}} END {print "GPU_UTILIZATION:", max}' gpu_utilization.log >> profile_log_$1
+    rm -rf gpu_use.log gpu_utilization.log
+    $PYTHONROOT/bin/python ../util/show_profile.py profile $thread_num >> profile_log
+    tail -n 8 profile >> profile_log
+    echo "" >> profile_log_$1
+done
+done
+
+#Divided log
+awk 'BEGIN{RS="\n\n"}{i++}{print > "ResNet_log_"i}' profile_log_$1
+mkdir $1_log && mv ResNet_log_* $1_log
+ps -ef|grep 'serving'|grep -v grep|cut -c 9-15 | xargs kill -9

examples/Cpp/PaddleClas/imagenet/daisy.jpg

@@ -0,0 +1,7 @@
+wget --no-check-certificate https://paddle-serving.bj.bcebos.com/imagenet-example/ResNet50_vd.tar.gz
+tar -xzvf ResNet50_vd.tar.gz
+wget --no-check-certificate https://paddle-serving.bj.bcebos.com/imagenet-example/ResNet101_vd.tar.gz
+tar -xzvf ResNet101_vd.tar.gz
+
+wget --no-check-certificate https://paddle-serving.bj.bcebos.com/imagenet-example/image_data.tar.gz
+tar -xzvf image_data.tar.gz