fmt

Author: lxning

cpp/build.sh

@@ -207,7 +207,7 @@ function prepare_test_files() {
       cd ${BASE_DIR}/../examples/cpp/aot_inductor/bert/
       python aot_compile_export.py
       mv bert-seq.so ${HANDLER_DIR}/bert-seq.so
-      mv Transformer_model/tokenizer_config.json ${HANDLER_DIR}/tokenizer_config.json
+      mv Transformer_model/tokenizer.json ${HANDLER_DIR}/tokenizer.json
       export TOKENIZERS_PARALLELISM=""
     fi
   fi

examples/cpp/aot_inductor/bert/README.md

@@ -34,7 +34,7 @@ handler:
 ### Generate Model Artifact Folder
 
 ```bash
-torch-model-archiver --model-name bertcppaot --version 1.0 --handler ../../../../cpp/_build/test/resources/examples/aot_inductor/bert/libbert_handler:BertCppHandler --runtime LSP --extra-files index_to_name.json,../../../../cpp/_build/test/resources/examples/aot_inductor/bert_handler/bert-seq.so,../../../../cpp/_build/test/resources/examples/aot_inductor/bert_handler/tokenizer.json  --config-file model-config.yaml --archive-format no-archive
+torch-model-archiver --model-name bertcppaot --version 1.0 --handler ../../../../cpp/_build/test/resources/examples/aot_inductor/bert_handler/libbert_handler:BertCppHandler --runtime LSP --extra-files index_to_name.json,../../../../cpp/_build/test/resources/examples/aot_inductor/bert_handler/bert-seq.so,../../../../cpp/_build/test/resources/examples/aot_inductor/bert_handler/tokenizer.json  --config-file model-config.yaml --archive-format no-archive
 ```
 
 Create model store directory and move the folder `bertcppaot`
@@ -56,11 +56,5 @@ Infer the model using the following command
 
 ```
 curl http://localhost:8080/predictions/bertcppaot -T ../../../../cpp/test/resources/examples/aot_inductor/bert_handler/sample_text.txt
-{
-  "lens_cap": 0.0022578993812203407,
-  "lynx": 0.0032067005522549152,
-  "Egyptian_cat": 0.046274684369564056,
-  "tiger_cat": 0.13740436732769012,
-  "tabby": 0.2724998891353607
-}
+Not Accepted
 ```

examples/cpp/aot_inductor/bert/aot_compile_export.py

@@ -11,6 +11,7 @@
 )
 
 set_seed(1)
+MAX_BATCH_SIZE = 15
 
 
 def transformers_model_dowloader(
@@ -100,15 +101,14 @@ def transformers_model_dowloader(
     else:
         filename = os.path.join(dirname, "model-config.yaml")
     with open(filename, "r") as f:
-        settings = yaml.load(f)
+        settings = yaml.safe_load(f)
 
     mode = settings["handler"]["mode"]
     model_name = settings["handler"]["model_name"]
     num_labels = int(settings["handler"]["num_labels"])
-    do_lower_case = settings["handler"]["do_lower_case"]
+    do_lower_case = bool(settings["handler"]["do_lower_case"])
     max_length = int(settings["handler"]["max_length"])
-    batch_size = int(settings.get("batch_size", "1"))
-
+    batch_size = int(settings["batchSize"])
     transformers_model_dowloader(
         mode,
         model_name,

examples/cpp/aot_inductor/bert/src/bert_handler.cc

@@ -177,8 +177,6 @@ c10::IValue BertCppHandler::Inference(
     }
 
     auto batch_output_tensor_vector = runner->run(inputs.toTensorVector());
-    auto batch_output_tensor_vector = runner->run(vec);
-
     return c10::IValue(batch_output_tensor_vector[0]);
   } catch (std::runtime_error& e) {
     TS_LOG(ERROR, e.what());