Skip to content

Commit 3d17a94

Browse files
udaij12mreso
udaij12
and
mreso
authored
Adding model-control-mode (#3165)
* adding model-control-mode * newman test fixes * adding cmd line and fixing pytest * fixes to tests * fixing frontend tests * adding doc for model mode control * updating docs and moving check * adding test and fixing check * adding test * small changes * removing unused files * changing explicit to api enabled * updating tests * fixing test * changing config properties * changing enabled to true * Change test_util.start_torchserve interface --------- Co-authored-by: Matthias Reso <[email protected]>
1 parent 36049cb commit 3d17a94

27 files changed

+292
-19
lines changed

‎docs/model_control_mode.md

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Model Control Mode
2+
3+
TorchServe now supports model control mode with two settings "none"(default) and "enabled"
4+
5+
## Two ways to set Model Control
6+
1. Add `--model-api-enabled` to command line when running TorchServe to switch from none to enabled mode. Command line cannot be used to set mode to none, can only be used to set to enabled
7+
2. Add `model_api_enabled=false` or `model_api_enabled=true` to config.properties file
8+
* `model_api_enabled=false` is default and prevents users from registering or deleting models once TorchServe is running
9+
* `model_api_enabled=true` is not default and allows users to register and delete models using the TorchServe model load APIs
10+
11+
Priority between cmd and config file follows the following [TorchServer standard](https://github.com/pytorch/serve/blob/c74a29e8144bc12b84196775076b0e8cf3c5a6fc/docs/configuration.md#advanced-configuration)
12+
* Example 1:
13+
* Config file: `model_api_enabled=false`
14+
15+
cmd line: `torchserve --start --ncs --model-store model_store --model-api-enabled`
16+
17+
Result: Model api mode enabled
18+
* Example 2:
19+
* Config file: `model_api_enabled=true`
20+
21+
cmd line: `torchserve --start --ncs --model-store model_store`
22+
23+
Result: Mode is enabled (no way to disable api mode through cmd)
24+
25+
## Model Control Mode Default
26+
At startup TorchServe loads only those models specified explicitly with the `--models` command-line option. After startup users will be unable to register or delete models in this mode.
27+
28+
### Example default
29+
```
30+
ubuntu@ip-172-31-11-32:~/serve$ torchserve --start --ncs --model-store model_store --models resnet-18=resnet-18.mar --ts-config config.properties
31+
...
32+
ubuntu@ip-172-31-11-32:~/serve$ curl -X POST "http://localhost:8081/models?url=https://torchserve.pytorch.org/mar_files/squeezenet1_1.mar"
33+
2024-05-30T21:46:03,625 [INFO ] epollEventLoopGroup-3-2 ACCESS_LOG - /127.0.0.1:53514 "POST /models?url=https://torchserve.pytorch.org/mar_files/squeezenet1_1.mar HTTP/1.1" 405 0
34+
2024-05-30T21:46:03,626 [INFO ] epollEventLoopGroup-3-2 TS_METRICS - Requests4XX.Count:1.0|#Level:Host|#hostname:ip-172-31-11-32,timestamp:1717105563
35+
{
36+
"code": 405,
37+
"type": "MethodNotAllowedException",
38+
"message": "Requested method is not allowed, please refer to API document."
39+
}
40+
```
41+
42+
## Model Control API Enabled
43+
Setting model control to `enabled` allows users to load and unload models using the model load APIs.
44+
45+
### Example using cmd line to set mode to enabled
46+
```
47+
ubuntu@ip-172-31-11-32:~/serve$ torchserve --start --ncs --model-store model_store --models resnet-18=resnet-18.mar --ts-config config.properties --model-api-enabled
48+
49+
ubuntu@ip-172-31-11-32:~/serve$ curl -X POST "http://localhost:8081/models?url=https://torchserve.pytorch.org/mar_files/squeezenet1_1.mar"
50+
{
51+
"status": "Model \"squeezenet1_1\" Version: 1.0 registered with 0 initial workers. Use scale workers API to add workers for the model."
52+
}
53+
ubuntu@ip-172-31-11-32:~/serve$ curl http://localhost:8081/models
54+
2024-05-30T21:41:47,098 [INFO ] epollEventLoopGroup-3-2 ACCESS_LOG - /127.0.0.1:36270 "GET /models HTTP/1.1" 200 2
55+
2024-05-30T21:41:47,099 [INFO ] epollEventLoopGroup-3-2 TS_METRICS - Requests2XX.Count:1.0|#Level:Host|#hostname:ip-172-31-11-32,timestamp:1717105307
56+
{
57+
"models": [
58+
{
59+
"modelName": "resnet-18",
60+
"modelUrl": "resnet-18.mar"
61+
},
62+
{
63+
"modelName": "squeezenet1_1",
64+
"modelUrl": "https://torchserve.pytorch.org/mar_files/squeezenet1_1.mar"
65+
}
66+
]
67+
}
68+
ubuntu@ip-172-31-11-32:~/serve$ torchserve --stop
69+
TorchServe has stopped.
70+
```

‎frontend/server/src/main/java/org/pytorch/serve/http/api/rest/ManagementRequestHandler.java

+10-2
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.pytorch.serve.openapi.OpenApiUtils;
3333
import org.pytorch.serve.servingsdk.ModelServerEndpoint;
3434
import org.pytorch.serve.util.ApiUtils;
35+
import org.pytorch.serve.util.ConfigManager;
3536
import org.pytorch.serve.util.JsonUtils;
3637
import org.pytorch.serve.util.NettyUtils;
3738
import org.pytorch.serve.util.messages.RequestInput;
@@ -48,9 +49,12 @@
4849
*/
4950
public class ManagementRequestHandler extends HttpRequestHandlerChain {
5051

52+
private ConfigManager configManager;
53+
5154
/** Creates a new {@code ManagementRequestHandler} instance. */
5255
public ManagementRequestHandler(Map<String, ModelServerEndpoint> ep) {
5356
endpointMap = ep;
57+
configManager = ConfigManager.getInstance();
5458
}
5559

5660
@Override
@@ -74,7 +78,7 @@ public void handleRequest(
7478
if (HttpMethod.GET.equals(method)) {
7579
handleListModels(ctx, decoder);
7680
return;
77-
} else if (HttpMethod.POST.equals(method)) {
81+
} else if (HttpMethod.POST.equals(method) && isModeEnabled()) {
7882
handleRegisterModel(ctx, decoder, req);
7983
return;
8084
}
@@ -93,7 +97,7 @@ public void handleRequest(
9397
} else {
9498
handleScaleModel(ctx, decoder, segments[2], modelVersion);
9599
}
96-
} else if (HttpMethod.DELETE.equals(method)) {
100+
} else if (HttpMethod.DELETE.equals(method) && isModeEnabled()) {
97101
handleUnregisterModel(ctx, segments[2], modelVersion);
98102
} else if (HttpMethod.OPTIONS.equals(method)) {
99103
ModelManager modelManager = ModelManager.getInstance();
@@ -129,6 +133,10 @@ private boolean isManagementReq(String[] segments) {
129133
|| endpointMap.containsKey(segments[1]);
130134
}
131135

136+
private boolean isModeEnabled() {
137+
return configManager.getModelControlMode();
138+
}
139+
132140
private boolean isKFV1ManagementReq(String[] segments) {
133141
return segments.length == 4 && "v1".equals(segments[1]) && "models".equals(segments[2]);
134142
}

‎frontend/server/src/main/java/org/pytorch/serve/util/ConfigManager.java

+24-1
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ public final class ConfigManager {
133133
private static final String MODEL_CONFIG = "models";
134134
private static final String VERSION = "version";
135135
private static final String SYSTEM_METRICS_CMD = "system_metrics_cmd";
136+
private static final String MODEL_CONTROL_MODE = "model_api_enabled";
136137

137138
// Configuration default values
138139
private static final String DEFAULT_TS_ALLOWED_URLS = "file://.*|http(s)?://.*";
@@ -254,6 +255,10 @@ private ConfigManager(Arguments args) throws IOException {
254255
prop.setProperty(TS_LOAD_MODELS, String.join(",", models));
255256
}
256257

258+
if (args.isModelEnabled().equals("true")) {
259+
prop.setProperty(MODEL_CONTROL_MODE, args.isModelEnabled());
260+
}
261+
257262
prop.setProperty(
258263
TS_NUMBER_OF_GPU,
259264
String.valueOf(
@@ -478,6 +483,10 @@ public int getNumberOfGpu() {
478483
return getIntProperty(TS_NUMBER_OF_GPU, 0);
479484
}
480485

486+
public boolean getModelControlMode() {
487+
return Boolean.parseBoolean(getProperty(MODEL_CONTROL_MODE, "false"));
488+
}
489+
481490
public String getMetricsConfigPath() {
482491
String path = getCanonicalPath(prop.getProperty(TS_METRICS_CONFIG));
483492
if (path == null) {
@@ -806,7 +815,9 @@ public String dumpConfigurations() {
806815
+ "\nModel config: "
807816
+ prop.getProperty(MODEL_CONFIG, "N/A")
808817
+ "\nSystem metrics command: "
809-
+ (getSystemMetricsCmd().isEmpty() ? "default" : getSystemMetricsCmd());
818+
+ (getSystemMetricsCmd().isEmpty() ? "default" : getSystemMetricsCmd())
819+
+ "\nModel API enabled: "
820+
+ (getModelControlMode() ? "true" : "false");
810821
}
811822

812823
public boolean useNativeIo() {
@@ -1117,6 +1128,7 @@ public static final class Arguments {
11171128
private boolean snapshotDisabled;
11181129
private String workflowStore;
11191130
private String cppLogConfigFile;
1131+
private boolean modelApiEnabled;
11201132

11211133
public Arguments() {}
11221134

@@ -1128,6 +1140,7 @@ public Arguments(CommandLine cmd) {
11281140
snapshotDisabled = cmd.hasOption("no-config-snapshot");
11291141
workflowStore = cmd.getOptionValue("workflow-store");
11301142
cppLogConfigFile = cmd.getOptionValue("cpp-log-config");
1143+
modelApiEnabled = cmd.hasOption("model-api-enabled");
11311144
}
11321145

11331146
public static Options getOptions() {
@@ -1180,6 +1193,12 @@ public static Options getOptions() {
11801193
.argName("CPP-LOG-CONFIG")
11811194
.desc("log configuration file for cpp backend.")
11821195
.build());
1196+
options.addOption(
1197+
Option.builder("mapi")
1198+
.longOpt("model-api-enabled")
1199+
.argName("MODEL-API-ENABLED")
1200+
.desc("sets model apis to enabled")
1201+
.build());
11831202
return options;
11841203
}
11851204

@@ -1215,6 +1234,10 @@ public void setModels(String[] models) {
12151234
this.models = models.clone();
12161235
}
12171236

1237+
public String isModelEnabled() {
1238+
return String.valueOf(modelApiEnabled);
1239+
}
1240+
12181241
public boolean isSnapshotDisabled() {
12191242
return snapshotDisabled;
12201243
}

‎frontend/server/src/test/resources/config.properties

+1
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,4 @@ models={\
4646
# install_py_dep_per_model=false
4747
# enable_metrics_api=false
4848
workflow_store=../archive/src/test/resources/workflows
49+
model_api_enabled=true

‎frontend/server/src/test/resources/config_snapshot.properties

+1
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,4 @@ enable_envvars_config=true
3333
# enable_metrics_api=false
3434
workflow_store=../archive/src/test/resources/workflows
3535
metrics_config=src/test/resources/metrics_default.yaml
36+
model_api_enabled=true

‎frontend/server/src/test/resources/config_test_env.properties

+1
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,4 @@ models={\
5353
}\
5454
}
5555
metrics_config=src/test/resources/metrics_default.yaml
56+
model_api_enabled=true

‎frontend/server/src/test/resources/config_test_workflow.properties

+1
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,4 @@ models={\
5353
}\
5454
}
5555
metrics_config=src/test/resources/metrics_default.yaml
56+
model_api_enabled=true

‎frontend/server/src/test/resources/snapshots/snapshot1.cfg

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ certificate_file=src/test/resources/certs.pem
1818
management_address=https\://127.0.0.1\:8444
1919
metrics_address=https\://127.0.0.1\:8445
2020
workflow_store=../archive/src/test/resources/workflows
21-
metrics_config=src/test/resources/metrics_default.yaml
21+
metrics_config=src/test/resources/metrics_default.yaml
22+
model_api_enabled=true

‎frontend/server/src/test/resources/snapshots/snapshot2.cfg

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ certificate_file=src/test/resources/certs.pem
1818
management_address=https\://127.0.0.1\:8444
1919
metrics_address=https\://127.0.0.1\:8445
2020
workflow_store=../archive/src/test/resources/workflows
21-
metrics_config=src/test/resources/metrics_default.yaml
21+
metrics_config=src/test/resources/metrics_default.yaml
22+
model_api_enabled=true

‎frontend/server/src/test/resources/snapshots/snapshot3.cfg

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ certificate_file=src/test/resources/certs.pem
1818
management_address=https\://127.0.0.1\:8444
1919
metrics_address=https\://127.0.0.1\:8445
2020
workflow_store=../archive/src/test/resources/workflows
21-
metrics_config=src/test/resources/metrics_default.yaml
21+
metrics_config=src/test/resources/metrics_default.yaml
22+
model_api_enabled=true

‎frontend/server/src/test/resources/snapshots/snapshot4.cfg

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ certificate_file=src/test/resources/certs.pem
1818
management_address=https\://127.0.0.1\:8444
1919
metrics_address=https\://127.0.0.1\:8445
2020
workflow_store=../archive/src/test/resources/workflows
21-
metrics_config=src/test/resources/metrics_default.yaml
21+
metrics_config=src/test/resources/metrics_default.yaml
22+
model_api_enabled=true

‎frontend/server/src/test/resources/snapshots/snapshot5.cfg

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ certificate_file=src/test/resources/certs.pem
1818
management_address=https\://127.0.0.1\:8444
1919
metrics_address=https\://127.0.0.1\:8445
2020
workflow_store=../archive/src/test/resources/workflows
21-
metrics_config=src/test/resources/metrics_default.yaml
21+
metrics_config=src/test/resources/metrics_default.yaml
22+
model_api_enabled=true

‎frontend/server/src/test/resources/snapshots/snapshot6.cfg

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ certificate_file=src/test/resources/certs.pem
1818
management_address=https\://127.0.0.1\:8444
1919
metrics_address=https\://127.0.0.1\:8445
2020
workflow_store=../archive/src/test/resources/workflows
21-
metrics_config=src/test/resources/metrics_default.yaml
21+
metrics_config=src/test/resources/metrics_default.yaml
22+
model_api_enabled=true

‎frontend/server/src/test/resources/snapshots/snapshot7.cfg

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ certificate_file=src/test/resources/certs.pem
1818
management_address=https\://127.0.0.1\:8444
1919
metrics_address=https\://127.0.0.1\:8445
2020
workflow_store=../archive/src/test/resources/workflows
21-
metrics_config=src/test/resources/metrics_default.yaml
21+
metrics_config=src/test/resources/metrics_default.yaml
22+
model_api_enabled=true

‎frontend/server/src/test/resources/snapshots/snapshot8.cfg

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ certificate_file=src/test/resources/certs.pem
1818
management_address=https\://127.0.0.1\:8444
1919
metrics_address=https\://127.0.0.1\:8445
2020
workflow_store=../archive/src/test/resources/workflows
21-
metrics_config=src/test/resources/metrics_default.yaml
21+
metrics_config=src/test/resources/metrics_default.yaml
22+
model_api_enabled=true

‎frontend/server/src/test/resources/snapshots/snapshot9.cfg

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ certificate_file=src/test/resources/certs.pem
1818
management_address=https\://127.0.0.1\:8444
1919
metrics_address=https\://127.0.0.1\:8445
2020
workflow_store=../archive/src/test/resources/workflows
21-
metrics_config=src/test/resources/metrics_default.yaml
21+
metrics_config=src/test/resources/metrics_default.yaml
22+
model_api_enabled=true

0 commit comments

Comments
 (0)