[BUGFIX] Fixing shape as string in manifest

mael-le-gal · mael-le-gal · commit 02ad8ac2562e · 2020-05-28T17:14:32.000+02:00
diff --git a/commons/src/main/java/com/ovh/mls/serving/runtime/core/tensor/TensorField.java b/commons/src/main/java/com/ovh/mls/serving/runtime/core/tensor/TensorField.java
@@ -57,8 +57,8 @@ public TensorShape getTensorShape() {
         return new TensorShape(shape);
     }
 
-    public void setShape(String shape) {
-        this.shape = new TensorShape(shape).getArrayShape();
+    public void setShape(int[] shape) {
+        this.shape = shape;
     }
 
     public List<TensorIndex> getFields() {
diff --git a/commons/src/main/java/com/ovh/mls/serving/runtime/core/tensor/TensorShape.java b/commons/src/main/java/com/ovh/mls/serving/runtime/core/tensor/TensorShape.java
@@ -12,41 +12,41 @@ public TensorShape(int[] shape) {
         this.shape = shape;
     }
 
-    public TensorShape(String shape) {
-        this.shape = getShapeFromString(shape);
-    }
+//    public TensorShape(String shape) {
+//        this.shape = getShapeFromString(shape);
+//    }
 
     public TensorShape(long[] shape) {
         this.shape = Arrays.stream(shape)
             .mapToInt(x -> (int) x)
             .toArray();
     }
 
-    /**
-     * Extract a shape from a string like (?, 2, 2)
-     */
-    private static int[] getShapeFromString(String stringShape) {
-        String removeParenthesis = stringShape.substring(1, stringShape.length() - 1);
-        String[] split = removeParenthesis.split(",");
-
-        if (removeParenthesis.isEmpty()) {
-            return new int[0];
-        }
-
-        Integer[] result = Arrays
-            .stream(split)
-            .map(String::trim)
-            .map(x -> {
-                if ("?".equals(x)) {
-                    return -1;
-                } else {
-                    return Integer.parseInt(x);
-                }
-            })
-            .toArray(Integer[]::new);
-
-        return ArrayUtils.toPrimitive(result);
-    }
+//    /**
+//     * Extract a shape from a string like (?, 2, 2)
+//     */
+//    private static int[] getShapeFromString(String stringShape) {
+//        String removeParenthesis = stringShape.substring(1, stringShape.length() - 1);
+//        String[] split = removeParenthesis.split(",");
+//
+//        if (removeParenthesis.isEmpty()) {
+//            return new int[0];
+//        }
+//
+//        Integer[] result = Arrays
+//            .stream(split)
+//            .map(String::trim)
+//            .map(x -> {
+//                if ("?".equals(x)) {
+//                    return -1;
+//                } else {
+//                    return Integer.parseInt(x);
+//                }
+//            })
+//            .toArray(Integer[]::new);
+//
+//        return ArrayUtils.toPrimitive(result);
+//    }
 
     public boolean handleBatch() {
         return shape.length != 0 && shape[0] == -1;
diff --git a/evaluator-onnx/src/test/resources/onnx/iris/manifest-same-output.json b/evaluator-onnx/src/test/resources/onnx/iris/manifest-same-output.json
@@ -4,7 +4,7 @@
   "inputs": [
     {
       "name": "float_input",
-      "shape": "(?, 4)",
+      "shape": [-1, 4],
       "type": "float",
       "fields": [
         {
@@ -29,7 +29,7 @@
   "outputs": [
     {
       "name": "output_label",
-      "shape": "(?)",
+      "shape": [-1],
       "type": "long",
       "fields": [
         {
@@ -40,7 +40,7 @@
     },
     {
       "name": "output_probability",
-      "shape": "(?, 3)",
+      "shape": [-1, 3],
       "type": "float",
       "fields": [
         {
diff --git a/evaluator-onnx/src/test/resources/onnx/iris/manifest.json b/evaluator-onnx/src/test/resources/onnx/iris/manifest.json
@@ -4,7 +4,7 @@
   "inputs": [
     {
       "name": "float_input",
-      "shape": "(?, 4)",
+      "shape": [-1, 4],
       "type": "float",
       "fields": [
         {
@@ -29,7 +29,7 @@
   "outputs": [
     {
       "name": "output_label",
-      "shape": "(?)",
+      "shape": [-1],
       "type": "long",
       "fields": [
         {
@@ -40,7 +40,7 @@
     },
     {
       "name": "output_probability",
-      "shape": "(?, 3)",
+      "shape": [-1, 3],
       "type": "float",
       "fields": [
         {
diff --git a/evaluator-onnx/src/test/resources/onnx/titanic/manifest.json b/evaluator-onnx/src/test/resources/onnx/titanic/manifest.json
@@ -4,7 +4,7 @@
   "inputs": [
     {
       "name": "pclass",
-      "shape": "(?, 1)",
+      "shape": [-1, 1],
       "type": "string",
       "fields": [
         {
@@ -15,7 +15,7 @@
     },
     {
       "name": "sex",
-      "shape": "(?, 1)",
+      "shape": [-1, 1],
       "type": "string",
       "fields": [
         {
@@ -26,7 +26,7 @@
     },
     {
       "name": "age",
-      "shape": "(?, 1)",
+      "shape": [-1, 1],
       "type": "float",
       "fields": [
         {
@@ -37,7 +37,7 @@
     },
     {
       "name": "fare",
-      "shape": "(?, 1)",
+      "shape": [-1, 1],
       "type": "float",
       "fields": [
         {
@@ -48,7 +48,7 @@
     },
     {
       "name": "embarked",
-      "shape": "(?, 1)",
+      "shape": [-1, 1],
       "type": "string",
       "fields": [
         {
@@ -61,7 +61,7 @@
   "outputs": [
     {
       "name": "output_label",
-      "shape": "(?)",
+      "shape": [-1],
       "type": "long",
       "fields": [
         {
@@ -71,7 +71,7 @@
     },
     {
       "name": "output_probability",
-      "shape": "(?, 2)",
+      "shape": [-1, 2],
       "type": "float",
       "fields": [
         {
diff --git a/evaluator-tensorflow/src/test/resources/tensorflow/2d_savedmodel/2d-tensorflow-model-manifest-same-output.json b/evaluator-tensorflow/src/test/resources/tensorflow/2d_savedmodel/2d-tensorflow-model-manifest-same-output.json
@@ -4,7 +4,7 @@
   "inputs": [
     {
       "name": "input",
-      "shape": "(?, 8)",
+      "shape": [-1, 8],
       "type": "float",
       "fields": [
         {
@@ -29,7 +29,7 @@
   "outputs": [
     {
       "name": "output",
-      "shape": "(?, 1)",
+      "shape": [-1, 1],
       "type": "float",
       "fields": [
         {
diff --git a/evaluator-tensorflow/src/test/resources/tensorflow/2d_savedmodel/2d-tensorflow-model-manifest.json b/evaluator-tensorflow/src/test/resources/tensorflow/2d_savedmodel/2d-tensorflow-model-manifest.json
@@ -4,7 +4,7 @@
   "inputs": [
     {
       "name": "input",
-      "shape": "(?, 8)",
+      "shape": [-1, 8],
       "type": "float",
       "fields": [
         {
@@ -29,7 +29,7 @@
   "outputs": [
     {
       "name": "output",
-      "shape": "(?, 1)",
+      "shape": [-1, 1],
       "type": "float",
       "fields": [
         {
diff --git a/evaluator-tensorflow/src/test/resources/tensorflow/3d_savedmodel/3d-tensorflow-model-manifest.json b/evaluator-tensorflow/src/test/resources/tensorflow/3d_savedmodel/3d-tensorflow-model-manifest.json
@@ -4,7 +4,7 @@
   "inputs": [
     {
       "name": "input",
-      "shape": "(?, 2, 4)",
+      "shape": [-1, 2, 4],
       "type": "float",
       "fields": [
         {
@@ -29,7 +29,7 @@
   "outputs": [
     {
       "name": "output",
-      "shape": "(?, 1)",
+      "shape": [-1, 1],
       "type": "float",
       "fields": [
         {
diff --git a/evaluator-timeseries/src/test/resources/timeseries/datetime-string-manifest.json b/evaluator-timeseries/src/test/resources/timeseries/datetime-string-manifest.json
@@ -3,69 +3,69 @@
     {
       "name": "date",
       "type": "string",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "yyyy-MM-dd HH:mm:ss"
     }
   ],
   "outputs": [
     {
       "name": "year",
       "type": "integer",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "YEAR"
     },
     {
       "name": "month",
       "type": "integer",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "MONTH"
     },
     {
       "name": "dayofmonth",
       "type": "integer",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "DAYOFMONTH"
     },
     {
       "name": "hourofday",
       "type": "integer",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "HOUROFDAY"
     },
     {
       "name": "minuteofhour",
       "type": "integer",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "MINUTEOFHOUR"
     },
     {
       "name": "dayofweek",
       "type": "integer",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "DAYOFWEEK"
     },
     {
       "name": "year-string",
       "type": "string",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "yyyy"
     },
     {
       "name": "year-float",
       "type": "float",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "YEAR"
     },
     {
       "name": "timestamp-seconds",
       "type": "long",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "TS_S"
     },
     {
       "name": "timestamp-years",
       "type": "long",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "TS_Y"
     }
   ]
diff --git a/evaluator-timeseries/src/test/resources/timeseries/datetime-timestamp-manifest.json b/evaluator-timeseries/src/test/resources/timeseries/datetime-timestamp-manifest.json
@@ -3,15 +3,15 @@
     {
       "name": "timestamp-seconds",
       "type": "long",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "TS_S"
     }
   ],
   "outputs": [
     {
       "name": "date",
       "type": "string",
-      "shape": "(-1, -1)",
+      "shape": [-1, -1],
       "format": "yyyy-MM-dd HH:mm:ss"
     }
   ]

Original file line number	Diff line number	Diff line change
`@@ -57,8 +57,8 @@ public TensorShape getTensorShape() {`
`57`	`57`	`return new TensorShape(shape);`
`58`	`58`	`}`
`59`	`59`
`60`		`- public void setShape(String shape) {`
`61`		`- this.shape = new TensorShape(shape).getArrayShape();`
	`60`	`+ public void setShape(int[] shape) {`
	`61`	`+ this.shape = shape;`
`62`	`62`	`}`
`63`	`63`
`64`	`64`	`public List<TensorIndex> getFields() {`
Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@`
`4`	`4`	`"inputs": [`
`5`	`5`	`{`
`6`	`6`	`"name": "float_input",`
`7`		`- "shape": "(?, 4)",`
	`7`	`+ "shape": [-1, 4],`
`8`	`8`	`"type": "float",`
`9`	`9`	`"fields": [`
`10`	`10`	`{`
`@@ -29,7 +29,7 @@`
`29`	`29`	`"outputs": [`
`30`	`30`	`{`
`31`	`31`	`"name": "output_label",`
`32`		`- "shape": "(?)",`
	`32`	`+ "shape": [-1],`
`33`	`33`	`"type": "long",`
`34`	`34`	`"fields": [`
`35`	`35`	`{`
`@@ -40,7 +40,7 @@`
`40`	`40`	`},`
`41`	`41`	`{`
`42`	`42`	`"name": "output_probability",`
`43`		`- "shape": "(?, 3)",`
	`43`	`+ "shape": [-1, 3],`
`44`	`44`	`"type": "float",`
`45`	`45`	`"fields": [`
`46`	`46`	`{`
Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@`
`4`	`4`	`"inputs": [`
`5`	`5`	`{`
`6`	`6`	`"name": "pclass",`
`7`		`- "shape": "(?, 1)",`
	`7`	`+ "shape": [-1, 1],`
`8`	`8`	`"type": "string",`
`9`	`9`	`"fields": [`
`10`	`10`	`{`
`@@ -15,7 +15,7 @@`
`15`	`15`	`},`
`16`	`16`	`{`
`17`	`17`	`"name": "sex",`
`18`		`- "shape": "(?, 1)",`
	`18`	`+ "shape": [-1, 1],`
`19`	`19`	`"type": "string",`
`20`	`20`	`"fields": [`
`21`	`21`	`{`
`@@ -26,7 +26,7 @@`
`26`	`26`	`},`
`27`	`27`	`{`
`28`	`28`	`"name": "age",`
`29`		`- "shape": "(?, 1)",`
	`29`	`+ "shape": [-1, 1],`
`30`	`30`	`"type": "float",`
`31`	`31`	`"fields": [`
`32`	`32`	`{`
`@@ -37,7 +37,7 @@`
`37`	`37`	`},`
`38`	`38`	`{`
`39`	`39`	`"name": "fare",`
`40`		`- "shape": "(?, 1)",`
	`40`	`+ "shape": [-1, 1],`
`41`	`41`	`"type": "float",`
`42`	`42`	`"fields": [`
`43`	`43`	`{`
`@@ -48,7 +48,7 @@`
`48`	`48`	`},`
`49`	`49`	`{`
`50`	`50`	`"name": "embarked",`
`51`		`- "shape": "(?, 1)",`
	`51`	`+ "shape": [-1, 1],`
`52`	`52`	`"type": "string",`
`53`	`53`	`"fields": [`
`54`	`54`	`{`
`@@ -61,7 +61,7 @@`
`61`	`61`	`"outputs": [`
`62`	`62`	`{`
`63`	`63`	`"name": "output_label",`
`64`		`- "shape": "(?)",`
	`64`	`+ "shape": [-1],`
`65`	`65`	`"type": "long",`
`66`	`66`	`"fields": [`
`67`	`67`	`{`
`@@ -71,7 +71,7 @@`
`71`	`71`	`},`
`72`	`72`	`{`
`73`	`73`	`"name": "output_probability",`
`74`		`- "shape": "(?, 2)",`
	`74`	`+ "shape": [-1, 2],`
`75`	`75`	`"type": "float",`
`76`	`76`	`"fields": [`
`77`	`77`	`{`
Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@`
`4`	`4`	`"inputs": [`
`5`	`5`	`{`
`6`	`6`	`"name": "input",`
`7`		`- "shape": "(?, 8)",`
	`7`	`+ "shape": [-1, 8],`
`8`	`8`	`"type": "float",`
`9`	`9`	`"fields": [`
`10`	`10`	`{`
`@@ -29,7 +29,7 @@`
`29`	`29`	`"outputs": [`
`30`	`30`	`{`
`31`	`31`	`"name": "output",`
`32`		`- "shape": "(?, 1)",`
	`32`	`+ "shape": [-1, 1],`
`33`	`33`	`"type": "float",`
`34`	`34`	`"fields": [`
`35`	`35`	`{`
Original file line number	Diff line number	Diff line change
`@@ -3,15 +3,15 @@`
`3`	`3`	`{`
`4`	`4`	`"name": "timestamp-seconds",`
`5`	`5`	`"type": "long",`
`6`		`- "shape": "(-1, -1)",`
	`6`	`+ "shape": [-1, -1],`
`7`	`7`	`"format": "TS_S"`
`8`	`8`	`}`
`9`	`9`	`],`
`10`	`10`	`"outputs": [`
`11`	`11`	`{`
`12`	`12`	`"name": "date",`
`13`	`13`	`"type": "string",`
`14`		`- "shape": "(-1, -1)",`
	`14`	`+ "shape": [-1, -1],`
`15`	`15`	`"format": "yyyy-MM-dd HH:mm:ss"`
`16`	`16`	`}`
`17`	`17`	`]`