diff --git a/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/tensorflow/v2/TestDatasets.java b/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/tensorflow/v2/TestDatasets.java index 384d913f1..96e6d72b6 100644 --- a/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/tensorflow/v2/TestDatasets.java +++ b/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/tensorflow/v2/TestDatasets.java @@ -919,13 +919,10 @@ public void testDatasetFromGeneratorDeclaredShapes2() /** * The {@code tf.TensorShape} literal form of {@link #testDatasetFromGeneratorDeclaredShapes()}: - * the declared shape is wrapped in a {@code TensorShape} constructor rather than a plain tuple. - * {@code tf.TensorShape} is unmodeled, so the argument is present but unparseable and the shape - * axis soundly degrades to ⊤ ({@code {? of int32}}) instead of composing {@code (2,)}. - * - *

TODO: Flip to expect {@code TensorType.of(INT_32, 2)} when wala/ML#789 parses {@code TensorShape} - * constructor literals in shape arguments. + * the declared shape is wrapped in a {@code TensorShape} constructor rather than a plain tuple, + * whose stored {@code dims} the shape-argument parser recurses into (wala/ML#789), composing {@code (2,)} exactly + * like the tuple forms. * * @throws ClassHierarchyException On WALA class-hierarchy error. * @throws IllegalArgumentException On illegal argument. @@ -940,7 +937,7 @@ public void testDatasetFromGeneratorDeclaredShapes3() "consume", 1, 1, - Map.of(2, Set.of(new TensorType(INT_32, null)))); + Map.of(2, Set.of(TensorType.of(INT_32, 2)))); } /** diff --git a/com.ibm.wala.cast.python.ml/data/tensorflow.xml b/com.ibm.wala.cast.python.ml/data/tensorflow.xml index ae8ba33dc..486a58095 100644 --- a/com.ibm.wala.cast.python.ml/data/tensorflow.xml +++ b/com.ibm.wala.cast.python.ml/data/tensorflow.xml @@ -197,6 +197,8 @@ + + @@ -1105,6 +1107,14 @@ + + + + + + + + diff --git a/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/client/TensorGenerator.java b/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/client/TensorGenerator.java index 995e6e1d7..14e329822 100644 --- a/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/client/TensorGenerator.java +++ b/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/client/TensorGenerator.java @@ -619,7 +619,8 @@ protected Set>> getShapesFromShapeArgument( if (innerReference.equals(tuple) || innerReference.equals(list) || innerReference.equals(TensorFlowTypes.TENSOR_SPEC) - || innerReference.equals(TensorFlowTypes.RAGGED_TENSOR_SPEC)) { + || innerReference.equals(TensorFlowTypes.RAGGED_TENSOR_SPEC) + || innerReference.equals(TensorFlowTypes.TENSOR_SHAPE)) { // Nested tuple/list or Spec. Recurse. Set>> nestedShapes = this.getShapesFromShapeArgument( @@ -782,17 +783,21 @@ protected Set>> getShapesFromShapeArgument( if (constantShapes == null) return null; ret.addAll(constantShapes); } else if (reference.equals(TensorFlowTypes.TENSOR_SPEC) - || reference.equals(TensorFlowTypes.RAGGED_TENSOR_SPEC)) { - // We have a TensorSpec or RaggedTensorSpec. These objects carry shape and dtype - // information in their fields. We extract the 'shape' field and recurse to - // parse the actual shape structure (usually a tuple or list of integers). + || reference.equals(TensorFlowTypes.RAGGED_TENSOR_SPEC) + || reference.equals(TensorFlowTypes.TENSOR_SHAPE)) { + // We have a TensorSpec, RaggedTensorSpec, or TensorShape. These objects carry their shape + // structure in a field ('shape' for the specs, the stored 'dims' argument for a + // TensorShape constructor, wala/ML#789); extract it and recurse to parse the actual + // structure (usually a tuple or list of integers). IField shapeField = builder .getClassHierarchy() .resolveField( reference.equals(TensorFlowTypes.TENSOR_SPEC) ? TensorFlowTypes.SPEC_SHAPE - : TensorFlowTypes.RAGGED_SPEC_SHAPE); + : reference.equals(TensorFlowTypes.RAGGED_TENSOR_SPEC) + ? TensorFlowTypes.RAGGED_SPEC_SHAPE + : TensorFlowTypes.TENSOR_SHAPE_DIMS); PointerKey shapePK = builder.getPointerKeyForInstanceField(instanceKey, shapeField); OrdinalSet shapePts = pointerAnalysis.getPointsToSet(shapePK); if (shapePts == null || shapePts.isEmpty()) return null; diff --git a/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/types/TensorFlowTypes.java b/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/types/TensorFlowTypes.java index e5fdba49b..bee347f74 100644 --- a/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/types/TensorFlowTypes.java +++ b/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/types/TensorFlowTypes.java @@ -329,6 +329,15 @@ public boolean canConvertTo(DType other) { public static final String RAGGED_TENSOR_SPEC_SIGNATURE = "tf.RaggedTensorSpec()"; + /** https://www.tensorflow.org/api_docs/python/tf/TensorShape. */ + public static final TypeReference TENSOR_SHAPE = + TypeReference.findOrCreate( + pythonLoader, TypeName.findOrCreate("Ltensorflow/framework/TensorShape")); + + /** The {@code dims} argument the {@code TensorShape} constructor stores (wala/ML#789). */ + public static final FieldReference TENSOR_SHAPE_DIMS = + FieldReference.findOrCreate(TENSOR_SHAPE, findOrCreateAsciiAtom("dims"), Root); + public static final FieldReference SPEC_SHAPE = FieldReference.findOrCreate(TENSOR_SPEC, findOrCreateAsciiAtom("shape"), Root);