Merge remote-tracking branch 'richardTingle/multi-scenario-screenshot-test' into fullscreen_triangle

Author: zzuegg

jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/Scenario.java

@@ -0,0 +1,13 @@
+package org.jmonkeyengine.screenshottests.testframework;
+
+import com.jme3.app.state.AppState;
+
+public class Scenario {
+    String scenarioName;
+    AppState[] states;
+
+    public Scenario(String scenarioName, AppState... states) {
+        this.scenarioName = scenarioName;
+        this.states = states;
+    }
+}

jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTest.java

@@ -47,7 +47,11 @@ public class ScreenshotTest{
 
     TestType testType = TestType.MUST_PASS;
 
-    AppState[] states;
+    /**
+     * Usually there will be a single scenario but sometimes it will be desirable to test that two ways
+     * of doing something produce the same result. In that case there will be multiple scenarios.
+     */
+    List<Scenario> scenarios = new ArrayList<>();
 
     List<Integer> framesToTakeScreenshotsOn = new ArrayList<>();
 
@@ -56,7 +60,11 @@ public class ScreenshotTest{
     String baseImageFileName = null;
 
     public ScreenshotTest(AppState... initialStates){
-        states = initialStates;
+        scenarios.add(new Scenario("SimpleSingleScenario", initialStates));
+        framesToTakeScreenshotsOn.add(1); //default behaviour is to take a screenshot on the first frame
+    }
+    public ScreenshotTest(Scenario... scenarios){
+        this.scenarios.addAll(Arrays.asList(scenarios));
         framesToTakeScreenshotsOn.add(1); //default behaviour is to take a screenshot on the first frame
     }
 
@@ -100,7 +108,7 @@ public void run(){
 
         String imageFilePrefix = baseImageFileName == null ? calculateImageFilePrefix() : baseImageFileName;
 
-        TestDriver.bootAppForTest(testType,settings,imageFilePrefix, framesToTakeScreenshotsOn, states);
+        TestDriver.bootAppForTest(testType,settings,imageFilePrefix, framesToTakeScreenshotsOn, scenarios);
     }
 
 

jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTestBase.java

@@ -47,10 +47,22 @@ public abstract class ScreenshotTestBase{
     /**
      * Initialises a screenshot test. The resulting object should be configured (if neccessary) and then started
      * by calling {@link ScreenshotTest#run()}.
-     * @param initialStates
+     * @param initialStates the states that will create the JME environment
      * @return
      */
     public ScreenshotTest screenshotTest(AppState... initialStates){
         return new ScreenshotTest(initialStates);
     }
+
+    /**
+     * Permits multiple scenarios to be tested in a single test. Each scenario should give identical results and
+     * will have a screenshot taken on the same frame.
+     *
+     * <p>
+     *     This is intended for testing migrations where the old and new approach should both give identical results.
+     * </p>
+     */
+    public ScreenshotTest screenshotMultiScenarioTest(Scenario... scenarios){
+        return new ScreenshotTest(scenarios);
+    }
 }

jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java

@@ -56,7 +56,9 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Executors;
@@ -78,7 +80,9 @@ public class TestDriver extends BaseAppState{
 
     private static final Logger logger = Logger.getLogger(TestDriver.class.getName());
 
-    public static final String IMAGES_ARE_DIFFERENT = "Images are different. (If you are running the test locally this is expected, images only reproducible on github CI infrastructure)";
+    public static final String IMAGES_ARE_DIFFERENT = "Generated images is different from committed image. (If you are running the test locally this is expected, images only reproducible on github CI infrastructure)";
+
+    public static final String IMAGES_ARE_DIFFERENT_BETWEEN_SCENARIOS = "Images are different between scenarios.";
 
     public static final String IMAGES_ARE_DIFFERENT_SIZES = "Images are different sizes.";
 
@@ -145,85 +149,127 @@ public void update(float tpf){
      * - After all the frames have been taken it stops the application
      * - Compares the screenshot to the expected screenshot (if any). Fails the test if they are different
      */
-    public static void bootAppForTest(TestType testType, AppSettings appSettings, String baseImageFileName, List<Integer> framesToTakeScreenshotsOn, AppState... initialStates){
-        FastMath.rand.setSeed(0); //try to make things deterministic by setting the random seed
+    public static void bootAppForTest(TestType testType, AppSettings appSettings, String baseImageFileName, List<Integer> framesToTakeScreenshotsOn, List<Scenario> scenarios){
+
         Collections.sort(framesToTakeScreenshotsOn);
 
-        Path imageTempDir;
+        List<Path> tempFolders = new ArrayList<>();
+        Map<Scenario, List<Path>> imageFilesPerScenario = new HashMap<>();
+
+        // usually there is a single scenario, but the framework can be set up to expect multiple scenarios that give identical results
+        for(Scenario scenario : scenarios) {
+            FastMath.rand.setSeed(0); //try to make things deterministic by setting the random seed
+            Path imageTempDir;
+            try {
+                imageTempDir = Files.createTempDirectory("jmeSnapshotTest");
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            tempFolders.add(imageTempDir);
 
-        try{
-            imageTempDir = Files.createTempDirectory("jmeSnapshotTest");
-        } catch(IOException e){
-            throw new RuntimeException(e);
-        }
+            ScreenshotNoInputAppState screenshotAppState = new ScreenshotNoInputAppState(imageTempDir.toString() + "/");
+            String screenshotAppFileNamePrefix = "Screenshot-";
+            screenshotAppState.setFileName(screenshotAppFileNamePrefix);
 
-        ScreenshotNoInputAppState screenshotAppState = new ScreenshotNoInputAppState(imageTempDir.toString() + "/");
-        String screenshotAppFileNamePrefix = "Screenshot-";
-        screenshotAppState.setFileName(screenshotAppFileNamePrefix);
+            List<AppState> states = new ArrayList<>(Arrays.asList(scenario.states));
+            TestDriver testDriver = new TestDriver(screenshotAppState, framesToTakeScreenshotsOn);
+            states.add(screenshotAppState);
+            states.add(testDriver);
 
-        List<AppState> states = new ArrayList<>(Arrays.asList(initialStates));
-        TestDriver testDriver = new TestDriver(screenshotAppState, framesToTakeScreenshotsOn);
-        states.add(screenshotAppState);
-        states.add(testDriver);
+            SimpleApplication app = new App(states.toArray(new AppState[0]));
+            app.setSettings(appSettings);
+            app.setShowSettings(false);
 
-        SimpleApplication app = new App(states.toArray(new AppState[0]));
-        app.setSettings(appSettings);
-        app.setShowSettings(false);
+            testDriver.waitLatch = new CountDownLatch(1);
+            executor.execute(() -> app.start(JmeContext.Type.Display));
 
-        testDriver.waitLatch = new CountDownLatch(1);
-        executor.execute(() -> app.start(JmeContext.Type.Display));
+            int maxWaitTimeMilliseconds = 45000;
 
-        int maxWaitTimeMilliseconds = 45000;
+            try {
+                boolean exitedProperly = testDriver.waitLatch.await(maxWaitTimeMilliseconds, TimeUnit.MILLISECONDS);
 
-        try {
-            boolean exitedProperly = testDriver.waitLatch.await(maxWaitTimeMilliseconds, TimeUnit.MILLISECONDS);
+                if (!exitedProperly) {
+                    logger.warning("Test driver did not exit in " + maxWaitTimeMilliseconds + "ms. Timed out");
+                    app.stop(true);
+                }
 
-            if(!exitedProperly){
-                logger.warning("Test driver did not exit in " + maxWaitTimeMilliseconds + "ms. Timed out");
-                app.stop(true);
+                Thread.sleep(1000); //give time for openGL is fully released before starting a new test (get random JVM crashes without this)
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException(e);
             }
 
-            Thread.sleep(1000); //give time for openGL is fully released before starting a new test (get random JVM crashes without this)
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            throw new RuntimeException(e);
-        }
+            //search the imageTempDir
+            List<Path> imageFiles = new ArrayList<>();
+            try (Stream<Path> paths = Files.list(imageTempDir)) {
+                paths.forEach(imageFiles::add);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
 
-        //search the imageTempDir
-        List<Path> imageFiles = new ArrayList<>();
-        try(Stream<Path> paths = Files.list(imageTempDir)){
-            paths.forEach(imageFiles::add);
-        } catch(IOException e){
-            throw new RuntimeException(e);
-        }
+            //this resorts with natural numeric ordering (so App10.png comes after App9.png)
+            imageFiles.sort(new Comparator<Path>() {
+                @Override
+                public int compare(Path p1, Path p2) {
+                    return extractNumber(p1).compareTo(extractNumber(p2));
+                }
 
-        //this resorts with natural numeric ordering (so App10.png comes after App9.png)
-        imageFiles.sort(new Comparator<Path>(){
-            @Override
-            public int compare(Path p1, Path p2){
-                return extractNumber(p1).compareTo(extractNumber(p2));
+                private Integer extractNumber(Path path) {
+                    String name = path.getFileName().toString();
+                    int numStart = screenshotAppFileNamePrefix.length();
+                    int numEnd = name.lastIndexOf(".png");
+                    return Integer.parseInt(name.substring(numStart, numEnd));
+                }
+            });
+            if (imageFiles.isEmpty()) {
+                fail("No screenshot found in the temporary directory. Did the application crash?");
             }
-
-            private Integer extractNumber(Path path){
-                String name = path.getFileName().toString();
-                int numStart = screenshotAppFileNamePrefix.length();
-                int numEnd = name.lastIndexOf(".png");
-                return Integer.parseInt(name.substring(numStart, numEnd));
+            if (imageFiles.size() != framesToTakeScreenshotsOn.size()) {
+                fail("Not all screenshots were taken, expected " + framesToTakeScreenshotsOn.size() + " but got " + imageFiles.size());
             }
-        });
 
-        if(imageFiles.isEmpty()){
-            fail("No screenshot found in the temporary directory. Did the application crash?");
+            imageFilesPerScenario.put(scenario, imageFiles);
         }
-        if(imageFiles.size() != framesToTakeScreenshotsOn.size()){
-            fail("Not all screenshots were taken, expected " + framesToTakeScreenshotsOn.size() + " but got " + imageFiles.size());
-        }
-
         String failureMessage = null;
 
         try {
+            List<Path> primeScenarioScreenshots = imageFilesPerScenario.get(scenarios.get(0));
+
+            if(imageFilesPerScenario.size()>1){
+                String primeScenarioName = scenarios.get(0).scenarioName;
+
+                // check each scenario gave the same results (before checking a single scenario against the reference images
+                for(int i=1;i<imageFilesPerScenario.size();i++){
+                    String thisScenarioName = scenarios.get(i).scenarioName;
+                    List<Path> otherScenarioScreenshots = imageFilesPerScenario.get(scenarios.get(i));
+                    for(int screenshotIndex=0;screenshotIndex<framesToTakeScreenshotsOn.size();screenshotIndex++) {
+                        Path primeImage = primeScenarioScreenshots.get(screenshotIndex);
+                        Path otherImage = otherScenarioScreenshots.get(screenshotIndex);
+
+                        BufferedImage img1 = ImageIO.read(primeImage.toFile());
+                        BufferedImage img2 = ImageIO.read(otherImage.toFile());
+
+                        int frame = framesToTakeScreenshotsOn.get(screenshotIndex);
+
+                        String thisFrameBaseImageFileName = baseImageFileName + "_f" + frame;
+
+                        if (!imagesAreTheSame(img1, img2)) {
+                            attachImage("Scenario " + primeScenarioName + " " + screenshotIndex, thisFrameBaseImageFileName + "_" + primeScenarioName + ".png", img1);
+                            attachImage("Scenario " + thisScenarioName + " " + screenshotIndex, thisFrameBaseImageFileName + "_" + thisScenarioName + ".png", img2);
+                            attachImage("Diff (between above scenarios)", thisFrameBaseImageFileName + "_" + primeScenarioName + "_" + thisScenarioName + "_diff.png", createComparisonImage(img1, img2));
+
+                            if(failureMessage==null){ //only want the first thing to go wrong as the junit test fail reason
+                                failureMessage = IMAGES_ARE_DIFFERENT_BETWEEN_SCENARIOS;
+                            }
+                            ExtentReportExtension.getCurrentTest().fail(IMAGES_ARE_DIFFERENT_BETWEEN_SCENARIOS);
+                        }
+                    }
+                }
+            }
+
+
             for(int screenshotIndex=0;screenshotIndex<framesToTakeScreenshotsOn.size();screenshotIndex++){
-                Path generatedImage = imageFiles.get(screenshotIndex);
+                Path generatedImage = primeScenarioScreenshots.get(screenshotIndex);
                 int frame = framesToTakeScreenshotsOn.get(screenshotIndex);
 
                 String thisFrameBaseImageFileName = baseImageFileName + "_f" + frame;
@@ -280,7 +326,9 @@ private Integer extractNumber(Path path){
         } catch (IOException e) {
             throw new RuntimeException("Error reading images", e);
         } finally{
-            clearTemporaryFolder(imageTempDir);
+            for(Path imageTempDir : tempFolders){
+                clearTemporaryFolder(imageTempDir);
+            }
         }
 
         if(failureMessage!=null){