[tests] Testing the diversty of generated mazes.

Testing the distribution of random mazes for two different mixer seeds, in the process validating the amount of overlaps remains below a 1% threshold.

Author: DeepMind Lab Team

deepmind/engine/BUILD

@@ -200,7 +200,7 @@ cc_library(
 
 cc_test(
     name = "lua_maze_generation_test",
-    size = "small",
+    size = "large",
     srcs = ["lua_maze_generation_test.cc"],
     tags = ["manual"],  # Platform specific tests.
     deps = [
@@ -210,6 +210,7 @@ cc_test(
         "//deepmind/lua:call",
         "//deepmind/lua:n_results_or_test_util",
         "//deepmind/lua:push_script",
+        "//deepmind/lua:table_ref",
         "//deepmind/lua:vm_test_util",
         "@com_google_googletest//:gtest_main",
     ],

deepmind/engine/lua_maze_generation_test.cc

@@ -29,6 +29,7 @@
 #include "deepmind/lua/call.h"
 #include "deepmind/lua/n_results_or_test_util.h"
 #include "deepmind/lua/push_script.h"
+#include "deepmind/lua/table_ref.h"
 #include "deepmind/lua/vm_test_util.h"
 
 namespace deepmind {
@@ -45,13 +46,19 @@ class LuaMazeGenerationTest : public lua::testing::TestWithVm {
     vm()->AddCModuleToSearchers(
         "dmlab.system.maze_generation", &lua::Bind<LuaMazeGeneration::Require>,
         {reinterpret_cast<void*>(static_cast<std::uintptr_t>(0))});
+    vm()->AddCModuleToSearchers(
+        "dmlab.system.maze_generation1", &lua::Bind<LuaMazeGeneration::Require>,
+        {reinterpret_cast<void*>(static_cast<std::uintptr_t>(1))});
     LuaRandom::Register(L);
     vm()->AddCModuleToSearchers(
         "dmlab.system.sys_random", &lua::Bind<LuaRandom::Require>,
-        {&prbg_, reinterpret_cast<void*>(static_cast<std::uintptr_t>(0))});
+        {&prbg_[0], reinterpret_cast<void*>(static_cast<std::uintptr_t>(0))});
+    vm()->AddCModuleToSearchers(
+        "dmlab.system.sys_random1", &lua::Bind<LuaRandom::Require>,
+        {&prbg_[1], reinterpret_cast<void*>(static_cast<std::uintptr_t>(1))});
   }
 
-  std::mt19937_64 prbg_;
+  std::mt19937_64 prbg_[2];
 };
 
 constexpr char kCreateMaze[] = R"(
@@ -468,6 +475,65 @@ TEST_F(LuaMazeGenerationTest, CreateRandomMazeNoRandom) {
   ASSERT_FALSE(lua::Call(L, 0).ok());
 }
 
+constexpr int kRandomMazeSeedCount = 10000;
+constexpr char kCreateRandomMazes[] = R"(
+local seed, maps, vers = ...
+local sys_random = require('dmlab.system.sys_random' .. vers)
+local maze_generation = require('dmlab.system.maze_generation' .. vers)
+sys_random:seed(seed)
+local maze = maze_generation.randomMazeGeneration{
+      random = sys_random,
+      width = 17,
+      height = 17,
+      extraConnectionProbability = 0.0,
+      hasDoors = false,
+      roomCount = 4,
+      maxRooms = 4,
+      roomMaxSize = 5,
+      roomMinSize = 3
+}
+local key = maze:entityLayer()
+local count = maps[key] or 0
+maps[key] = count + 1
+)";
+
+TEST_F(LuaMazeGenerationTest, CreateRandomMazes) {
+  auto maps = lua::TableRef::Create(L);
+  lua::PushScript(L, kCreateRandomMazes, sizeof(kCreateRandomMazes) - 1,
+                  "kCreateRandomMazes");
+  for (int i = 0; i < kRandomMazeSeedCount; ++i) {
+    LOG(INFO) << "Phase 1: step " << i + 1 << " out of "
+              << kRandomMazeSeedCount;
+    lua_pushvalue(L, -1);
+    lua::Push(L, i);
+    lua::Push(L, maps);
+    lua::Push(L, "");
+    ASSERT_THAT(lua::Call(L, 3), IsOkAndHolds(0));
+  }
+  const auto mixer_seed_0_maps = maps.KeyCount();
+  const auto mixer_seed_0_repeat_ratio =
+      (kRandomMazeSeedCount - mixer_seed_0_maps) /
+      static_cast<float>(kRandomMazeSeedCount);
+  ASSERT_GE(mixer_seed_0_repeat_ratio, 0.0);
+  ASSERT_LE(mixer_seed_0_repeat_ratio, 0.01);
+  for (int i = 0; i < kRandomMazeSeedCount; ++i) {
+    LOG(INFO) << "Phase 2: step " << i + 1 << " out of "
+              << kRandomMazeSeedCount;
+    lua_pushvalue(L, -1);
+    lua::Push(L, i);
+    lua::Push(L, maps);
+    lua::Push(L, "1");
+    ASSERT_THAT(lua::Call(L, 3), IsOkAndHolds(0));
+  }
+  const auto mixer_seed_1_maps = maps.KeyCount() - mixer_seed_0_maps;
+  const auto mixer_seed_1_repeat_ratio =
+      (kRandomMazeSeedCount - mixer_seed_1_maps) /
+      static_cast<float>(kRandomMazeSeedCount);
+  ASSERT_GE(mixer_seed_1_repeat_ratio, 0.0);
+  ASSERT_LE(mixer_seed_1_repeat_ratio, 0.01);
+  lua_pop(L, 1);
+}
+
 constexpr char kVisitRandomPath[] = R"(
 local sys_random = require 'dmlab.system.sys_random'
 local maze_generation = require 'dmlab.system.maze_generation'

game_scripts/levels/tests/maze_generation_test.lua

@@ -0,0 +1,57 @@
+--[[ Copyright (C) 2018 Google Inc.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+]]
+
+local custom_observations = require 'decorators.custom_observations'
+local debug_observations = require 'decorators.debug_observations'
+local make_map = require 'common.make_map'
+local map_maker = require 'dmlab.system.map_maker'
+local maze_generation = require 'dmlab.system.maze_generation'
+local random = require 'common.random'
+
+local randomMap = random(map_maker:randomGen())
+
+local api = {}
+
+function api:nextMap()
+  return api._map
+end
+
+function api:start(episode, seed)
+  local mapName = 'maze'
+  random:seed(seed)
+  randomMap:seed(seed)
+  api._maze = maze_generation.randomMazeGeneration{
+      seed = seed,
+      width = 17,
+      height = 17,
+      maxRooms = 4,
+      roomMaxSize = 5,
+      roomMinSize = 3,
+      extraConnectionProbability = 0.0,
+  }
+  api._map = make_map.makeMap{
+      mapName = mapName,
+      mapEntityLayer = api._maze:entityLayer(),
+      mapVariationsLayer = api._maze:variationsLayer(),
+      useSkybox = true,
+  }
+  debug_observations.setMaze(api._maze)
+end
+
+custom_observations.decorate(api)
+
+return api

python/tests/BUILD

@@ -139,3 +139,12 @@ py_test(
     data = ["//:deepmind_lab.so"],
     imports = ["python"],
 )
+
+py_test(
+    name = "maze_generation_test",
+    size = "enormous",
+    srcs = ["maze_generation_test.py"],
+    tags = ["manual"],
+    data = ["//:deepmind_lab.so"],
+    imports = ["python"],
+)

python/tests/maze_generation_test.py

@@ -0,0 +1,68 @@
+# Copyright 2018 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import unittest
+import numpy as np
+
+import deepmind_lab
+
+MAZE_LAYOUT_OBSERVATION = 'DEBUG.MAZE.LAYOUT'
+MAZE_LAYOUT_TRIALS = 50
+
+
+class MazeGenerationTest(unittest.TestCase):
+
+  def test_maze_layout_spread(self):
+    layouts = set()
+    for i in xrange(MAZE_LAYOUT_TRIALS):
+      print('phase 1: trial {} out of {}'.format(i+1, MAZE_LAYOUT_TRIALS))
+      env = deepmind_lab.Lab(
+          'tests/maze_generation_test', [MAZE_LAYOUT_OBSERVATION], config={})
+      env.reset(seed=i+1)
+      layouts.add(env.observations()[MAZE_LAYOUT_OBSERVATION])
+    num_layouts = len(layouts)
+    self.assertTrue(np.isclose(num_layouts, MAZE_LAYOUT_TRIALS))
+    for i in xrange(MAZE_LAYOUT_TRIALS):
+      print('phase 2: trial {} out of {}'.format(i+1, MAZE_LAYOUT_TRIALS))
+      env = deepmind_lab.Lab(
+          'tests/maze_generation_test', [MAZE_LAYOUT_OBSERVATION],
+          config={
+              'mixerSeed': '0',
+          })
+      env.reset(seed=i+1)
+      layouts.add(env.observations()[MAZE_LAYOUT_OBSERVATION])
+    self.assertEqual(len(layouts), num_layouts)
+    for i in xrange(MAZE_LAYOUT_TRIALS):
+      print('phase 3: trial {} out of {}'.format(i+1, MAZE_LAYOUT_TRIALS))
+      env = deepmind_lab.Lab(
+          'tests/maze_generation_test', [MAZE_LAYOUT_OBSERVATION],
+          config={
+              'mixerSeed': '1',
+          })
+      env.reset(seed=i+1)
+      layouts.add(env.observations()[MAZE_LAYOUT_OBSERVATION])
+    self.assertTrue(np.isclose(len(layouts) - num_layouts, MAZE_LAYOUT_TRIALS))
+
+if __name__ == '__main__':
+  if 'TEST_SRCDIR' in os.environ:
+    deepmind_lab.set_runfiles_path(
+        os.path.join(os.environ['TEST_SRCDIR'],
+                     'org_deepmind_lab'))
+  unittest.main()