changed directory structure; test for dynamic Scenic

BerkeleyLearnVerify · Sep 15, 2020 · c951eff · c951eff
1 parent dab2bef
commit c951eff
Show file tree

Hide file tree

Showing 89 changed files with 7,833 additions and 15 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,6 @@
+# Scenic cache files
+*.snet
+
 # Webots temporary files
 .*.wbproj
 

diff --git a/README.md b/README.md
@@ -24,4 +24,6 @@ If you have any problems using VerifAI, please submit an issue to the GitHub rep
 
 * _examples_: examples and additional documentation for particular simulators, including CARLA, Webots, X-Plane, and OpenAI Gym;
 
-* _verifai_: the source for the `verifai` package proper.
+* _src/verifai_: the source for the `verifai` package proper;
+
+* _tests_: the VerifAI test suite.
diff --git a/verifai/__init__.py → src/verifai/__init__.py b/verifai/__init__.py → src/verifai/__init__.py
diff --git a/verifai/client.py → src/verifai/client.py b/verifai/client.py → src/verifai/client.py
diff --git a/verifai/error_table.py → src/verifai/error_table.py b/verifai/error_table.py → src/verifai/error_table.py
diff --git a/verifai/falsifier.py → src/verifai/falsifier.py b/verifai/falsifier.py → src/verifai/falsifier.py
diff --git a/verifai/features/__init__.py → src/verifai/features/__init__.py b/verifai/features/__init__.py → src/verifai/features/__init__.py
diff --git a/verifai/features/features.py → src/verifai/features/features.py b/verifai/features/features.py → src/verifai/features/features.py
diff --git a/verifai/monitor.py → src/verifai/monitor.py b/verifai/monitor.py → src/verifai/monitor.py
diff --git a/verifai/samplers/__init__.py → src/verifai/samplers/__init__.py b/verifai/samplers/__init__.py → src/verifai/samplers/__init__.py
diff --git a/verifai/samplers/bayesian_optimization.py → ...verifai/samplers/bayesian_optimization.py b/verifai/samplers/bayesian_optimization.py → ...verifai/samplers/bayesian_optimization.py
diff --git a/verifai/samplers/cross_entropy.py → src/verifai/samplers/cross_entropy.py b/verifai/samplers/cross_entropy.py → src/verifai/samplers/cross_entropy.py
diff --git a/verifai/samplers/dist_BO.py → src/verifai/samplers/dist_BO.py b/verifai/samplers/dist_BO.py → src/verifai/samplers/dist_BO.py
diff --git a/verifai/samplers/domain_sampler.py → src/verifai/samplers/domain_sampler.py b/verifai/samplers/domain_sampler.py → src/verifai/samplers/domain_sampler.py
diff --git a/verifai/samplers/feature_sampler.py → src/verifai/samplers/feature_sampler.py b/verifai/samplers/feature_sampler.py → src/verifai/samplers/feature_sampler.py
diff --git a/verifai/samplers/grid_sampler.py → src/verifai/samplers/grid_sampler.py b/verifai/samplers/grid_sampler.py → src/verifai/samplers/grid_sampler.py
diff --git a/verifai/samplers/halton.py → src/verifai/samplers/halton.py b/verifai/samplers/halton.py → src/verifai/samplers/halton.py
diff --git a/verifai/samplers/random_sampler.py → src/verifai/samplers/random_sampler.py b/verifai/samplers/random_sampler.py → src/verifai/samplers/random_sampler.py
diff --git a/verifai/samplers/rejection.py → src/verifai/samplers/rejection.py b/verifai/samplers/rejection.py → src/verifai/samplers/rejection.py
diff --git a/verifai/samplers/scenic_sampler.py → src/verifai/samplers/scenic_sampler.py b/verifai/samplers/scenic_sampler.py → src/verifai/samplers/scenic_sampler.py
diff --git a/verifai/samplers/simulated_annealing.py → src/verifai/samplers/simulated_annealing.py b/verifai/samplers/simulated_annealing.py → src/verifai/samplers/simulated_annealing.py
diff --git a/verifai/scenic_server.py → src/verifai/scenic_server.py b/verifai/scenic_server.py → src/verifai/scenic_server.py
diff --git a/verifai/server.py → src/verifai/server.py b/verifai/server.py → src/verifai/server.py
diff --git a/verifai/simulators/__init__.py → src/verifai/simulators/__init__.py b/verifai/simulators/__init__.py → src/verifai/simulators/__init__.py
diff --git a/verifai/simulators/car_simulator/__init__.py → ...ifai/simulators/car_simulator/__init__.py b/verifai/simulators/car_simulator/__init__.py → ...ifai/simulators/car_simulator/__init__.py
diff --git a/.../simulators/car_simulator/car_dynamics.py → .../simulators/car_simulator/car_dynamics.py b/.../simulators/car_simulator/car_dynamics.py → .../simulators/car_simulator/car_dynamics.py
diff --git a/...ai/simulators/car_simulator/car_object.py → ...ai/simulators/car_simulator/car_object.py b/...ai/simulators/car_simulator/car_object.py → ...ai/simulators/car_simulator/car_object.py
diff --git a/...imulators/car_simulator/client_car_sim.py → ...imulators/car_simulator/client_car_sim.py b/...imulators/car_simulator/client_car_sim.py → ...imulators/car_simulator/client_car_sim.py
diff --git a/...lators/car_simulator/examples/__init__.py → ...lators/car_simulator/examples/__init__.py b/...lators/car_simulator/examples/__init__.py → ...lators/car_simulator/examples/__init__.py
diff --git a/...examples/control_utils/LQR_computation.py → ...examples/control_utils/LQR_computation.py b/...examples/control_utils/LQR_computation.py → ...examples/control_utils/LQR_computation.py
diff --git a/...ulator/examples/control_utils/__init__.py → ...ulator/examples/control_utils/__init__.py b/...ulator/examples/control_utils/__init__.py → ...ulator/examples/control_utils/__init__.py
diff --git a/...ator/examples/lanekeeping_LQR/__init__.py → ...ator/examples/lanekeeping_LQR/__init__.py b/...ator/examples/lanekeeping_LQR/__init__.py → ...ator/examples/lanekeeping_LQR/__init__.py
diff --git a/.../lanekeeping_LQR/lanekeeping_falsifier.py → .../lanekeeping_LQR/lanekeeping_falsifier.py b/.../lanekeeping_LQR/lanekeeping_falsifier.py → .../lanekeeping_LQR/lanekeeping_falsifier.py
diff --git a/...lanekeeping_LQR/lanekeeping_simulation.py → ...lanekeeping_LQR/lanekeeping_simulation.py b/...lanekeeping_LQR/lanekeeping_simulation.py → ...lanekeeping_LQR/lanekeeping_simulation.py
diff --git a/.../car_simulator/examples/simple_control.py → .../car_simulator/examples/simple_control.py b/.../car_simulator/examples/simple_control.py → .../car_simulator/examples/simple_control.py
diff --git a/...imulators/car_simulator/imgs/car-blue.png → ...imulators/car_simulator/imgs/car-blue.png b/...imulators/car_simulator/imgs/car-blue.png → ...imulators/car_simulator/imgs/car-blue.png
diff --git a/...imulators/car_simulator/imgs/car-gray.png → ...imulators/car_simulator/imgs/car-gray.png b/...imulators/car_simulator/imgs/car-gray.png → ...imulators/car_simulator/imgs/car-gray.png
diff --git a/...ulators/car_simulator/imgs/car-orange.png → ...ulators/car_simulator/imgs/car-orange.png b/...ulators/car_simulator/imgs/car-orange.png → ...ulators/car_simulator/imgs/car-orange.png
diff --git a/...ulators/car_simulator/imgs/car-purple.png → ...ulators/car_simulator/imgs/car-purple.png b/...ulators/car_simulator/imgs/car-purple.png → ...ulators/car_simulator/imgs/car-purple.png
diff --git a/...simulators/car_simulator/imgs/car-red.png → ...simulators/car_simulator/imgs/car-red.png b/...simulators/car_simulator/imgs/car-red.png → ...simulators/car_simulator/imgs/car-red.png
diff --git a/...mulators/car_simulator/imgs/car-white.png → ...mulators/car_simulator/imgs/car-white.png b/...mulators/car_simulator/imgs/car-white.png → ...mulators/car_simulator/imgs/car-white.png
diff --git a/...ulators/car_simulator/imgs/car-yellow.png → ...ulators/car_simulator/imgs/car-yellow.png b/...ulators/car_simulator/imgs/car-yellow.png → ...ulators/car_simulator/imgs/car-yellow.png
diff --git a/...ai/simulators/car_simulator/imgs/cone.png → ...ai/simulators/car_simulator/imgs/cone.png b/...ai/simulators/car_simulator/imgs/cone.png → ...ai/simulators/car_simulator/imgs/cone.png
diff --git a/...mulators/car_simulator/imgs/firetruck.png → ...mulators/car_simulator/imgs/firetruck.png b/...mulators/car_simulator/imgs/firetruck.png → ...mulators/car_simulator/imgs/firetruck.png
diff --git a/...i/simulators/car_simulator/imgs/grass.png → ...i/simulators/car_simulator/imgs/grass.png b/...i/simulators/car_simulator/imgs/grass.png → ...i/simulators/car_simulator/imgs/grass.png
diff --git a/verifai/simulators/car_simulator/lane.py → src/verifai/simulators/car_simulator/lane.py b/verifai/simulators/car_simulator/lane.py → src/verifai/simulators/car_simulator/lane.py
diff --git a/...fai/simulators/car_simulator/simulator.py → ...fai/simulators/car_simulator/simulator.py b/...fai/simulators/car_simulator/simulator.py → ...fai/simulators/car_simulator/simulator.py
diff --git a/...ai/simulators/carla/agents/brake_agent.py → ...ai/simulators/carla/agents/brake_agent.py b/...ai/simulators/carla/agents/brake_agent.py → ...ai/simulators/carla/agents/brake_agent.py
diff --git a/...simulators/carla/agents/overtake_agent.py → ...simulators/carla/agents/overtake_agent.py b/...simulators/carla/agents/overtake_agent.py → ...simulators/carla/agents/overtake_agent.py
diff --git a/verifai/simulators/carla/agents/pid_agent.py → ...ifai/simulators/carla/agents/pid_agent.py b/verifai/simulators/carla/agents/pid_agent.py → ...ifai/simulators/carla/agents/pid_agent.py
diff --git a/...ors/carla/agents/pid_follow_controller.py → ...ors/carla/agents/pid_follow_controller.py b/...ors/carla/agents/pid_follow_controller.py → ...ors/carla/agents/pid_follow_controller.py
diff --git a/...fai/simulators/carla/carla_scenic_task.py → ...fai/simulators/carla/carla_scenic_task.py b/...fai/simulators/carla/carla_scenic_task.py → ...fai/simulators/carla/carla_scenic_task.py
diff --git a/verifai/simulators/carla/carla_task.py → src/verifai/simulators/carla/carla_task.py b/verifai/simulators/carla/carla_task.py → src/verifai/simulators/carla/carla_task.py
diff --git a/verifai/simulators/carla/carla_world.py → src/verifai/simulators/carla/carla_world.py b/verifai/simulators/carla/carla_world.py → src/verifai/simulators/carla/carla_world.py
diff --git a/verifai/simulators/carla/client_carla.py → src/verifai/simulators/carla/client_carla.py b/verifai/simulators/carla/client_carla.py → src/verifai/simulators/carla/client_carla.py
diff --git a/verifai/simulators/openai_gym/__init__.py → ...verifai/simulators/openai_gym/__init__.py b/verifai/simulators/openai_gym/__init__.py → ...verifai/simulators/openai_gym/__init__.py
diff --git a/...i/simulators/openai_gym/baselines_task.py → ...i/simulators/openai_gym/baselines_task.py b/...i/simulators/openai_gym/baselines_task.py → ...i/simulators/openai_gym/baselines_task.py
diff --git a/verifai/simulators/openai_gym/client_gym.py → ...rifai/simulators/openai_gym/client_gym.py b/verifai/simulators/openai_gym/client_gym.py → ...rifai/simulators/openai_gym/client_gym.py
diff --git a/verifai/simulators/webots/__init__.py → src/verifai/simulators/webots/__init__.py b/verifai/simulators/webots/__init__.py → src/verifai/simulators/webots/__init__.py
diff --git a/verifai/simulators/webots/client_webots.py → ...erifai/simulators/webots/client_webots.py b/verifai/simulators/webots/client_webots.py → ...erifai/simulators/webots/client_webots.py
diff --git a/verifai/simulators/webots/webots_task.py → src/verifai/simulators/webots/webots_task.py b/verifai/simulators/webots/webots_task.py → src/verifai/simulators/webots/webots_task.py
diff --git a/verifai/simulators/xplane/__init__.py → src/verifai/simulators/xplane/__init__.py b/verifai/simulators/xplane/__init__.py → src/verifai/simulators/xplane/__init__.py
diff --git a/verifai/simulators/xplane/server.py → src/verifai/simulators/xplane/server.py b/verifai/simulators/xplane/server.py → src/verifai/simulators/xplane/server.py
diff --git a/verifai/simulators/xplane/utils/__init__.py → ...rifai/simulators/xplane/utils/__init__.py b/verifai/simulators/xplane/utils/__init__.py → ...rifai/simulators/xplane/utils/__init__.py
diff --git a/...fai/simulators/xplane/utils/controller.py → ...fai/simulators/xplane/utils/controller.py b/...fai/simulators/xplane/utils/controller.py → ...fai/simulators/xplane/utils/controller.py
diff --git a/verifai/simulators/xplane/utils/geometry.py → ...rifai/simulators/xplane/utils/geometry.py b/verifai/simulators/xplane/utils/geometry.py → ...rifai/simulators/xplane/utils/geometry.py
diff --git a/verifai/simulators/xplane/utils/images.py → ...verifai/simulators/xplane/utils/images.py b/verifai/simulators/xplane/utils/images.py → ...verifai/simulators/xplane/utils/images.py
diff --git a/verifai/utils/__init__.py → src/verifai/utils/__init__.py b/verifai/utils/__init__.py → src/verifai/utils/__init__.py
diff --git a/verifai/utils/frozendict.py → src/verifai/utils/frozendict.py b/verifai/utils/frozendict.py → src/verifai/utils/frozendict.py
diff --git a/verifai/utils/utils.py → src/verifai/utils/utils.py b/verifai/utils/utils.py → src/verifai/utils/utils.py
diff --git a/tests/Town01.xodr b/tests/Town01.xodr
diff --git a/verifai/tests/__init__.py → tests/__init__.py b/verifai/tests/__init__.py → tests/__init__.py
diff --git a/verifai/tests/conftest.py → tests/conftest.py b/verifai/tests/conftest.py → tests/conftest.py
diff --git a/verifai/tests/road.wbt → tests/road.wbt b/verifai/tests/road.wbt → tests/road.wbt
diff --git a/tests/scenic_driving.scenic b/tests/scenic_driving.scenic
@@ -0,0 +1,10 @@
+import scenic
+
+param map = localPath('Town01.xodr')
+param carla_map = 'Town01'
+
+model scenic.domains.driving.model
+
+simulator scenic.core.simulators.DummySimulator()
+
+ego = Car at Range(-5, 5) @ 2, with behavior FollowLaneBehavior
diff --git a/verifai/tests/scenic_mars.scenic → tests/scenic_mars.scenic b/verifai/tests/scenic_mars.scenic → tests/scenic_mars.scenic
diff --git a/verifai/tests/scenic_road.scenic → tests/scenic_road.scenic b/verifai/tests/scenic_road.scenic → tests/scenic_road.scenic
diff --git a/verifai/tests/test_bayesianOptimization.py → tests/test_bayesianOptimization.py b/verifai/tests/test_bayesianOptimization.py → tests/test_bayesianOptimization.py
@@ -3,7 +3,7 @@
 from dotmap import DotMap
 
 import pytest
-from verifai.tests.utils import sampleWithFeedback, checkSaveRestore
+from tests.utils import sampleWithFeedback, checkSaveRestore
 
 pytest.importorskip('GPyOpt')
 

diff --git a/verifai/tests/test_crossEntropy.py → tests/test_crossEntropy.py b/verifai/tests/test_crossEntropy.py → tests/test_crossEntropy.py
@@ -2,7 +2,7 @@
 from verifai.samplers import *
 from dotmap import DotMap
 
-from verifai.tests.utils import sampleWithFeedback, checkSaveRestore
+from tests.utils import sampleWithFeedback, checkSaveRestore
 
 def test_crossEntropy():
     carDomain = Struct({

diff --git a/verifai/tests/test_distBO.py → tests/test_distBO.py b/verifai/tests/test_distBO.py → tests/test_distBO.py
@@ -2,7 +2,7 @@
 from verifai.samplers import *
 from dotmap import DotMap
 from verifai.samplers.dist_BO import DistBayesOptSampler
-from verifai.tests.utils import sampleWithFeedback
+from tests.utils import sampleWithFeedback
 
 import pytest
 

diff --git a/verifai/tests/test_domains.py → tests/test_domains.py b/verifai/tests/test_domains.py → tests/test_domains.py
diff --git a/verifai/tests/test_examples.py → tests/test_examples.py b/verifai/tests/test_examples.py → tests/test_examples.py
diff --git a/verifai/tests/test_features.py → tests/test_features.py b/verifai/tests/test_features.py → tests/test_features.py
diff --git a/verifai/tests/test_grid.py → tests/test_grid.py b/verifai/tests/test_grid.py → tests/test_grid.py
diff --git a/verifai/tests/test_halton.py → tests/test_halton.py b/verifai/tests/test_halton.py → tests/test_halton.py
@@ -2,7 +2,7 @@
 
 from verifai.features import *
 from verifai.samplers import *
-from verifai.tests.utils import checkSaveRestore
+from tests.utils import checkSaveRestore
 
 def test_halton():
     carDomain = Struct({

diff --git a/verifai/tests/test_samplers.py → tests/test_samplers.py b/verifai/tests/test_samplers.py → tests/test_samplers.py
diff --git a/verifai/tests/test_scenic.py → tests/test_scenic.py b/verifai/tests/test_scenic.py → tests/test_scenic.py
@@ -1,8 +1,11 @@
 
 import pytest
+from dotmap import DotMap
 
 from verifai.samplers.scenic_sampler import ScenicSampler
-from verifai.tests.utils import sampleWithFeedback, checkSaveRestore
+from verifai.scenic_server import ScenicServer
+from verifai.falsifier import generic_falsifier
+from tests.utils import sampleWithFeedback, checkSaveRestore
 
 ## Basic
 
@@ -87,20 +90,42 @@ def test_active_save_restore(tmpdir):
 
 ## Webots
 
-def test_webots_mars(pathToLocalFile):
-    path = pathToLocalFile('scenic_mars.scenic')
-    sampler = ScenicSampler.fromScenario(path)
-
+def runSampler(sampler):
     for i in range(3):
         sample = sampler.nextSample()
         print(f'Sample #{i}:')
         print(sample)
 
+def test_webots_mars(pathToLocalFile):
+    path = pathToLocalFile('scenic_mars.scenic')
+    sampler = ScenicSampler.fromScenario(path)
+    runSampler(sampler)
+
 def test_webots_road(pathToLocalFile):
     path = pathToLocalFile('scenic_road.scenic')
     sampler = ScenicSampler.fromScenario(path)
+    runSampler(sampler)
 
-    for i in range(3):
-        sample = sampler.nextSample()
-        print(f'Sample #{i}:')
-        print(sample)
+## Driving domain
+
+def test_driving(pathToLocalFile):
+    path = pathToLocalFile('scenic_driving.scenic')
+    sampler = ScenicSampler.fromScenario(path)
+    runSampler(sampler)
+
+## Dynamic scenarios
+
+def test_driving_dynamic(pathToLocalFile):
+    path = pathToLocalFile('scenic_driving.scenic')
+    sampler = ScenicSampler.fromScenario(path)
+    falsifier_params = DotMap(
+        n_iters=3,
+        save_error_table=False,
+        save_safe_table=False,
+    )
+    server_options = DotMap(maxSteps=2, verbosity=3)
+    falsifier = generic_falsifier(sampler=sampler,
+                                  falsifier_params=falsifier_params,
+                                  server_class=ScenicServer,
+                                  server_options=server_options)
+    falsifier.run_falsifier()
diff --git a/verifai/tests/test_simulatedAnnealing.py → tests/test_simulatedAnnealing.py b/verifai/tests/test_simulatedAnnealing.py → tests/test_simulatedAnnealing.py
@@ -3,7 +3,7 @@
 
 from verifai.features import *
 from verifai.samplers import *
-from verifai.tests.utils import sampleWithFeedback, checkSaveRestore
+from tests.utils import sampleWithFeedback, checkSaveRestore
 
 ## Utilities
 

diff --git a/verifai/tests/utils.py → tests/utils.py b/verifai/tests/utils.py → tests/utils.py