Imitation learning with dagger

.gitignore

@@ -111,3 +111,5 @@ flow.ang
 *.ang.old
 *.sang
 
+# local configuration file for data pipeline
+**/data_pipeline_config

environment.yml

@@ -1,18 +1,17 @@
 name: flow
 
 dependencies:
-    - python==3.6.8
-    - scipy==1.1.0
-    - lxml==4.4.1
-    - six==1.11.0
-    - path.py
-    - python-dateutil==2.7.3
-    - pip>=18.0
-    - tensorflow==1.9.0
-    - cloudpickle==1.2.1
-    - setuptools==41.0.0
-    - plotly==2.4.0
+    - python==3.7.3
     - pip:
+        - scipy==1.1.0
+        - lxml==4.4.1
+        - six==1.11.0
+        - path.py
+        - python-dateutil==2.7.3
+        - pip>=18.0
+        - tensorflow==1.15.2
+        - setuptools==41.0.0
+        - plotly==2.4.0
         - gym==0.14.0
         - pyprind==2.11.2
         - nose2==0.8.0
@@ -21,9 +20,11 @@ dependencies:
         - matplotlib==3.0.0
         - dill
         - lz4
-        - ray==0.7.3
+        - ray==0.8.0
         - setproctitle
         - psutil
         - opencv-python
-        - boto3==1.4.8
+        - boto3==1.10.45
         - redis~=2.10.6
+        - tabulate
+        - pytz

examples/README.md

@@ -53,11 +53,11 @@ trained through RL algorithms provided by *RLlib*.
 To execute these examples, run
 
 ```shell script
-python train.py EXP_CONFIG --rl_trainer "rllib"
+python train.py EXP_CONFIG --rl_trainer "rllib" --algorithm <ALGORITHM NAME>
 ```
 where `EXP_CONFIG` is the name of the experiment configuration file, as located
-in `exp_configs/rl/singleagent` or  `exp_configs/rl/multiagent.`
-
+in `exp_configs/rl/singleagent` or  `exp_configs/rl/multiagent.` Here `<ALGORITHM NAME>`
+should be the name of your desired algorithm. Currently we support PPO and TD3.
 
 ### stable-baselines
 

examples/exp_configs/non_rl/highway_single.py

@@ -1,12 +1,17 @@
-"""Example of an open network with human-driven vehicles."""
+"""Example of an open network with human-driven vehicles and a wave."""
+
+import numpy as np
+
 from flow.controllers import IDMController
+from flow.controllers.velocity_controllers import FollowerStopper
 from flow.core.params import EnvParams
 from flow.core.params import NetParams
 from flow.core.params import InitialConfig
 from flow.core.params import InFlows
 from flow.core.params import VehicleParams
 from flow.core.params import SumoParams
 from flow.core.params import SumoLaneChangeParams
+from flow.core.rewards import instantaneous_mpg
 from flow.core.params import SumoCarFollowingParams
 from flow.networks import HighwayNetwork
 from flow.envs import TestEnv
@@ -22,6 +27,8 @@
 HORIZON = 1500
 # whether to include noise in the car-following models
 INCLUDE_NOISE = True
+# penetration rate of the follower-stopper vehicles
+PENETRATION_RATE = 0.0
 
 additional_net_params = ADDITIONAL_NET_PARAMS.copy()
 additional_net_params.update({
@@ -59,15 +66,38 @@
     ),
 )
 
+if PENETRATION_RATE > 0.0:
+    vehicles.add(
+        "av",
+        color='red',
+        num_vehicles=0,
+        acceleration_controller=(FollowerStopper, {
+            "v_des": 5.0,
+            "control_length": [500, 2300]
+        }),
+    )
+
 inflows = InFlows()
+
 inflows.add(
     veh_type="human",
     edge="highway_0",
-    vehs_per_hour=TRAFFIC_FLOW,
+    vehs_per_hour=int(TRAFFIC_FLOW * (1 - PENETRATION_RATE / 100)),
     depart_lane="free",
     depart_speed=TRAFFIC_SPEED,
     name="idm_highway_inflow")
 
+if PENETRATION_RATE > 0.0:
+    inflows.add(
+        veh_type="av",
+        edge="highway_0",
+        vehs_per_hour=int(TRAFFIC_FLOW * (PENETRATION_RATE / 100)),
+        depart_lane="free",
+        depart_speed=TRAFFIC_SPEED,
+        name="av_highway_inflow")
+
+# SET UP FLOW PARAMETERS
+
 flow_params = dict(
     # name of the experiment
     exp_tag='highway-single',
@@ -111,3 +141,13 @@
     # reset (see flow.core.params.InitialConfig)
     initial=InitialConfig(),
 )
+
+custom_callables = {
+    "avg_merge_speed": lambda env: np.nan_to_num(np.mean(
+        env.k.vehicle.get_speed(env.k.vehicle.get_ids()))),
+    "avg_outflow": lambda env: np.nan_to_num(
+        env.k.vehicle.get_outflow_rate(120)),
+    "miles_per_gallon": lambda env: np.nan_to_num(
+        instantaneous_mpg(env, env.k.vehicle.get_ids(), gain=1.0)
+    )
+}

examples/exp_configs/non_rl/i210_subnetwork.py

@@ -2,18 +2,21 @@
 import os
 import numpy as np
 
-from flow.controllers import IDMController
-from flow.controllers import I210Router
+from flow.controllers.car_following_models import IDMController
+from flow.controllers.velocity_controllers import FollowerStopper
+from flow.controllers.routing_controllers import I210Router
 from flow.core.params import SumoParams
 from flow.core.params import EnvParams
 from flow.core.params import NetParams
 from flow.core.params import SumoLaneChangeParams
 from flow.core.params import VehicleParams
 from flow.core.params import InitialConfig
 from flow.core.params import InFlows
-import flow.config as config
+from flow.core.rewards import instantaneous_mpg
+from flow.networks import I210SubNetwork
+from flow.networks.i210_subnetwork import EDGES_DISTRIBUTION
 from flow.envs import TestEnv
-from flow.networks.i210_subnetwork import I210SubNetwork, EDGES_DISTRIBUTION
+import flow.config as config
 
 # =========================================================================== #
 # Specify some configurable constants.                                        #
@@ -27,8 +30,14 @@
 ON_RAMP = False
 # the inflow rate of vehicles (in veh/hr)
 INFLOW_RATE = 2050
+# on-ramp inflow_rate
+ON_RAMP_INFLOW_RATE = 500
 # the speed of inflowing vehicles from the main edge (in m/s)
 INFLOW_SPEED = 25.5
+# fraction of vehicles that are follower-stoppers. 0.10 corresponds to 10%
+PENETRATION_RATE = 0.0
+# desired speed of the follower stopper vehicles
+V_DES = 5.0
 # horizon over which to run the env
 HORIZON = 1500
 # steps to run before follower-stopper is allowed to take control
@@ -63,11 +72,12 @@
 # =========================================================================== #
 
 vehicles = VehicleParams()
+
 vehicles.add(
     "human",
     num_vehicles=0,
     lane_change_params=SumoLaneChangeParams(
-        lane_change_mode="strategic",
+        lane_change_mode="sumo_default",
     ),
     acceleration_controller=(IDMController, {
         "a": 1.3,
@@ -77,29 +87,54 @@
     routing_controller=(I210Router, {}) if ON_RAMP else None,
 )
 
+vehicles.add(
+    "av",
+    num_vehicles=0,
+    color="red",
+    acceleration_controller=(FollowerStopper, {
+        "v_des": V_DES,
+        "no_control_edges": ["ghost0", "119257908#3"]
+    }),
+    routing_controller=(I210Router, {}) if ON_RAMP else None,
+)
+
 inflow = InFlows()
+
 # main highway
+highway_start_edge = "ghost0" if WANT_GHOST_CELL else "119257914"
+
 for lane in [0, 1, 2, 3, 4]:
     inflow.add(
         veh_type="human",
-        edge="ghost0" if WANT_GHOST_CELL else "119257914",
-        vehs_per_hour=INFLOW_RATE,
-        departLane=lane,
-        departSpeed=INFLOW_SPEED)
+        edge=highway_start_edge,
+        vehs_per_hour=INFLOW_RATE * (1 - PENETRATION_RATE),
+        depart_lane=lane,
+        depart_speed=INFLOW_SPEED)
+
+    if PENETRATION_RATE > 0.0:
+        inflow.add(
+            veh_type="av",
+            edge=highway_start_edge,
+            vehs_per_hour=INFLOW_RATE * PENETRATION_RATE,
+            depart_lane=lane,
+            depart_speed=INFLOW_SPEED)
+
 # on ramp
 if ON_RAMP:
     inflow.add(
         veh_type="human",
         edge="27414345",
-        vehs_per_hour=500,
-        departLane="random",
-        departSpeed=10)
-    inflow.add(
-        veh_type="human",
-        edge="27414342#0",
-        vehs_per_hour=500,
-        departLane="random",
-        departSpeed=10)
+        vehs_per_hour=int(ON_RAMP_INFLOW_RATE * (1 - PENETRATION_RATE)),
+        depart_speed=10,
+    )
+
+    if PENETRATION_RATE > 0.0:
+        inflow.add(
+            veh_type="av",
+            edge="27414345",
+            vehs_per_hour=int(ON_RAMP_INFLOW_RATE * PENETRATION_RATE),
+            depart_lane="random",
+            depart_speed=10)
 
 # =========================================================================== #
 # Generate the flow_params dict with all relevant simulation information.     #
@@ -122,7 +157,7 @@
     sim=SumoParams(
         sim_step=0.4,
         render=False,
-        color_by_speed=True,
+        color_by_speed=False,
         use_ballistic=True
     ),
 
@@ -160,14 +195,21 @@
 # =========================================================================== #
 
 edge_id = "119257908#1-AddedOnRampEdge"
+
+
+def valid_ids(env, veh_ids):
+    """Return the names of vehicles within the controllable edges."""
+    return [
+        veh_id for veh_id in veh_ids
+        if env.k.vehicle.get_edge(veh_id) not in ["ghost0", "119257908#3"]
+    ]
+
+
 custom_callables = {
     "avg_merge_speed": lambda env: np.nan_to_num(np.mean(
-        env.k.vehicle.get_speed(env.k.vehicle.get_ids_by_edge(edge_id)))),
+        env.k.vehicle.get_speed(valid_ids(env, env.k.vehicle.get_ids())))),
     "avg_outflow": lambda env: np.nan_to_num(
         env.k.vehicle.get_outflow_rate(120)),
-    # we multiply by 5 to account for the vehicle length and by 1000 to convert
-    # into veh/km
-    "avg_density": lambda env: 5 * 1000 * len(env.k.vehicle.get_ids_by_edge(
-        edge_id)) / (env.k.network.edge_length(edge_id)
-                     * env.k.network.num_lanes(edge_id)),
+    "mpg": lambda env: instantaneous_mpg(
+        env,  valid_ids(env, env.k.vehicle.get_ids()), gain=1.0),
 }