Add headless rendering with TinyRenderer

README.md

@@ -127,10 +127,10 @@ The dependencies are gym, Qt5, assimp, tinyxml, and bullet (from a branch). For
 Compile and install bullet as follows. Note that `make install` will merely copy files into the roboschool directory.
 
 ```bash
-git clone https://github.com/olegklimov/bullet3 -b roboschool_self_collision
+git clone https://github.com/bulletphysics/bullet3
 mkdir bullet3/build
 cd    bullet3/build
-cmake -DBUILD_SHARED_LIBS=ON -DUSE_DOUBLE_PRECISION=1 -DCMAKE_INSTALL_PREFIX:PATH=$ROBOSCHOOL_PATH/roboschool/cpp-household/bullet_local_install -DBUILD_CPU_DEMOS=OFF -DBUILD_BULLET2_DEMOS=OFF -DBUILD_EXTRAS=OFF  -DBUILD_UNIT_TESTS=OFF -DBUILD_CLSOCKET=OFF -DBUILD_ENET=OFF -DBUILD_OPENGL3_DEMOS=OFF ..
+cmake -DBUILD_SHARED_LIBS=ON -DUSE_DOUBLE_PRECISION=1 -DCMAKE_INSTALL_PREFIX:PATH=$ROBOSCHOOL_PATH/roboschool/cpp-household/bullet_local_install -DBUILD_CPU_DEMOS=OFF -DBUILD_BULLET2_DEMOS=OFF -DBUILD_EXTRAS=ON  -DBUILD_UNIT_TESTS=OFF -DBUILD_CLSOCKET=OFF -DBUILD_ENET=OFF -DBUILD_OPENGL3_DEMOS=OFF ..
 make -j4
 make install
 cd ../..

roboschool/cpp-household/Makefile

@@ -49,9 +49,9 @@ endif
 
 $(info Link against python $(PYTHON))
 INC     += `$(PKG) --cflags Qt5Widgets Qt5OpenGL assimp python-$(PYTHON)`
-LIBS    += -lstdc++ `$(PKG) --libs Qt5OpenGL Qt5Widgets assimp python-$(PYTHON)`
+LIBS    += -lstdc++ `$(PKG) --libs Qt5OpenGL Qt5Widgets assimp tinyxml python-$(PYTHON)`
 INC     += -Ibullet_local_install/include -Ibullet_local_install/include/bullet -I/usr/local/include/bullet
-LIBS    += $(RPATH) -Lbullet_local_install/lib -lLinearMath -lBullet3Common -lBulletCollision -lBulletDynamics -lBulletInverseDynamics -lPhysicsClientC_API
+LIBS    += $(RPATH) -Lbullet_local_install/lib -lLinearMath -lBullet3Common -lBulletCollision -lBulletDynamics -lBulletInverseDynamics -lBulletRobotics
 
 ifeq ($(PYTHON),2.7)
     BOOST_PYTHON = -lboost_python

roboschool/cpp-household/physics-bullet.cpp

@@ -30,6 +30,11 @@ void World::bullet_init(float gravity, float timestep)
 		fprintf(stderr, "Start chrome trace log, handle %i\n", chrome_trace_log);
 	}
 #endif
+  {
+    b3SharedMemoryCommandHandle commandHandle = b3InitConfigureOpenGLVisualizer(client);
+    b3ConfigureOpenGLVisualizerSetVisualizationFlags(commandHandle, COV_ENABLE_TINY_RENDERER, 1);
+    b3SubmitClientCommandAndWaitStatus(client, commandHandle);
+  }
 }
 
 World::~World()
@@ -95,7 +100,6 @@ shared_ptr<Robot> World::load_urdf(const std::string& fn, const btTransform& tr,
 std::list<shared_ptr<Robot>> World::load_sdf_mjcf(const std::string& fn, bool mjcf)
 {
 	std::list<shared_ptr<Robot>> ret;
-	const int MAX_SDF_BODIES = 512;
 	int bodyIndicesOut[MAX_SDF_BODIES];
 	int N;
 	if (mjcf) {

roboschool/cpp-household/python-binding.cpp

@@ -2,6 +2,8 @@
 #include <boost/weak_ptr.hpp>
 
 #include "render-glwidget.h"
+#include <bullet/Bullet3Common/b3Vector3.h>
+#include <bullet/Bullet3Common/b3Matrix3x3.h>
 
 #include <QtWidgets/QApplication>
 #include <QtWidgets/QDesktopWidget>
@@ -238,6 +240,52 @@ struct Camera {
 				);
 	}
 
+  boost::python::object render_direct()
+  {
+    auto const camera_pose = cref->camera_pose;
+
+    b3Scalar viewMatrix[16];
+
+    {
+      auto const bt_eye = camera_pose.getOrigin();
+      auto const bt_cup = btMatrix3x3(camera_pose.getRotation()) * btVector3(0, 1, 0);
+      b3Scalar camera_position[3] = {bt_eye.x(), bt_eye.y(), bt_eye.z()};
+      b3Scalar target_position[3] = {0, 0, 0};
+      b3Scalar camera_up[3]       = {bt_cup.x(), bt_cup.y(), bt_cup.z()};
+      b3ComputeViewMatrixFromPositions(camera_position, target_position, camera_up, viewMatrix);
+    }
+
+    b3Scalar projectionMatrix[16];
+    b3ComputeProjectionMatrixFOV(cref->camera_hfov, cref->camera_res_w / cref->camera_res_h, cref->camera_near, cref->camera_far, projectionMatrix);
+
+
+    b3SharedMemoryCommandHandle command;
+    command = b3InitRequestCameraImage(wref->client);
+    b3RequestCameraImageSetPixelResolution(command, cref->camera_res_w, cref->camera_res_h);
+
+    b3RequestCameraImageSetCameraMatrices(command, viewMatrix, projectionMatrix);
+
+    b3RequestCameraImageSetShadow(command, false);
+    b3RequestCameraImageSetLightAmbientCoeff(command, 0.6);
+    b3RequestCameraImageSetLightDiffuseCoeff(command, 0.35);
+    b3RequestCameraImageSetLightSpecularCoeff(command, 0.05);
+    b3RequestCameraImageSelectRenderer(command, ER_TINY_RENDERER);
+
+    if (b3CanSubmitCommand(wref->client))
+    {
+      auto const statusHandle = b3SubmitClientCommandAndWaitStatus(wref->client, command);
+      int statusType = b3GetStatusType(statusHandle);
+      if (statusType == CMD_CAMERA_IMAGE_COMPLETED)
+      {
+        b3CameraImageData imageData;
+        b3GetCameraImageData(wref->client, &imageData);
+
+        return boost::python::object(boost::python::handle<>(PyBytes_FromStringAndSize(reinterpret_cast<const char*>(imageData.m_rgbColorData), imageData.m_pixelWidth * imageData.m_pixelHeight * 4)));
+      }
+    }
+    return boost::python::object();
+  }
+
 	void move_and_look_at(float from_x, float from_y, float from_z, float obj_x, float obj_y, float obj_z)
 	{
 		Pose pose;
@@ -641,6 +689,7 @@ void cpp_household_init()
 	.add_property("name", &Camera::name)
 	.add_property("resolution", &Camera::resolution)
 	.def("render", &Camera::render)
+	.def("render_direct", &Camera::render_direct)
 	.def("test_window", &Camera::test_window)
 	.def("test_window_score", &Camera::test_window_score)
 	.def("set_key_callback", &Camera::set_key_callback)

roboschool/gym_mujoco_xml_env.py

@@ -82,9 +82,9 @@ def _render(self, mode, close):
             return self.scene.cpp_world.test_window()
         elif mode=="rgb_array":
             self.camera_adjust()
-            rgb, _, _, _, _ = self.camera.render(False, False, False) # render_depth, render_labeling, print_timing)
-            rendered_rgb = np.fromstring(rgb, dtype=np.uint8).reshape( (self.VIDEO_H,self.VIDEO_W,3) )
-            return rendered_rgb
+            rgb = self.camera.render_direct()
+            rgb = np.fromstring(rgb, dtype=np.uint8).reshape((self.VIDEO_H, self.VIDEO_W, 4))
+            return rgb
         else:
             assert(0)
 

roboschool/gym_urdf_robot_env.py

@@ -92,9 +92,9 @@ def _render(self, mode, close):
             return self.scene.cpp_world.test_window()
         elif mode=="rgb_array":
             self.camera_adjust()
-            rgb, _, _, _, _ = self.camera.render(False, False, False) # render_depth, render_labeling, print_timing)
-            rendered_rgb = np.fromstring(rgb, dtype=np.uint8).reshape( (self.VIDEO_H,self.VIDEO_W,3) )
-            return rendered_rgb
+            rgb = self.camera.render_direct()
+            rgb = np.fromstring(rgb, dtype=np.uint8).reshape((self.VIDEO_H, self.VIDEO_W, 4))
+            return rgb
         else:
             assert(0)
 

setup.py

@@ -27,7 +27,7 @@
 from sys import platform
 if platform=="darwin":
     bulletlibs  = "libBullet2FileLoader libBulletCollision libBullet3Collision libBulletDynamics libBullet3Common libBulletInverseDynamics".split()
-    bulletlibs += "libBullet3Dynamics libBulletSoftBody libBullet3Geometry libLinearMath libBullet3OpenCL_clew libPhysicsClientC_API".split()
+    bulletlibs += "libBullet3Dynamics libBulletSoftBody libBullet3Geometry libLinearMath libBullet3OpenCL_clew libBulletRobotics".split()
     for x in bulletlibs:
         os.system("install_name_tool -id @loader_path/cpp-household/bullet_local_install/lib/%s.2.87.dylib %s/%s.2.87.dylib" % (x,blib,x))
         for y in bulletlibs: