diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..6657a1b8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +.vs/ +*.user +CompiledShaders/ +*.pdb +*.deps +*.cache +*.log +*.tlog +*FileListAbsolute.txt +*.obj +*.exe +*.ipdb +*.pch +*.iobj diff --git a/DepthSensingCUDA/DepthSensing.vcxproj b/DepthSensingCUDA/DepthSensing.vcxproj index ff205104..3877e0ec 100644 --- a/DepthSensingCUDA/DepthSensing.vcxproj +++ b/DepthSensingCUDA/DepthSensing.vcxproj @@ -21,18 +21,18 @@ Application true Unicode - v120 + v142 Application false true Unicode - v120 + v142 - + @@ -50,8 +50,8 @@ false $(DXSDK_DIR)Utilities\bin\x64;$(DXSDK_DIR)Utilities\bin\x86;$(ExecutablePath) - $(IncludePath);$(WindowsSDK_IncludePath);$(KINECTSDK10_DIR)\inc;$(KINECTSDK20_DIR)\inc;$(OPENNI2_INCLUDE64);$(DXSDK_DIR)Include;./Source/;./Include;./Include/Intel;./Include/cutil/inc;./Include/Uplink - $(LibraryPath);$(DXSDK_DIR)Lib\x64;$(KINECTSDK10_DIR)\lib\amd64;$(CUDA_LIB_PATH);$(KINECTSDK20_DIR)\lib\x64;$(OPENNI2_LIB64);Libs + $(IncludePath);$(WindowsSDK_IncludePath);$(KINECTSDK10_DIR)\inc;$(KINECTSDK20_DIR)\inc;$(OPENNI2_INCLUDE64);$(DXSDK_DIR)Include;./Source/;./Include;./Include/Intel;./Include/cutil/inc;./Include/Uplink;C:\Users\14109\Downloads\opencv\build\include + $(LibraryPath);$(DXSDK_DIR)Lib\x64;$(KINECTSDK10_DIR)\lib\amd64;$(CUDA_LIB_PATH);$(KINECTSDK20_DIR)\lib\x64;$(OPENNI2_LIB64);Libs;C:\Users\14109\Downloads\opencv\build\x64\vc15\lib @@ -103,12 +103,12 @@ 4819 - d3d11.lib;d3dcompiler.lib;d3dx11.lib;d3dx9.lib;dxerr.lib;dxguid.lib;winmm.lib;comctl32.lib;Shlwapi.lib;cudart.lib;cublas.lib;FreeImage.lib;zlib64.lib;%(AdditionalDependencies) + d3d11.lib;d3dcompiler.lib;d3dx11.lib;d3dx9.lib;dxerr.lib;dxguid.lib;winmm.lib;comctl32.lib;Shlwapi.lib;cudart.lib;cublas.lib;FreeImage.lib;zlib64.lib;opencv_world440.lib;opencv_world440d.lib;%(AdditionalDependencies) true true Console true - ../opencv/lib/;%(AdditionalLibraryDirectories) + ../opencv/lib/;C:\Users\14109\Downloads\opencv\build\x64\vc15\lib;%(AdditionalLibraryDirectories) true @@ -203,6 +203,8 @@ + + Create Create @@ -269,6 +271,8 @@ + + @@ -310,6 +314,6 @@ - + \ No newline at end of file diff --git a/DepthSensingCUDA/DepthSensing.vcxproj.filters b/DepthSensingCUDA/DepthSensing.vcxproj.filters index c37c4fad..d3190d00 100644 --- a/DepthSensingCUDA/DepthSensing.vcxproj.filters +++ b/DepthSensingCUDA/DepthSensing.vcxproj.filters @@ -167,6 +167,12 @@ Sensors + + Sensors + + + Sensors + @@ -333,6 +339,12 @@ CUDA + + Sensors + + + Sensors + diff --git a/DepthSensingCUDA/Include/mLib/include/core-mesh/meshData.cpp b/DepthSensingCUDA/Include/mLib/include/core-mesh/meshData.cpp index d579849c..47c9f878 100644 --- a/DepthSensingCUDA/Include/mLib/include/core-mesh/meshData.cpp +++ b/DepthSensingCUDA/Include/mLib/include/core-mesh/meshData.cpp @@ -477,7 +477,18 @@ void MeshData::merge( const MeshData& other ) return; } if (isEmpty()) { - *this = other; + + m_Vertices = std::move(other.m_Vertices); + m_Normals = std::move(other.m_Normals); + m_TextureCoords = std::move(other.m_TextureCoords); + m_Colors = std::move(other.m_Colors); + m_FaceIndicesVertices = std::move(other.m_FaceIndicesVertices); + m_FaceIndicesNormals = std::move(other.m_FaceIndicesNormals); + m_FaceIndicesTextureCoords = std::move(other.m_FaceIndicesTextureCoords); + m_FaceIndicesColors = std::move(other.m_FaceIndicesColors); + m_MaterialFile = std::move(other.m_MaterialFile); + m_MaterialIndices = std::move(other.m_MaterialIndices); + //*this = other; return; } diff --git a/DepthSensingCUDA/Source/DepthSensing.cpp b/DepthSensingCUDA/Source/DepthSensing.cpp index 0864ad0f..d1e070f5 100644 --- a/DepthSensingCUDA/Source/DepthSensing.cpp +++ b/DepthSensingCUDA/Source/DepthSensing.cpp @@ -3,6 +3,7 @@ #include "DepthSensing.h" #include "StructureSensor.h" #include "SensorDataReader.h" +#include "SensorDataReaderZhou.h" //-------------------------------------------------------------------------------------- // Global variables @@ -119,7 +120,15 @@ RGBDSensor* getRGBDSensor() g_sensor = new SensorDataReader; return g_sensor; #else - throw MLIB_EXCEPTION("Requires STRUCTURE_SENSOR macro"); + throw MLIB_EXCEPTION("Requires SENSOR_DATA_READER macro"); +#endif + } + if (GlobalAppState::get().s_sensorIdx == GlobalAppState::Sensor_SensorDataFileReader) { +#ifdef SENSOR_DATA_FILE_READER + g_sensor = new SensorDataReaderZhou(1, GlobalAppState::get().s_nVideoFrame, GlobalAppState::get().s_startFrame); + return g_sensor; +#else + throw MLIB_EXCEPTION("Requires SENSOR_DATA_FILE_READER macro"); #endif } @@ -652,7 +661,7 @@ void reconstruction() { //only if binary dump - if (GlobalAppState::get().s_sensorIdx == GlobalAppState::Sensor_BinaryDumpReader || GlobalAppState::get().s_sensorIdx == GlobalAppState::Sensor_SensorDataReader) { + if (GlobalAppState::get().s_sensorIdx == GlobalAppState::Sensor_BinaryDumpReader || GlobalAppState::get().s_sensorIdx == GlobalAppState::Sensor_SensorDataReader || GlobalAppState::get().s_sensorIdx == GlobalAppState::Sensor_SensorDataFileReader) { unsigned int heapFreeCount = g_sceneRep->getHeapFreeCount(); std::cout << "[ frame " << g_RGBDAdapter.getFrameNumber() << " ] " << " [Free SDFBlocks " << heapFreeCount << " ] " << std::endl; if (heapFreeCount < 5000) std::cout << "WARNING: Heap Free Count is low! if crash, increase s_hashNumSDFBlocks" << std::endl; @@ -733,6 +742,28 @@ void reconstruction() && GlobalAppState::get().s_binaryDumpSensorUseTrajectoryOnlyInit) { //deltaTransformEstimate = lastTransform.getInverse() * transformation; //simple case; not ideal in case of drift //deltaTransformEstimate = g_RGBDAdapter.getRigidTransform(-1).getInverse() * transformation; + + /*mat4f lastTransform = g_sceneRep->getLastRigidTransform(); + mat4f deltaTransformEstimate = mat4f::identity(); + + transformation = g_cameraTracking->applyCT( + g_CudaDepthSensor.getCameraSpacePositionsFloat4(), g_CudaDepthSensor.getNormalMapFloat4(), g_CudaDepthSensor.getColorMapFilteredFloat4(), + //g_rayCast->getRayCastData().d_depth4Transformed, g_CudaDepthSensor.getNormalMapNoRefinementFloat4(), g_CudaDepthSensor.getColorMapFilteredFloat4(), + g_rayCast->getRayCastData().d_depth4, g_rayCast->getRayCastData().d_normals, g_rayCast->getRayCastData().d_colors, + lastTransform, + GlobalCameraTrackingState::getInstance().s_maxInnerIter, GlobalCameraTrackingState::getInstance().s_maxOuterIter, + GlobalCameraTrackingState::getInstance().s_distThres, GlobalCameraTrackingState::getInstance().s_normalThres, + 100.0f, 3.0f, + deltaTransformEstimate, + GlobalCameraTrackingState::getInstance().s_residualEarlyOut, + g_RGBDAdapter.getDepthIntrinsics(), g_CudaDepthSensor.getDepthCameraData(), + NULL); + + transformation = g_RGBDAdapter.getRigidTransform(); + if (transformation[0] == -std::numeric_limits::infinity()) { + std::cout << "INVALID FRAME" << std::endl; + return; + }*/ } const bool useRGBDTracking = false; //Depth vs RGBD @@ -846,9 +877,9 @@ void CALLBACK OnD3D11FrameRender( ID3D11Device* pd3dDevice, ID3D11DeviceContext* pd3dImmediateContext->ClearRenderTargetView(pRTV, ClearColor); pd3dImmediateContext->ClearDepthStencilView(pDSV, D3D11_CLEAR_DEPTH, 1.0f, 0); -#ifdef SENSOR_DATA_READER +#if defined(SENSOR_DATA_READER) || defined(SENSOR_DATA_FILE_READER) //only if sensor data reader - if (GlobalAppState::get().s_sensorIdx == GlobalAppState::Sensor_SensorDataReader && GlobalAppState::get().s_playData) { + if ((GlobalAppState::get().s_sensorIdx == GlobalAppState::Sensor_SensorDataReader || GlobalAppState::get().s_sensorIdx == GlobalAppState::Sensor_SensorDataFileReader) && GlobalAppState::get().s_playData) { SensorDataReader* sensor = (SensorDataReader*)getRGBDSensor(); if (sensor->getCurrFrame() >= sensor->getNumFrames()) { diff --git a/DepthSensingCUDA/Source/DepthSensing.h b/DepthSensingCUDA/Source/DepthSensing.h index 69c4825d..5bb8c411 100644 --- a/DepthSensingCUDA/Source/DepthSensing.h +++ b/DepthSensingCUDA/Source/DepthSensing.h @@ -43,6 +43,7 @@ #include "CUDASceneRepChunkGrid.h" #include +#pragma comment(lib, "legacy_stdio_definitions.lib") #ifdef KINECT #pragma comment(lib, "Kinect10.lib") diff --git a/DepthSensingCUDA/Source/GlobalAppState.h b/DepthSensingCUDA/Source/GlobalAppState.h index 62585052..384ad707 100644 --- a/DepthSensingCUDA/Source/GlobalAppState.h +++ b/DepthSensingCUDA/Source/GlobalAppState.h @@ -8,6 +8,7 @@ //#define REAL_SENSE //#define STRUCTURE_SENSOR #define SENSOR_DATA_READER +#define SENSOR_DATA_FILE_READER //#define OBJECT_SENSING @@ -97,7 +98,9 @@ X(bool, s_reconstructionEnabled) \ X(bool, s_renderToFile) \ X(std::string, s_renderToFileDir) \ - X(bool, s_offlineProcessing) + X(bool, s_offlineProcessing) \ + X(int, s_nVideoFrame) \ + X(int, s_startFrame) #ifndef VAR_NAME @@ -119,7 +122,8 @@ class GlobalAppState Sensor_IntelSensor = 5, Sensor_RealSense = 6, Sensor_StructureSensor = 7, - Sensor_SensorDataReader = 8 + Sensor_SensorDataReader = 8, + Sensor_SensorDataFileReader = 9 }; #define X(type, name) type name; diff --git a/DepthSensingCUDA/Source/SensorDataReaderZhou.cpp b/DepthSensingCUDA/Source/SensorDataReaderZhou.cpp new file mode 100644 index 00000000..2617bd39 --- /dev/null +++ b/DepthSensingCUDA/Source/SensorDataReaderZhou.cpp @@ -0,0 +1,173 @@ + +#include "stdafx.h" + +#include "SensorDataReaderZhou.h" +#include "GlobalAppState.h" +#include "MatrixConversion.h" + +#ifdef SENSOR_DATA_FILE_READER + +#include "sensorData/sensorData.h" + +#include +#include +#include +#include +#include +#include + +#include + +SensorDataReaderZhou::SensorDataReaderZhou() +{ + m_numFrames = 0; + m_currFrame = 0; + m_bHasColorData = false; + + m_sensorData = NULL; + m_sensorDataCache = NULL; +} + +SensorDataReaderZhou::SensorDataReaderZhou(int x, int nFrames, int startFrame) +{ + m_numFrames = nFrames; + + m_currFrame = 0; + m_startFrame = startFrame; + m_bHasColorData = false; + + m_sensorData = NULL; + m_sensorDataCache = NULL; + m_type = x; +} + +SensorDataReaderZhou::~SensorDataReaderZhou() +{ + releaseData(); +} + + +HRESULT SensorDataReaderZhou::createFirstConnected() +{ + releaseData(); + + std::string filename = GlobalAppState::get().s_binaryDumpSensorFile[0]; + + std::cout << "Start loading binary dump... " << filename; + m_sensorData = new SensorDataZhou; + + if (m_type == 0) { + m_sensorData->loadFromFile(filename, m_numFrames); + std::cout << "DONE!" << std::endl; + } + if (m_type == 1) + m_sensorData->loadFromFileObject(filename, m_numFrames, m_startFrame); + std::cout << "DONE!" << std::endl; + std::cout << *m_sensorData << std::endl; + + std::cout << m_sensorData->m_depthWidth << " " << m_sensorData->m_depthHeight << " " << m_sensorData->m_colorWidth <<" "<< m_sensorData->m_colorHeight << std::endl; + + RGBDSensor::init(m_sensorData->m_depthWidth, m_sensorData->m_depthHeight, std::max(m_sensorData->m_colorWidth, 1u), std::max(m_sensorData->m_colorHeight, 1u), 1); + initializeDepthIntrinsics(m_sensorData->m_calibrationDepth.m_intrinsic(0, 0), m_sensorData->m_calibrationDepth.m_intrinsic(1, 1), m_sensorData->m_calibrationDepth.m_intrinsic(0, 2), m_sensorData->m_calibrationDepth.m_intrinsic(1, 2)); + initializeColorIntrinsics(m_sensorData->m_calibrationColor.m_intrinsic(0, 0), m_sensorData->m_calibrationColor.m_intrinsic(1, 1), m_sensorData->m_calibrationColor.m_intrinsic(0, 2), m_sensorData->m_calibrationColor.m_intrinsic(1, 2)); + + initializeDepthExtrinsics(m_sensorData->m_calibrationDepth.m_extrinsic); + initializeColorExtrinsics(m_sensorData->m_calibrationColor.m_extrinsic); + + + m_numFrames = (unsigned int)m_sensorData->m_frames.size(); + + + if (m_numFrames > 0 && m_sensorData->m_frames[0].getColorCompressed()) { + m_bHasColorData = true; + } + else { + m_bHasColorData = false; + } + + const unsigned int cacheSize = 10; + m_sensorDataCache = new RGBDFrameCacheRead(m_sensorData, cacheSize); + + return S_OK; +} + +ml::mat4f SensorDataReaderZhou::getRigidTransform(int offset) const +{ + unsigned int idx = m_currFrame - 1 + offset; + if (idx >= m_sensorData->m_frames.size()) throw MLIB_EXCEPTION("invalid trajectory index " + std::to_string(idx)); + const mat4f& transform = m_sensorData->m_frames[idx].getCameraToWorld(); + return transform; + //return m_data.m_trajectory[idx]; +} + +HRESULT SensorDataReaderZhou::processDepth() +{ + //if (m_currFrame >= m_numFrames) + //{ + // GlobalAppState::get().s_playData = false; + // //std::cout << "binary dump sequence complete - press space to run again" << std::endl; + // stopReceivingFrames(); + // std::cout << "binary dump sequence complete - stopped receiving frames" << std::endl; + // m_currFrame = 0; + //} + //if (m_currFrame >= m_numFrames) + //{ + // + // int da = m_currFrame / m_numFrames; + // //std::cout << "binary dump sequence complete - press space to run again" << std::endl; + + //} + if (m_sensorDataCache->getCheck()) GlobalAppState::get().s_integrationEnabled = false; + + if (GlobalAppState::get().s_playData) { + + float* depth = getDepthFloat(); + + //TODO check why the frame cache is not used? + RGBDFrameCacheRead::FrameState frameState = m_sensorDataCache->getNextOurs(); + + int depthsize = getDepthWidth()*getDepthHeight(); + + for (unsigned int i = 0; i ::infinity(); + else depth[i] = (float)frameState.m_depthFrame[i] / m_sensorData->m_depthShift; + } + + incrementRingbufIdx(); + + if (m_bHasColorData) { + int colorsize= getColorWidth()*getColorHeight(); + for (unsigned int i = 0; i < colorsize; i++) { + m_colorRGBX[i] = vec4uc(frameState.m_colorFrame[i]); + } + } + //frameState.free(); + + m_currFrame++; + return S_OK; + } + else { + return S_FALSE; + } +} + +std::string SensorDataReaderZhou::getSensorName() const +{ + return m_sensorData->m_sensorName; +} + +void SensorDataReaderZhou::releaseData() +{ + m_currFrame = 0; + m_bHasColorData = false; + + + SAFE_DELETE(m_sensorDataCache); + + if (m_sensorData) { + m_sensorData->free(); + SAFE_DELETE(m_sensorData); + } +} + +#endif diff --git a/DepthSensingCUDA/Source/SensorDataReaderZhou.h b/DepthSensingCUDA/Source/SensorDataReaderZhou.h new file mode 100644 index 00000000..9acf5583 --- /dev/null +++ b/DepthSensingCUDA/Source/SensorDataReaderZhou.h @@ -0,0 +1,73 @@ +#pragma once + + +/************************************************************************/ +/* Reads sensor data files from .sens files */ +/************************************************************************/ + +#include "GlobalAppState.h" +#include "RGBDSensor.h" +#include "sensordatazhou.h" +#include "stdafx.h" + +#ifdef SENSOR_DATA_READER + +namespace ml { + class SensorDataZhou; + class RGBDFrameCacheRead; +} + +class SensorDataReaderZhou : public RGBDSensor +{ +public: + + //load Kun's dataset. + //1234567 - 123456789ABC + //0004443 - 000148226209 : dept + //0000001 - 000000000000 : image + //frame num - timestamp + void loadFromFileZhou(const std::string& filename); + + //! Constructor + SensorDataReaderZhou(); + + SensorDataReaderZhou(int x, int nFrames = 0, int startFrame=0); + + //! Destructor; releases allocated ressources + ~SensorDataReaderZhou(); + + //! initializes the sensor + HRESULT createFirstConnected(); + + //! reads the next depth frame + HRESULT processDepth(); + + HRESULT processColor() { + //everything done in process depth since order is relevant (color must be read first) + return S_OK; + } + + std::string getSensorName() const; + + mat4f getRigidTransform(int offset) const; + +private: + //! deletes all allocated data + void releaseData(); + + //ml::SensorData* m_sensorData; + //ml::SensorData::RGBDFrameCacheRead* m_sensorDataCache; + ml::SensorDataZhou* m_sensorData; + ml::RGBDFrameCacheRead* m_sensorDataCache; + + unsigned int m_numFrames; + unsigned int m_currFrame; + unsigned int m_startFrame; + bool m_bHasColorData; + int m_nFrames; + int m_type; + +}; + + +#endif //sensor data reader diff --git a/DepthSensingCUDA/Source/sensorData/sensorData.h b/DepthSensingCUDA/Source/sensorData/sensorData.h index c7d42fbf..0c6544b1 100644 --- a/DepthSensingCUDA/Source/sensorData/sensorData.h +++ b/DepthSensingCUDA/Source/sensorData/sensorData.h @@ -185,6 +185,13 @@ namespace ml { m_extrinsic.setIdentity(); } + void setMatrices(mat4f& intrinsic, mat4f& extrinsic = mat4f::identity()) { + m_intrinsic = intrinsic; + m_extrinsic = extrinsic; + //m_intr = m_intrinsic.getInverse(); + //m_ExtrinsicInverse = m_extrinsic.getInverse(); + } + void saveToFile(std::ofstream& out) const { out.write((const char*)&m_intrinsic, sizeof(mat4f)); out.write((const char*)&m_extrinsic, sizeof(mat4f)); @@ -281,6 +288,7 @@ namespace ml { private: friend class SensorData; + friend class SensorDataZhou; RGBDFrame( const vec3uc* color, unsigned int colorWidth, unsigned int colorHeight, @@ -1222,6 +1230,31 @@ namespace ml { return FrameState(); } + FrameState getNextOurs() { + while (1) { + //if (m_nextFromSensorCache >= m_sensorData->m_frames.size()) { + // m_bTerminateThread = true; // should be already true anyway + // break; //we're done + //} + if (m_data.size() > 0 && m_data.front().m_bIsReady) { + m_mutexList.lock(); + FrameState fs = m_data.front(); + m_data.erase(m_data.begin()); + m_mutexList.unlock(); + m_nextFromSensorCache++; + return fs; + } + else { + Sleep(0); + } + } + return FrameState(); + } + + bool getCheck() { + return check; + } + private: void startDecompBackgroundThread() { m_decompThread = std::thread(decompFunc, this); @@ -1258,6 +1291,8 @@ namespace ml { std::mutex m_mutexList; std::atomic m_bTerminateThread; + bool check; + unsigned int m_nextFromSensorData; unsigned int m_nextFromSensorCache; }; diff --git a/DepthSensingCUDA/Source/sensordatazhou.cpp b/DepthSensingCUDA/Source/sensordatazhou.cpp new file mode 100644 index 00000000..2dda2568 --- /dev/null +++ b/DepthSensingCUDA/Source/sensordatazhou.cpp @@ -0,0 +1,19 @@ +#include "sensordatazhou.h" +#include "stdafx.h" + + +void read_directory(const std::string& name, std::vector& v) +{ + std::string pattern(name); + pattern.append("\\*"); + WIN32_FIND_DATAA data; + HANDLE hFind; + hFind = FindFirstFileA(pattern.c_str(), &data); + + if (hFind != INVALID_HANDLE_VALUE) { + do { + v.push_back(std::string((char*)data.cFileName)); + } while (FindNextFileA(hFind, &data) != 0); + FindClose(hFind); + } +} \ No newline at end of file diff --git a/DepthSensingCUDA/Source/sensordatazhou.h b/DepthSensingCUDA/Source/sensordatazhou.h new file mode 100644 index 00000000..8749254d --- /dev/null +++ b/DepthSensingCUDA/Source/sensordatazhou.h @@ -0,0 +1,291 @@ + + +#ifndef _SENSOR_FILE_ZHOU_H_ +#define _SENSOR_FILE_ZHOU_H_ + +#include "mlib.h" +//#include "opencv2\core.hpp" +#include "opencv2\opencv.hpp" +#include "sensorData\sensorData.h" + +void read_directory(const std::string& name, std::vector& v); + +namespace ml { + + class SensorDataZhou : public SensorData { + public: + + //the input is a path! + //load Zhou's dataset. + //1234567 - 123456789ABC + //0004443 - 000148226209 : dept + //0000001 - 000000000000 : image + //frame num - timestamp + + void loadFromFile(const std::string& filename, int nFrame) { + + std::cout << filename + "\\camerainfo.txt" << std::endl; + + //camera info + + // filename + std::ifstream in(filename + "\\camerainfo.txt", std::ios::in); + + if (!in.is_open()) { + throw MLIB_EXCEPTION("could not open file " + filename); + } + + float rgbwidth, rgbheight; + float dwidth, dheight; + float rgbfx, rgbfy; + float dfx, dfy; + float rgbcx, rgbcy, dcx, dcy; + float dshift; + in >> rgbwidth >> rgbheight; + in >> rgbfx >> rgbfy >> rgbcx >> rgbcy; + in >> dwidth >> dheight; + in >> dfx >> dfy >> dcx >> dcy; + in >> dshift; + //debugging + + //rgb + printf("rgb w: %f h: %f\n", rgbwidth, rgbheight); + printf("rgb fx: %f fy: %f\n", rgbfx, rgbfy); + printf("rgb cx: %f cy: %f\n", rgbcx, rgbcy); + + //depth + printf("depth w: %f h: %f\n", dwidth, dheight); + printf("depth fx: %f fy: %f\n", dfx, dfy); + printf("depth cx: %f cy: %f\n", dcx, dcy); + printf("depth shift: %f\n", dshift); + + //set intrinsic matrix + m_calibrationColor.setMatrices(m_calibrationColor.makeIntrinsicMatrix(rgbfx, rgbfy, rgbcx, rgbcy)); + m_calibrationDepth.setMatrices(m_calibrationDepth.makeIntrinsicMatrix(dfx, dfy, dcx, dcy)); + + m_colorCompressionType = ml::SensorData::COMPRESSION_TYPE_COLOR::TYPE_RAW; + m_depthCompressionType = ml::SensorData::COMPRESSION_TYPE_DEPTH::TYPE_RAW_USHORT; + + m_colorWidth = rgbwidth; + m_colorHeight = rgbheight; + m_depthWidth = dwidth; + m_depthHeight = dheight; + m_depthShift = dshift; + + std::cout << filename + "\\depth_vga.match" << std::endl; + + // filename + in = std::ifstream(filename + "\\depth_vga.match", std::ios::in); + + if (!in.is_open()) { + throw MLIB_EXCEPTION("could not open file " + filename); + } + + //read rgbd data + std::vector color(m_colorWidth * m_colorHeight * 3); + std::vector depth(m_depthWidth * m_depthHeight); + + int i; + std::string dfilename, vgafilename; + //char timestamp[30], frame[30]; + int timestamp, frame; + RGBDFrame* rgbdframe; + + m_frames.resize(nFrame); + int framecnt = 0; + + for (i = 0; !in.eof() && framecnt < nFrame; i++) { + + rgbdframe = &m_frames[framecnt]; + + //read match + in >> dfilename >> vgafilename; + std::cout << dfilename << " " << vgafilename << std::endl; + + framecnt++; + + //read rgb image + std::cout << filename + "\\" + vgafilename << " "; + + sscanf(vgafilename.data(), "vga\\%07d-000%09d.jpg", &frame, ×tamp); + printf("%d %d\n", frame, timestamp); + + cv::Mat bgr = cv::imread(filename + "\\" + vgafilename, cv::IMREAD_COLOR), bgra; + if (bgr.empty()) + continue; + + size_t nbytes = bgr.cols * bgr.rows * 3; + if (nbytes != color.size()) + continue; + + cv::cvtColor(bgr, bgr, cv::COLOR_BGR2RGB); + memcpy_s(color.data(), nbytes, bgr.data, nbytes); + + //set rgb timestamp and data + rgbdframe->setTimeStampColor(timestamp); + rgbdframe->m_colorSizeBytes = m_colorWidth * m_colorHeight * 3; + + rgbdframe->compressColor((vec3uc*)color.data(), m_colorWidth, m_colorHeight, TYPE_RAW); + + //read depth image + std::cout << filename + "\\" + dfilename << " "; + + sscanf(dfilename.data(), "depth\\%07d-000%09d.png", &frame, ×tamp); + printf("%d %d\n", frame, timestamp); + + cv::Mat depthmat = cv::imread(filename + "\\" + dfilename, cv::IMREAD_ANYDEPTH); + if (depthmat.empty()) + continue; + nbytes = depthmat.cols * depthmat.rows * 2; + if (nbytes != depth.size() * 2) + continue; + + depthmat.convertTo(depthmat, CV_16UC1); + memcpy_s(depth.data(), nbytes, depthmat.data, nbytes); + + //set depth time stemp + rgbdframe->setTimeStampDepth(timestamp); + rgbdframe->m_depthSizeBytes = m_depthWidth * m_depthHeight * 4; + rgbdframe->compressDepth((unsigned short*)depth.data(), m_depthWidth, m_depthHeight, TYPE_RAW_USHORT); + + } + } + + void loadFromFileObject(const std::string& filename, int nFrame, int startFrame) { + + std::cout << filename + "\\camerainfo.txt" << std::endl; + + //camera info + + // filename + std::ifstream in(filename + "\\camerainfo.txt", std::ios::in); + + if (!in.is_open()) { + throw MLIB_EXCEPTION("could not open file " + filename); + } + + float rgbwidth, rgbheight; + float dwidth, dheight; + float rgbfx, rgbfy; + float dfx, dfy; + float rgbcx, rgbcy, dcx, dcy; + float dshift; + in >> rgbwidth >> rgbheight; + in >> rgbfx >> rgbfy >> rgbcx >> rgbcy; + in >> dwidth >> dheight; + in >> dfx >> dfy >> dcx >> dcy; + in >> dshift; + //debugging + + //rgb + printf("rgb w: %f h: %f\n", rgbwidth, rgbheight); + printf("rgb fx: %f fy: %f\n", rgbfx, rgbfy); + printf("rgb cx: %f cy: %f\n", rgbcx, rgbcy); + + //depth + printf("depth w: %f h: %f\n", dwidth, dheight); + printf("depth fx: %f fy: %f\n", dfx, dfy); + printf("depth cx: %f cy: %f\n", dcx, dcy); + printf("depth shift: %f\n", dshift); + + //set intrinsic matrix + m_calibrationColor.setMatrices(m_calibrationColor.makeIntrinsicMatrix(rgbfx, rgbfy, rgbcx, rgbcy)); + m_calibrationDepth.setMatrices(m_calibrationDepth.makeIntrinsicMatrix(dfx, dfy, dcx, dcy)); + + m_colorCompressionType = ml::SensorData::COMPRESSION_TYPE_COLOR::TYPE_RAW; + m_depthCompressionType = ml::SensorData::COMPRESSION_TYPE_DEPTH::TYPE_RAW_USHORT; + + m_colorWidth = rgbwidth; + m_colorHeight = rgbheight; + m_depthWidth = dwidth; + m_depthHeight = dheight; + m_depthShift = dshift; + + std::cout << filename + "\\depth_vga.match" << std::endl; + + // filename + in = std::ifstream(filename + "\\depth_vga.match", std::ios::in); + + if (!in.is_open()) { + throw MLIB_EXCEPTION("could not open file " + filename); + } + + //read rgbd data + std::vector color(m_colorWidth * m_colorHeight * 3); + std::vector depth(m_depthWidth * m_depthHeight); + + int i; + std::string dfilename, vgafilename; + //char timestamp[30], frame[30]; + int timestamp, frame; + RGBDFrame* rgbdframe; + + m_frames.resize(nFrame); + int framecnt = 0; + + for (i = 0; !in.eof() && framecnt < nFrame; i++) { + + //rgbdframe = new RGBDFrame; + rgbdframe = &m_frames[framecnt]; + + //read match + in >> dfilename >> vgafilename; + std::cout << dfilename << " " << vgafilename << std::endl; + + if (i < startFrame) + continue; + framecnt++; + //read rgb image + std::cout << filename + "/" + vgafilename << " "; + + //sscanf(vgafilename.data(), "vga/%07d-000%09d.jpg", &frame, ×tamp); + printf("%d %d\n", frame, timestamp); + + cv::Mat bgr = cv::imread(filename + "/" + vgafilename, cv::IMREAD_COLOR); + if (bgr.empty()) + continue; + + printf("%d %d\n", bgr.rows, bgr.cols); + + size_t nbytes = bgr.cols * bgr.rows * 3; + if (nbytes != color.size()) + continue; + + cv::cvtColor(bgr, bgr, cv::COLOR_BGR2RGB); + memcpy_s(color.data(), nbytes, bgr.data, nbytes); + + //set rgb timestamp and data + rgbdframe->setTimeStampColor(timestamp); + rgbdframe->m_colorSizeBytes = m_colorWidth * m_colorHeight * 3; + + rgbdframe->compressColor((vec3uc*)color.data(), m_colorWidth, m_colorHeight, TYPE_RAW); + + + //read depth image + std::cout << filename + "/" + dfilename << " "; + + sscanf(dfilename.data(), "depth/%07d-000%09d.png", &frame, ×tamp); + printf("%d %d\n", frame, timestamp); + + cv::Mat depthmat = cv::imread(filename + "\\" + dfilename, cv::IMREAD_ANYDEPTH); + if (depthmat.empty()) + continue; + nbytes = depthmat.cols * depthmat.rows * 2; + if (nbytes != depth.size() * 2) + continue; + + depthmat.convertTo(depthmat, CV_16UC1); + memcpy_s(depth.data(), nbytes, depthmat.data, nbytes); + + //set depth time stemp + rgbdframe->setTimeStampDepth(timestamp); + rgbdframe->m_depthSizeBytes = m_depthWidth * m_depthHeight * 4; + rgbdframe->compressDepth((unsigned short*)depth.data(), m_depthWidth, m_depthHeight, TYPE_RAW_USHORT); + + } + } + + }; +}; + +#endif //_SENSOR_FILE_ZHOU_H_ diff --git a/DepthSensingCUDA/zParametersManolisScan.txt b/DepthSensingCUDA/zParametersManolisScan.txt index 31261f03..7cb5b19d 100644 --- a/DepthSensingCUDA/zParametersManolisScan.txt +++ b/DepthSensingCUDA/zParametersManolisScan.txt @@ -1,5 +1,11 @@ -// 0=Kinect; 1=PrimeSense; 2=KinectOne; 3=BinaryDumpReader; 4=NetworkSensor; 5=IntelSensor; 6=RealSense; 7=StructureSensor; 8=SensorDataReader -s_sensorIdx = 0; +// 0=Kinect; 1=PrimeSense; 2=KinectOne; 3=BinaryDumpReader; 4=NetworkSensor; 5=IntelSensor; 6=RealSense; 7=StructureSensor; 8=SensorDataReader; 9=SensorDataFileReader +s_sensorIdx = 9; + + +// used when we use record files +s_nVideoFrame = 587; +//s_nVideoFrame = 0; +s_startFrame = 0; //s_windowWidth = 1280; //render window width //s_windowHeight = 960; //render window height @@ -43,7 +49,8 @@ s_SDFUseGradients = false; //analytic gradients for rendering //s_binaryDumpSensorFile[0] = "./DumpOutput/8194nk5LbLH_0.sens"; //s_binaryDumpSensorFile[1] = "./DumpOutput/8194nk5LbLH_1.sens"; //s_binaryDumpSensorFile[2] = "./DumpOutput/8194nk5LbLH_2.sens"; -s_binaryDumpSensorFile[0] = "Dump/test.sens"; +//s_binaryDumpSensorFile[0] = "C:\Users\14109\Downloads\fountain_all\scene0012_01.sens"; +s_binaryDumpSensorFile[0] = "C:\Users\14109\Downloads\fountain_all\fountain_all\"; s_binaryDumpSensorUseTrajectory = true; s_binaryDumpSensorUseTrajectoryOnlyInit = false; //only valid if prev is true; precomputed traj for ICP, which is run