Develop

.gitignore

@@ -47,3 +47,5 @@ lib/local/LandmarkDetector/Debug/
 matlab_runners/Head Pose Experiments/experiments/biwi_out/
 matlab_runners/Head Pose Experiments/experiments/bu_out/
 matlab_runners/Head Pose Experiments/experiments/ict_out/
+
+OpenFace\.VC\.db

CMakeLists.txt

@@ -42,6 +42,19 @@ foreach(file ${files})
 	install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model)
 endforeach()
 
+#copy Dlib Landmark model
+file(GLOB files "lib/local/LandmarkDetector/model/*.dat")
+foreach(file ${files})
+	if (MSVC)
+		file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/Debug/model)
+		file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/Release/model)
+	else(MSVC)
+		file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/model)
+	endif(MSVC)
+
+	install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model)
+endforeach()
+
 # Move the hierarchical LandmarkDetector models
 file(GLOB files "lib/local/LandmarkDetector/model/model*")
 foreach(file ${files})
@@ -178,3 +191,4 @@ add_subdirectory(exe/FaceLandmarkImg)
 add_subdirectory(exe/FaceLandmarkVid)
 add_subdirectory(exe/FaceLandmarkVidMulti)
 add_subdirectory(exe/FeatureExtraction)
+add_subdirectory(python)

Copyright.txt

@@ -1,12 +1,11 @@
-///////////////////////////////////////////////////////////////////////////////
-// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge,
-// all rights reserved.
-//
-// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
-//
-// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
-// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
-//
+Copyright (C) 2017, University of Southern California, University of Cambridge, and Carnegie Mellon University, all rights reserved
+
+ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+
+THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Notwithstanding the license granted herein, Licensee acknowledges that certain components of the Software may be covered by so-called �open source� software licenses (�Open Source Components�), which means any software licenses approved as open source licenses by the Open Source Initiative or any substantially similar licenses, including without limitation any license that, as a condition of distribution of the software licensed under such license, requires that the distributor make the software available in source code format. Carnegie Mellon shall provide a list of Open Source Components for a particular version of the Software upon Licensee�s request.  Licensee will comply with the applicable terms of such licenses and to the extent required by the licenses covering Open Source Components, the terms of such licenses will apply in lieu of the terms of this Agreement.  To the extent the terms of the licenses applicable to Open Source Components prohibit any of the restrictions in this License Agreement with respect to such Open Source Component, such restrictions will not apply to such Open Source Component.  To the extent the terms of the licenses applicable to Open Source Components require Carnegie Mellon to make an offer to provide source code or related information in connection with the Software, such offer is hereby made.  Any request for source code or related information should be directed to Tadas Baltrusaitis.  Licensee acknowledges receipt of notices for the Open Source Components for the initial delivery of the Software.
+
 // License can be found in OpenFace-license.txt
 //
 //     * Any publications arising from the use of this software, including but

README.md

@@ -33,7 +33,7 @@ The system is capable of performing a number of facial analysis tasks:
 
 - Gaze tracking (image of it in action)
 
-<img src="https://github.com/TadasBaltrusaitis/OpenFace/blob/master/imgs/gaze_ex.png" height="378" width="567" >
+<img src="https://github.com/TadasBaltrusaitis/OpenFace/blob/master/imgs/gaze_ex.png" height="182" width="600" >
 
 - Facial Feature Extraction (aligned faces and HOG features)
 
@@ -68,13 +68,17 @@ Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
 in *Facial Expression Recognition and Analysis Challenge*, 
 *IEEE International Conference on Automatic Face and Gesture Recognition*, 2015 
 
-# Final remarks
-
-I did my best to make sure that the code runs out of the box but there are always issues and I would be grateful for your understanding that this is research code and not a commercial level product. However, if you encounter any problems/bugs/issues please contact me on github or by emailing me at [email protected] for any bug reports/questions/suggestions. 
-
 # Copyright
 
 Copyright can be found in the Copyright.txt
 
 You have to respect boost, TBB, dlib, and OpenCV licenses.
 
+# Commercial license
+
+For inquiries about the commercial licensing of the OpenFace toolkit please contact [email protected]
+
+# Final remarks
+
+I did my best to make sure that the code runs out of the box but there are always issues and I would be grateful for your understanding that this is research code and not full fledged product. However, if you encounter any problems/bugs/issues please contact me on github or by emailing me at [email protected] for any bug reports/questions/suggestions. 
+

lib/3rdParty/dlib/CMakeLists.txt

@@ -361,6 +361,16 @@ if (NOT TARGET dlib)
 
       add_library(dlib STATIC ${source_files} )
       target_link_libraries(dlib ${dlib_needed_libraries} )
+      if (UNIX AND NOT DLIB_IN_PROJECT_BUILD)
+         if (DLIB_USE_CUDA)
+            cuda_add_library(dlib_shared SHARED ${source_files} )
+         else()
+            add_library(dlib_shared SHARED ${source_files} )
+         endif()
+         target_link_libraries(dlib_shared ${dlib_needed_libraries} )
+         install(TARGETS dlib_shared LIBRARY DESTINATION lib PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ GROUP_WRITE WORLD_WRITE WORLD_EXECUTE WORLD_READ)
+
+      endif()
 
    endif ()  ##### end of if NOT DLIB_ISO_CPP_ONLY ##########################################################
 
@@ -429,4 +439,4 @@ if (NOT TARGET dlib)
    endif()
 
 
-endif()
+endif()

lib/local/LandmarkDetector/CMakeLists.txt

@@ -1,6 +1,5 @@
 #TBB library
 include_directories(${TBB_ROOT_DIR}/include)
-
 include_directories(${BOOST_INCLUDE_DIR})
 
 SET(SOURCE
@@ -36,6 +35,9 @@ include_directories(./include)
 include_directories(${LandmarkDetector_SOURCE_DIR}/include)
 
 add_library( LandmarkDetector ${SOURCE} ${HEADERS} )
+add_library( LandmarkDetectorShared SHARED ${SOURCE} ${HEADERS} )
+target_link_libraries( LandmarkDetectorShared ${OpenCV_LIBS} ${Boost_LIBRARIES} ${TBB_LIBRARIES} )
 
 install (TARGETS LandmarkDetector DESTINATION lib)
+install (TARGETS  LandmarkDetectorShared LIBRARY DESTINATION lib PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ GROUP_WRITE WORLD_WRITE WORLD_EXECUTE WORLD_READ)
 install (FILES ${HEADERS} DESTINATION include/OpenFace)

python/CMakeLists.txt

@@ -0,0 +1,29 @@
+# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
+
+find_package( Boost 1.5.9 REQUIRED COMPONENTS filesystem system python)
+
+# Python stuff.
+find_package(PythonLibs 2.7 REQUIRED)
+include_directories(${PYTHON_INCLUDE_DIRS})
+
+# TBB library.
+include_directories(${TBB_ROOT_DIR}/include)
+
+# Local libraries.
+include_directories(${LandmarkDetector_SOURCE_DIR}/include)
+
+include_directories(../lib/local/LandmarkDetector/include)
+include_directories(../lib/local/FaceAnalyser/include)  
+
+add_library(pyopenface SHARED pyopenface.cpp detector.cpp tracker.cpp)
+target_link_libraries(pyopenface LandmarkDetectorShared)
+target_link_libraries(pyopenface ${OpenCV_LIBS} ${Boost_LIBRARIES} ${PYTHON_LIBRARIES} ${TBB_LIBRARIES})
+target_link_libraries(pyopenface dlib_shared)
+
+set_property(TARGET pyopenface PROPERTY CXX_STANDARD 11)
+set_property(TARGET pyopenface PROPERTY SUFFIX .so)
+set_property(TARGET pyopenface PROPERTY PREFIX "")
+
+install(TARGETS pyopenface LIBRARY DESTINATION /usr/lib/python2.7/dist-packages PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ GROUP_WRITE WORLD_WRITE WORLD_EXECUTE WORLD_READ)
+

python/README.md

@@ -0,0 +1,36 @@
+# Testing 
+```
+  python python/test.py
+
+```
+# Depend Boost-python and numpy
+### On Ubuntu 14.04
+```
+    sudo apt-get install libboost-all-dev python-dev
+```
+### On Mac
+```
+   TODO
+```
+
+# PyOpenFace
+## Detector
+### class instance
+    detector = pyopenface.Detector(base_path + "OpenFace/lib/local/LandmarkDetector/model/main_clnf_general.txt")
+### get face rect
+    max_face_rect = detector.detect(img_gray)
+### get face landmark
+    re = detector.landmark(img, max_face_rect)
+    face_keypoints = [(int(kp[0]), int(kp[1])) for kp in zip(re[0], re[1])]
+
+## Tracker()
+### class instance
+    tracker = pyopenface.Tracker()
+### tracking with frame
+    tr_results = tracker.tracking(img_gray)
+### get face rect
+    face_rects = tr_results[0]
+### get face landmark
+    face_keypoints = tr_results[1]
+    face_keypoints = [(int(kp[0]), int(kp[1])) for kp in zip(face_keypoints[0], face_keypoints[1])]
+

python/detector.cpp

@@ -0,0 +1,111 @@
+#include "detector.hpp"
+
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+#include <LandmarkCoreIncludes.h>
+#include <dlib/image_processing/frontal_face_detector.h>
+
+using std::cout;
+using std::string;
+using std::vector;
+
+Detector * Detector::Create(const char *binary_path) {
+
+  vector<string> arguments;
+  arguments.push_back(string("  ")); // if CPP this is the application name
+  arguments.push_back(string("-mloc"));
+  arguments.push_back(string(binary_path));
+
+  LandmarkDetector::FaceModelParameters det_parameters(arguments);
+  det_parameters.track_gaze = false;
+  // No need to validate detections, as we're not doing tracking.
+  det_parameters.validate_detections = false;
+
+  // Grab camera parameters, if they are not defined 
+  // (approximate values will be used).
+  float fx = 0, fy = 0, cx = 0, cy = 0;
+  int device = -1;
+  // Get camera parameters
+  LandmarkDetector::get_camera_params(device, fx, fy, cx, cy, arguments);
+  // If cx (optical axis centre) is undefined will use the image size/2 as 
+  // an estimate.
+  bool cx_undefined = false;
+  bool fx_undefined = false;
+  if (cx == 0 || cy == 0) {
+    cx_undefined = true;
+  }
+  if (fx == 0 || fy == 0) {
+    fx_undefined = true;
+  }
+
+  // The modules that are being used for tracking.
+  LandmarkDetector::CLNF clnf_model(det_parameters.model_location);
+
+  cv::CascadeClassifier classifier(det_parameters.face_detector_location);
+  dlib::frontal_face_detector face_detector_hog = dlib::get_frontal_face_detector();
+
+  return new Detector(det_parameters, clnf_model, classifier, face_detector_hog);
+
+}
+
+Detector::Detector(LandmarkDetector::FaceModelParameters &det_parameters,
+                   LandmarkDetector::CLNF &clnf_model,
+                   cv::CascadeClassifier &classifier,
+                   dlib::frontal_face_detector &face_detector_hog) : det_parameters_(std::move(det_parameters)), clnf_model_(std::move(clnf_model)),
+                                                                     classifier_(std::move(classifier)),
+                                                                     face_detector_hog_(std::move(face_detector_hog)) {}
+
+bool CompareRect(cv::Rect_<double> r1, cv::Rect_<double> r2) {
+
+  return r1.height < r2.height;
+
+}
+
+cv::Rect_<double> Detector::DetectFace(const cv::Mat &grayscale_frame) {
+
+  vector<cv::Rect_<double>> face_detections;
+
+  if(det_parameters_.curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR) {
+    vector<double> confidences;
+    //cout << "  DetectFacesHOG" << std::endl;
+    LandmarkDetector::DetectFacesHOG( face_detections, grayscale_frame, face_detector_hog_, confidences);
+  } else {
+    //cout << "  DetectFaces" << std::endl;
+    LandmarkDetector::DetectFaces( face_detections, grayscale_frame, classifier_);
+  }
+
+  // Finding the biggest face among the detected ones.
+  // cout << "  Find biggest face" << std::endl;
+  if (face_detections.empty()) {
+      std::cout<< " No faces detected " << std::endl;
+    //throw std::invalid_argument("No faces detected");
+  }
+  cv::Rect_<double> face_l = *max_element( face_detections.begin(), face_detections.end(), CompareRect);
+
+  // Return the biggest face.
+  return face_l;
+
+}
+
+cv::Mat_<int> Detector::GetVisibilities() {
+  int idx = clnf_model_.patch_experts.GetViewIdx(clnf_model_.params_global, 0);
+  return clnf_model_.patch_experts.visibilities[0][idx];
+}
+
+cv::Mat_<double> Detector::Run(const cv::Mat &grayscale_frame, const cv::Rect_<double> &face_rect){
+
+  cv::Mat_<float> depth_image;
+  bool success = LandmarkDetector::DetectLandmarksInImage( grayscale_frame, depth_image, face_rect, clnf_model_, det_parameters_);
+
+  if (!success) {
+    throw std::runtime_error("Unable to detect landmarks");
+  }
+
+  cv::Mat_<double> landmarks_2d = clnf_model_.detected_landmarks;
+  landmarks_2d = landmarks_2d.reshape(1, 2);
+
+  return landmarks_2d;
+
+}

python/detector.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+#include <opencv2/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/core.hpp>
+#include <LandmarkCoreIncludes.h>
+
+using std::unique_ptr;
+
+class Detector {
+
+public:
+  static Detector * Create(const char *binary_path);
+  cv::Mat_<double> Run(const cv::Mat &input_frame, const cv::Rect_<double> &face_rect);
+  cv::Rect_<double> DetectFace(const cv::Mat &grayscale_frame);
+
+  cv::Mat_<uchar> grayscale_frame_;
+
+private:
+  Detector(LandmarkDetector::FaceModelParameters &det_parameters,
+           LandmarkDetector::CLNF &clnf_model,
+           cv::CascadeClassifier &classifier,
+           dlib::frontal_face_detector &face_detector_hog);
+
+  cv::Mat_<int> GetVisibilities();
+
+  LandmarkDetector::FaceModelParameters det_parameters_;
+  LandmarkDetector::CLNF clnf_model_;
+  cv::CascadeClassifier classifier_;
+  dlib::frontal_face_detector face_detector_hog_;
+
+};