Merge branch 'next' of git://github.com/glinscott/leela-chess into next

lc0/README.md

@@ -1,26 +1,41 @@
-# Building and running this version of lc0.
+# Building and running Lc0
 
-Building is not very streightforward, but here's roughly the process:
+Building right now may be a bit rough, we are improving and finding ways to simplifying the build process.
 
 ## Linux
 
-1. (if you want version with tensorflow) Install tensorflow_cc by following steps described [here](https://github.com/FloopCZ/tensorflow_cc).
+1. (if you want version with tensorflow) Install `tensorflow_cc` by following steps described [here](https://github.com/FloopCZ/tensorflow_cc).
 2. (if you want cuDNN version) Install CUDA and cuDNN.
-3. Install ninja and meson
+3. Install ninja build (`ninja-build`), meson, and (optionally) gtest (`libgtest-dev`).
 4. Go to lc0/
-5. If you decided not to install tensorflow or CUDA, comment out building network_tf.cc and/or network_cudnn.cu from meson.build.
+5. If you decided not to install tensorflow or CUDA, comment out building `network_tf.cc` and/or `network_cudnn.cu` from meson.build.
 6. Run ./build.sh
 
+If you want to build with a different compiler, pass the `CC` and `CXX` environment variables:
+
+    CC=clang-6.0 CXX=clang++-6.0 ./build.sh
+
+### Ubuntu 16.04
+
+For Ubuntu 16.04 you need the latest version of meson and clang-6.0 before performing the steps above:
+
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+    sudo apt-get update
+    sudo apt-get install clang-6.0 ninja-build
+    pip3 install meson --user
+    CC=clang-6.0 CXX=clang++-6.0 INSTALL_PREFIX=~/.local ./build.sh
+
+Make sure that `~/.local/bin` is in your `PATH` environment variable. You can now type `lc0 --help` and start.
+
+
 ## Windows
 
 Building for windows is currently complicated, you can try executeing steps listed [here](https://github.com/glinscott/leela-chess/issues/334#issuecomment-382848569).
 
 Alternatively, you can use pre-built binary:
 
 1. Install CUDA v9.0 (not 9.1)
-2. Inscall cuDNN for CUDA v9.0
+2. Install cuDNN for CUDA v9.0
 3. Take latest binary from [here](https://crem.xyz/lc0/)
 
-That's it.
-
 For the cudnn (non-TF) version, building is much simpler. Install latest CUDA toolkit and compatible cuDNN, and build using the visual studio project.

lc0/build-cuda.cmd

@@ -0,0 +1,16 @@
+rd /s build
+
+call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
+meson.py build --backend vs2015 --buildtype release ^
+-Dcudnn_libdirs="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\lib\x64","C:\dev\cuDNN\cuda\lib\x64" ^
+-Dcudnn_include="C:\dev\cuDNN\cuda\include"
+
+pause
+
+cd build
+"C:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe" ^
+/p:Configuration=Release ^
+/p:Platform=x64 ^
+/p:PreferredToolArchitecture=x64 lc0@exe.vcxproj ^
+/filelogger
+

lc0/build-tensorflow.cmd

@@ -0,0 +1,16 @@
+rd /s build
+
+call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
+meson.py build --backend vs2015 --buildtype release ^
+-Dtensorflow_libdirs= ^
+-Dtensorflow_include=""
+
+pause
+
+cd build
+"C:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe" ^
+/p:Configuration=Release ^
+/p:Platform=x64 ^
+/p:PreferredToolArchitecture=x64 lc0@exe.vcxproj ^
+/filelogger
+

lc0/build.sh

@@ -1,9 +1,26 @@
-#!/bin/bash
-
-rm -fr build
-CC=clang CXX=clang++ meson build --buildtype release # -Db_ndebug=true
-# CC=clang CXX=clang++ meson build --buildtype debugoptimized -Db_asneeded=false
-# CC=clang CXX=clang++ meson build --buildtype debug
-cp testdata/* build
-cd build
-ninja
+#!/usr/bin/env bash
+
+set -e
+
+BUILDTYPE=$1
+
+if [ -z "${BUILDTYPE}" ]
+then
+  BUILDTYPE=release
+fi
+
+BUILDDIR=build/${BUILDTYPE}
+
+rm -fr ${BUILDDIR}
+meson ${BUILDDIR} --buildtype ${BUILDTYPE} --prefix ${INSTALL_PREFIX:-/usr/local}
+
+pushd ${BUILDDIR}
+
+if [ -n "${INSTALL_PREFIX}" ]
+then
+  ninja install
+else
+  ninja
+fi
+
+popd

lc0/meson.build

@@ -1,47 +1,51 @@
-project('lc0', 'cpp', default_options : ['cpp_std=c++14'])
+# This file is part of Leela Chess Zero.
+# Copyright (C) 2018 The LCZero Authors
+#
+# Leela Chess is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Leela Chess is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Leela Chess.  If not, see <http://www.gnu.org/licenses/>.
+
+project('lc0', 'cpp',
+        default_options : ['cpp_std=c++14', 'b_ndebug=if-release'],
+        meson_version: '>=0.45')
 
-add_global_arguments('-Wthread-safety', language : 'cpp')
 cc = meson.get_compiler('cpp')
+if cc.get_id() == 'clang'
+  # Thread safety annotation
+  add_project_arguments('-Wthread-safety', language : 'cpp')
+endif
+if cc.get_id() == 'clang' or cc.get_id() == 'gcc'
+  add_project_arguments('-Wextra', language : 'cpp')
+  add_project_arguments('-pedantic', language : 'cpp')
+
+  if get_option('buildtype') == 'release'
+    add_project_arguments('-march=native', language : 'cpp')
+    add_project_arguments('-mtune=native', language : 'cpp')
+  endif
+endif
 
-# Installed from https://github.com/FloopCZ/tensorflow_cc
-tensorflow_cc = declare_dependency(
-  include_directories: include_directories(
-    '/usr/local/include/tensorflow',
-    '/usr/local/include/tensorflow/bazel-genfiles',
-    '/usr/local/include/tensorflow/tensorflow/contrib/makefile/downloads',
-    '/usr/local/include/tensorflow/tensorflow/contrib/makefile/downloads/eigen',
-    '/usr/local/include/tensorflow/tensorflow/contrib/makefile/downloads/gemmlowp',
-    '/usr/local/include/tensorflow/tensorflow/contrib/makefile/downloads/nsync/public',
-    '/usr/local/include/tensorflow/tensorflow/contrib/makefile/gen/protobuf-host/include',
-  ),
-  dependencies: [
-      cc.find_library('libtensorflow_cc', dirs: '/usr/local/lib/tensorflow_cc/'),
-      cc.find_library('dl'),
-      cc.find_library('pthread'),
-      cc.find_library('libprotobuf', dirs: '/usr/local/lib/tensorflow_cc/'),
-  ],
-)
 
+# Files to compile.
 deps = []
-deps += tensorflow_cc
-deps += cc.find_library('stdc++fs')
-deps += cc.find_library('pthread')
-deps += cc.find_library('libcublas', dirs: ['/opt/cuda/lib64/', '/usr/local/cuda/lib64/'])
-deps += cc.find_library('libcudnn', dirs: ['/opt/cuda/lib64/', '/usr/local/cuda/lib64/'])
-deps += cc.find_library('libcudart', dirs: ['/opt/cuda/lib64/', '/usr/local/cuda/lib64/'])
-# deps += cc.find_library('libprofiler', dirs: ['/usr/local/lib'])
-
-nvcc = find_program('nvcc')
-cuda_files = [
-  'src/neural/network_cudnn.cu',
-]
+files = []
+includes = []
+has_backends = false
 
-cuda_gen = generator(nvcc,
-    output: '@[email protected]',
-    arguments: ['--std=c++14', '-c', '@INPUT@', '-o', '@OUTPUT@', '-I', '../src'],
-)
 
-files = [
+#############################################################################
+## Main files
+#############################################################################
+files += [
+  'src/engine.cc',
   'src/analyzer/analyzer.cc',
   'src/analyzer/table.cc',
   'src/chess/bitboard.cc',
@@ -54,41 +58,153 @@ files = [
   'src/neural/encoder.cc',
   'src/neural/factory.cc',
   'src/neural/loader.cc',
-  'src/neural/writer.cc',
   'src/neural/network_mux.cc',
   'src/neural/network_random.cc',
-  'src/neural/network_tf.cc',
+  'src/neural/writer.cc',
+  'src/selfplay/game.cc',
+  'src/selfplay/loop.cc',
+  'src/selfplay/tournament.cc',
   'src/utils/commandline.cc',
   'src/utils/optionsdict.cc',
   'src/utils/optionsparser.cc',
   'src/utils/random.cc',
   'src/utils/string.cc',
   'src/utils/transpose.cc',
-  'src/engine.cc',
-  'src/selfplay/game.cc',
-  'src/selfplay/tournament.cc',
-  'src/selfplay/loop.cc',
-  cuda_gen.process(cuda_files)
 ]
-
-includes = []
 includes += include_directories('src')
 
+
+#############################################################################
+## Platform specific files
+############################################################################
+if host_machine.system() == 'windows'
+  files += 'src/utils/filesystem.win32.cc'
+else
+  files += 'src/utils/filesystem.posix.cc'
+  deps += [
+    cc.find_library('pthread'),
+    ]
+endif
+
+
+
+#############################################################################
+## BACKENDS
+#############################################################################
+
+if get_option('build_backends')
+  ## ~~~~~~~~~~
+  ## Tensorflow
+  ## ~~~~~~~~~~
+  # Installed from https://github.com/FloopCZ/tensorflow_cc
+  tensorflow_include = get_option('tensorflow_include')
+  tensorflow_libdir = get_option('tensorflow_libdir')
+  tf_dl_lib = cc.find_library('dl', required: false)
+  tf_tensorflow_lib = cc.find_library('libtensorflow_cc',
+            dirs: tensorflow_libdir, required: false)
+  tf_protobuf_lib = cc.find_library('libprotobuf',
+            dirs: tensorflow_libdir, required: false)
+
+  if tf_dl_lib.found() and tf_tensorflow_lib.found() and tf_protobuf_lib.found()
+    includes += include_directories(
+      tensorflow_include,
+      tensorflow_include[0] + '/bazel-genfiles',
+      tensorflow_include[0] + '/tensorflow/contrib/makefile/downloads',
+      tensorflow_include[0] + '/tensorflow/contrib/makefile/downloads/eigen',
+      tensorflow_include[0] + '/tensorflow/contrib/makefile/downloads/gemmlowp',
+      tensorflow_include[0] + '/tensorflow/contrib/makefile/downloads/nsync/public',
+      tensorflow_include[0] + '/tensorflow/contrib/makefile/gen/protobuf-host/include',
+      is_system: true
+    )
+    deps += [tf_dl_lib, tf_tensorflow_lib, tf_protobuf_lib]
+    files += 'src/neural/network_tf.cc'
+    has_backends = true
+  endif
+
+
+  ## ~~~~~
+  ## cuDNN
+  ## ~~~~~
+  cudnn_libdirs = get_option('cudnn_libdirs')
+  cu_blas = cc.find_library('cublas', dirs: cudnn_libdirs, required: false)
+  cu_dnn = cc.find_library('cudnn', dirs: cudnn_libdirs, required: false)
+  cu_dart = cc.find_library('cudart', dirs: cudnn_libdirs, required: false)
+  nvcc = find_program('/usr/local/cuda-9.2/bin/nvcc',
+                      '/usr/local/cuda-9.1/bin/nvcc',
+                      'nvcc', required: false)
+
+  cuda_files = [
+    'src/neural/network_cudnn.cu',
+  ]
+
+  if cu_blas.found() and cu_dnn.found() and cu_dart.found() and nvcc.found()
+    includes += include_directories(get_option('cudnn_include'))
+    deps += [cu_blas, cu_dnn, cu_dart]
+  cuda_arguments = ['-c', '@INPUT@', '-o', '@OUTPUT@',
+                    '-I', meson.current_source_dir() + '/src']
+  if host_machine.system() == 'windows'
+    cuda_arguments += ['-Xcompiler', '-MD']
+  else
+    cuda_arguments += ['--std=c++14', '-Xcompiler', '-fPIC']
+  endif
+  foreach x : get_option('cudnn_include')
+     cuda_arguments += ['-I', x]
+  endforeach
+  if host_machine.system() == 'windows'
+    outputname = '@[email protected]'
+  else
+    outputname = '@[email protected]'
+  endif
+  cuda_gen = generator(nvcc,
+      output: outputname,
+      arguments: cuda_arguments,
+  )
+    files += cuda_gen.process(cuda_files)
+    has_backends = true
+  endif
+
+endif # if get_option('build_backends')
+
+if not has_backends and get_option('build_backends')
+  error('''
+
+        No usable computation backends (cudnn/tensorflow/etc) are found. 
+        If you want to build it with random only backend, pass
+        -D build_backends=false to a meson build.''')
+endif
+
+
+#############################################################################
+## Profiler
+#############################################################################
+if get_option('buildtype') != 'release'
+  deps += cc.find_library('libprofiler',
+    dirs: ['/usr/local/lib'], required: false)
+endif
+
+
+#############################################################################
+## Main Executable
+#############################################################################
+
 executable('lc0', 'src/main.cc',
-  files, include_directories: includes, dependencies: deps)
+  files, include_directories: includes, dependencies: deps, install: true)
 
 
 ### Tests
+gtest = dependency('gtest', required: false)
 
 test_deps = deps
-test_deps += dependency('gtest')
+if gtest.found()
+  test_deps += gtest
 
-test('ChessBoard',
-  executable('chessboard_test', 'src/chess/board_test.cc',
-  files, include_directories: includes, dependencies: test_deps
-))
+  test('ChessBoard',
+    executable('chessboard_test', 'src/chess/board_test.cc',
+    files, include_directories: includes, dependencies: test_deps
+  ))
 
-test('HashCat',
-  executable('hashcat_test', 'src/utils/hashcat_test.cc',
-  files, include_directories: includes, dependencies: test_deps
-))
+  test('HashCat',
+    executable('hashcat_test', 'src/utils/hashcat_test.cc',
+    files, include_directories: includes, dependencies: test_deps
+  ))
+endif