From 5fb6f96cfa5e7c884f05060417a2ab3ca979021b Mon Sep 17 00:00:00 2001
From: Maxim Yurkin <yurkin@gmail.com>
Date: Wed, 16 Dec 2020 02:26:21 +0700
Subject: [PATCH] A lot of changes to build and packaging scripts. Fixes #210
 (finalizes).

---
 devtools/README.md            | 14 ++++++
 devtools/build                |  6 ++-
 devtools/build_debug          | 38 +++++++++++++++
 devtools/build_misc           | 41 ++++++++++++++++
 devtools/commit_docs          | 23 +++++++++
 devtools/make_branch          | 23 ---------
 devtools/make_tag             | 21 --------
 devtools/release_sequence.txt | 92 +++++++++++++++++++----------------
 devtools/test_new             |  2 +-
 devtools/win_all.sh           | 31 ++++++------
 devtools/win_build_adda.sh    | 50 -------------------
 devtools/win_build_misc.sh    | 49 -------------------
 devtools/win_commit.sh        | 13 +++--
 devtools/zip_packages         | 33 +++----------
 14 files changed, 202 insertions(+), 234 deletions(-)
 create mode 100644 devtools/README.md
 create mode 100644 devtools/build_debug
 create mode 100644 devtools/build_misc
 create mode 100644 devtools/commit_docs
 delete mode 100755 devtools/make_branch
 delete mode 100755 devtools/make_tag
 delete mode 100644 devtools/win_build_adda.sh
 delete mode 100644 devtools/win_build_misc.sh

diff --git a/devtools/README.md b/devtools/README.md
new file mode 100644
index 00000000..c84e8ab9
--- /dev/null
+++ b/devtools/README.md
@@ -0,0 +1,14 @@
+Tools used by ADDA developers for building, testing, and packaging of ADDA:
+* `build` - script to build ADDA with various flags and optionally copy the obtained executables
+* `build_degug` - compiles ADDA (in debug mode) with various compilation flags
+* `build_misc` - builds misc tools and optionally copies (installs) the obtained executables
+* `commit_docs` - script to commit `docs/` and `src/const.h` for release
+* `release_sequence.txt` - sequence of tasks to be made during release
+* `svg_images.txt` - typical workflow (guidelines) for producing `.svg` images
+* `test_new` - simple wrapper to build and test ADDA
+* `versions.txt` - note on files that need to be checked prior to release (e.g. contain version number)
+* `win_all.sh` - script to perform all Windows-specific tasks during release
+* `win_commit.sh` - script to run a sample simulation and commit `sample/` and  `win64/`
+* `zip_packages` - script to export (archive) source and binary packages for a given version
+
+
diff --git a/devtools/build b/devtools/build
index 2359f858..15fda0fe 100755
--- a/devtools/build
+++ b/devtools/build
@@ -1,7 +1,9 @@
 #!/bin/bash
+# Builds several versions of ADDA and optionally copies (installs) the obtained executables
 # First optional parameter is compilation modes: seq, mpi, ocl, spa, any combination of them (separated by space in
 # quotes, implies cross modes), or all (default). 'spa' is sparse and requires combination with something.
 # Second optional parameter is destination path to copy executables (otherwise, they are left in compilation folders)
+# Compilation is performed silently with (-s flag)
 #
 # Copyright (C) ADDA contributors
 # GNU General Public License version 3
@@ -11,7 +13,7 @@ MODES=${1:-all}
 DEST=""
 if [ -n "$2" ]; then
   if [ -d "$2" ]; then
-    DEST=`cd "$2"; pwd`
+    DEST="`cd "$2"; pwd`"
   else
     echo "ERROR: could not find the directory '$2'" >&2
     exit 1
@@ -43,7 +45,7 @@ function check_mode {
 }
 function compile_copy {
   # $1 is mode, $2 - name of executable, $3 - extra options
-  if ! make $1 $3 $XFL; then
+  if ! make -s $1 $3 $XFL; then
     echo "ERROR during compiling ADDA with options '$1 $3 $XFL'" >&2
     exit 1
   fi
diff --git a/devtools/build_debug b/devtools/build_debug
new file mode 100644
index 00000000..fb9df271
--- /dev/null
+++ b/devtools/build_debug
@@ -0,0 +1,38 @@
+#!/bin/bash
+# Compiles ADDA (in debug mode) with various compilation flags - used to find compilation warnings
+# Compilation is performed silently with (-s flag)
+#
+# All warnings and errors go to stderr, so one may look mostly at this stream to catch failures 
+# (i.e. redirect it to file), however it is commonly colorized by make, so should be OK as is
+#
+# Copyright (C) ADDA contributors
+# GNU General Public License version 3
+
+function compile_debug {
+  # $1 is mode, $2 - extra options
+  if ! make -s $1 OPTIONS="DEBUG $2"; then
+    echo "ERROR during compiling ADDA with options '$1 OPTIONS=\"DEBUG $2\"'" >&2
+    exit 1
+  fi
+}
+
+# main code; we do not do any special cleaning, since believe that make tracks all dependences
+cd ./../src
+compile_debug all ""
+compile_debug all "DEBUGFULL"
+compile_debug all "FFT_TEMPERTON"
+compile_debug all "NO_FORTRAN"
+compile_debug all "PRECISE_TIMING"
+compile_debug all "USE_SSE3"
+
+compile_debug seq "NOT_USE_LOCK"
+compile_debug seq "ONLY_LOCKFILE"
+compile_debug seq "OVERRIDE_STDC_TEST"
+compile_debug seq "NO_GITHASH"
+
+compile_debug ocl "OCL_READ_SOURCE_RUNTIME"
+compile_debug ocl "OCL_BLAS"
+compile_debug ocl "CLFFT_APPLE"
+
+compile_debug "seq mpi" "SPARSE"
+compile_debug "seq mpi" "SPARSE USE_SSE3"
\ No newline at end of file
diff --git a/devtools/build_misc b/devtools/build_misc
new file mode 100644
index 00000000..85b80656
--- /dev/null
+++ b/devtools/build_misc
@@ -0,0 +1,41 @@
+#!/bin/bash
+# Builds misc tools and optionally copies (installs) the obtained executables
+# The only optional parameter is destination path to copy executables (otherwise, they are left in compilation folders)
+# Executables of each misc package are copied into a separate directory in this path
+#
+# Copyright (C) ADDA contributors
+# GNU General Public License version 3
+
+# process input variables
+MISCDIR="`pwd`/../misc"
+DEST="$MISCDIR"
+if [ -n "$1" ]; then
+  if [ -d "$1" ]; then
+    DEST="`cd "$1"; pwd`"
+  else
+    echo "ERROR: could not find the directory '$1'" >&2
+    exit 1
+  fi
+fi
+
+MISCDIR="`pwd`/../misc"
+
+for dir in `ls "$MISCDIR"`; do
+  MAKEFILE="$MISCDIR/$dir/Makefile"
+  if [ -f "$MAKEFILE" ]; then
+    echo "Processing misc/$dir"
+    # The following is a bit complicated. Ideally should be replaced by something like
+    # make install
+    dest="$DEST/$dir"
+    if [ ! -d "$dest" ]; then
+        mkdir "$dest"
+    fi
+    cd "$dest"
+    make -f "$MAKEFILE" srcdir="$MISCDIR/$dir"
+    if [ $? -ne 0 ]; then
+      echo "ERROR: compilation in 'misc/$dir' failed"
+      exit 1
+    fi
+    rm -f *.o
+  fi
+done
diff --git a/devtools/commit_docs b/devtools/commit_docs
new file mode 100644
index 00000000..01c3e1b1
--- /dev/null
+++ b/devtools/commit_docs
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Commits docs and const.h for release
+# Accepts exactly one argument - version number (like 1.4.0)
+
+# Copyright (C) ADDA contributors
+# GNU General Public License version 3
+
+# Test arguments
+if [ $# -ne 1 ]; then
+  echo "ERROR: requires 1 argument"
+  exit 1
+fi
+
+git pull
+if [ $? -ne 0 ]; then
+  echo "ERROR: error during git pull"
+  exit 1
+fi
+git commit -m "Preparing file for release $1: docs and const.h" ../doc/manual.docx ../doc/manual.pdf ../doc/history ../src/const.h 
+if [ $? -ne 0 ]; then
+  echo "ERROR: error during commiting docs"
+  exit 1
+fi
\ No newline at end of file
diff --git a/devtools/make_branch b/devtools/make_branch
deleted file mode 100755
index a84b91f8..00000000
--- a/devtools/make_branch
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-# Creates a new branch in svn, using the current trunk.
-# Two obligatory argument:
-# 1) name of branch (like new_feature)
-# 2) commit message (a short description of the branch)
-
-# Test arguments
-if [ $# -ne 2 ]; then
-  echo "ERROR: requires 2 arguments"
-  exit 1
-fi
-
-# define variables
-URL="https://a-dda.googlecode.com/svn"
-branchname="$1"
-branchURL="$URL/branches/$branchname"
-
-# create new branch
-svn copy -m "$2" "$URL/trunk" "$branchURL"
-if [ $? -ne 0 ]; then
-  echo "ERROR: svn copy failed"
-  exit 1
-fi
diff --git a/devtools/make_tag b/devtools/make_tag
deleted file mode 100755
index 9f08d8db..00000000
--- a/devtools/make_tag
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-# Creates a new (version) tag in svn, using the current trunk.
-# Single obligatory argument - name of version (like 1.0)
-
-# Test arguments
-if [ $# -ne 1 ]; then
-  echo "ERROR: requires 1 arguments"
-  exit 1
-fi
-
-# define variables
-URL="https://a-dda.googlecode.com/svn"
-tagname=rel_"$1"
-tagURL="$URL/tags/$tagname"
-
-# create new tag
-svn copy -m "new release $1" "$URL/trunk" "$tagURL"
-if [ $? -ne 0 ]; then
-  echo "ERROR: svn copy failed"
-  exit 1
-fi
diff --git a/devtools/release_sequence.txt b/devtools/release_sequence.txt
index 18a39739..15a1bae7 100644
--- a/devtools/release_sequence.txt
+++ b/devtools/release_sequence.txt
@@ -1,41 +1,51 @@
-1) test the current source (standard tests)
-> cd ..../adda/src
-> make -s
-> cd ../tests/2exec
-edit comp2exec to compare against previous version
-> sh comp2exec seq, mpi, ocl + mpi_seq, ocl_seq
-
-# extra surface tests (optional, causes a number of false positives)
-edit comp2exec to enable SURF_EXT
-> sh comp2exec seq, mpi, ocl
-edit comp2exec to enable SURF_STANDARD (instead of SURF_EXT) and to compare against adda, etc. in the test directory (.)
-> cp ../../src/seq/adda .
-> cp ../../src/mpi/adda_mpi .
-> cp ../../src/ocl/adda_ocl .
-> sh comp2exec seq, mpi, ocl
-
-# sparse tests (first two lines are not required if done above)
-#> cp ../../src/seq/adda .
-#> cp ../../src/mpi/adda_mpi .
-> cd ../../src
-> make seq OPTIONS=SPARSE -s
-> cd ../tests/2exec
-edit comp2exec to enable SPARSE
-> sh comp2exec seq, mpi
-edit comp2exec to compare against adda and adda_mpi + enable SPARSE_STANDARD
-> sh comp2exec seq, mpi
-
-2) make sure that manual.doc/pdf, history, and const.h are up to date. In particular, version number in const.h should
-not contain "a" or "b" (if not explicitly making beta release).
-
-3) Update CodeDesign and corresponding schemes in doc/gv/
-
-(on 64-bit Windows)
-4) sh win_all.sh #.#
-
-(on Linux)
-5) ./zip_packages #.#
-
-6) Upload packages to GoogleCode
-7) Update Wiki pages: PackageDescription, ReleaseNotes, Features
-8) Send announcement to users.
\ No newline at end of file
+Use some normal shell, under Windows - this can be MSYS. Then 'sh' can be omitted in the following commands.
+
+1) Test for compiler warnings (about 10 min)
+> sh build_debug 2> log_debug
+
+2) Build and test the current source (a few hours)
+> sh test_new 2> log_test
+Before running the tests, make sure that ../tests/2exec/comp2exec script is properly set up. This includes
+the REFPATH (when comparing against previous version) and GUIDIFF. If the latter is used, you generally do
+not need to redirect stdout.
+It is also possible to separately run build and ../tests/2exec/test_all scripts (see help inside them)
+
+3) If any changes are made to the code or tests, commit them now. 
+   It is also a good idea to repeat some of the above on different systems.
+
+4) make sure that manual.doc/pdf, history, and const.h are up to date. See also versions.txt
+- version number in const.h should not contain "alpha" or "beta" (if not explicitly making beta release).
+- history should contain version date of the in-progress release.
+- manual should have correct version (including links on the first page). Look through the whole manual
+  (for figure placement, etc.). Update sample outputs in appendices, if needed. 
+- use doiLink macro in Word and produce pdf (better with Actobat plugin - leads to twice smaller size). 
+  Look through the pdf once more.
+
+5) Update CodeDesign and corresponding schemes in doc/gv/
+
+6) Make sure that the DLLs in win64/ correspond to the latest executables (to be copied there). They can be 
+   staged for commit (including deleted older ones) - then they will be committed together with executables.  
+
+7) Run the following aggregate script (on Windows), inserting appropriate version number:
+> sh win_all.sh #.#.#
+You can also go through it line by line.
+
+8) It is a good idea to test that the resulting executables can be run with given DLLs. For example you can run
+'set PATH=""' or 'export PATH=""' in a terminal before running the executables (to make other DLLs on your system
+unavailable).
+
+9) Now you are ready to go live (check your commits):
+> git push --tags
+
+10) Prepare binary and source packages
+> zip_packages #.#.#
+
+11) Create release at GitHub using this tag (Release notes is a shortened version of history). Attach the above 
+packages to it.
+
+12) Update wiki page Features
+
+13) Send announcement to users.
+
+14) With the next commit update ignore patterns in tests/2exec/comp2exec and, possibly, testing suites to compare 
+against the latest release
\ No newline at end of file
diff --git a/devtools/test_new b/devtools/test_new
index 9e724800..56cda195 100755
--- a/devtools/test_new
+++ b/devtools/test_new
@@ -26,7 +26,7 @@ function test_error {
   fi
 }
 
-# uncomment any of the following for quicker tests
+# comment out any of the following for quicker tests
 test_error prev
 test_error cross
 test_error extra
diff --git a/devtools/win_all.sh b/devtools/win_all.sh
index ea96a61f..a950f808 100644
--- a/devtools/win_all.sh
+++ b/devtools/win_all.sh
@@ -1,7 +1,10 @@
 #!/bin/bash
-# Master script to builds and commit Windows executables, all functionality is inside child scripts
-# Accepts exactly one argument - version number (like 1.0)
-# Should be executed on Windows 64-bit
+# Master script to builds and commit Windows executables, most functionality is inside child scripts
+# Accepts exactly one argument - version number (like 1.4.0)
+# Should be executed on Windows
+#
+# Copyright (C) ADDA contributors
+# GNU General Public License version 3
 
 # Test arguments
 if [ $# -ne 1 ]; then
@@ -11,33 +14,27 @@ fi
 
 SHELL=/bin/bash
 
-$SHELL win_build_adda.sh 64
+$SHELL commit_docs $1
 if [ $? -ne 0 ]; then
-  echo "ERROR: error during building 64-bit ADDA"
+  echo "ERROR: error during commiting docs"
   exit 1
 fi
-$SHELL win_build_adda.sh 32
+$SHELL build all ../win64
 if [ $? -ne 0 ]; then
-  echo "ERROR: error during building 32-bit ADDA"
+  echo "ERROR: error during building ADDA"
   exit 1
 fi
-$SHELL win_build_misc.sh 64
+$SHELL build_misc ../win64/misc
 if [ $? -ne 0 ]; then
-  echo "ERROR: error during building 64-bit misc tools"
+  echo "ERROR: error during building misc tools"
   exit 1
 fi
-$SHELL win_build_misc.sh 32
-if [ $? -ne 0 ]; then
-  echo "ERROR: error during building 32-bit misc tools"
-  exit 1
-fi
-
 $SHELL win_commit.sh $1
 if [ $? -ne 0 ]; then
-  echo "ERROR: error during commit"
+  echo "ERROR: error during commit of binaries"
   exit 1
 fi
-$SHELL make_tag $1
+git tag -a v$1 -m "New release $1"
 if [ $? -ne 0 ]; then
   echo "ERROR: creating tag failed"
   exit 1
diff --git a/devtools/win_build_adda.sh b/devtools/win_build_adda.sh
deleted file mode 100644
index bb55f77c..00000000
--- a/devtools/win_build_adda.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-# Builds Windows executables of main ADDA source
-# Accepts exactly one argument - number of bits (32 or 64)
-# Should be run on 64-bit Windows
-
-# Test arguments
-if [ $# -ne 1 ]; then
-  echo "ERROR: requires 1 argument - number of bits for OS (32 or 64)"
-  exit 1
-fi
-if [ $1 -ne "32" -a $1 -ne "64" ]; then
-  echo "ERROR: argument 1 should be either 32 or 64"
-  exit 1
-fi
-
-ROOTDIR=`pwd`/../
-WINDIR="$ROOTDIR/win$1"
-# We assume that compilation is performed on 64 bit system, thus a separate flag is added only for 32-bit case
-if [ $1 -eq "32" ]; then
-  XFL="EXTRA_FLAGS=-m32"
-fi
-
-# Build ADDA (all versions)
-cd "$ROOTDIR/src"
-# update src
-svn update
-if [ $? -ne 0 ]; then
-  echo "ERROR: svn update of src/ failed"
-  exit 1
-fi
-
-# Build FFT versions
-make -s $XFL
-if [ $? -ne 0 ]; then
-  echo "ERROR: compilation of ADDA failed"
-  exit 1
-fi
-# this should be replaced by make install
-cp -p seq/adda.exe mpi/adda_mpi.exe ocl/adda_ocl.exe "$WINDIR"
-
-# Build sparse versions
-make -s $XFL OPTIONS=SPARSE seq mpi
-if [ $? -ne 0 ]; then
-  echo "ERROR: compilation of sparse ADDA failed"
-  exit 1
-fi
-# this should be replaced by make install
-cp -p seq/adda.exe "$WINDIR"/adda_spa.exe
-cp -p mpi/adda_mpi.exe "$WINDIR"/adda_spa_mpi.exe
-
diff --git a/devtools/win_build_misc.sh b/devtools/win_build_misc.sh
deleted file mode 100644
index ed7bf7b2..00000000
--- a/devtools/win_build_misc.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/bash
-# Builds Windows executables of misc tools
-# Accepts exactly one argument - number of bits (32 or 64)
-# Should be run on 64-bit Windows
-
-# Test arguments
-if [ $# -ne 1 ]; then
-  echo "ERROR: requires 1 argument - number of bits for OS (32 or 64)"
-  exit 1
-fi
-if [ $1 -ne "32" -a $1 -ne "64" ]; then
-  echo "ERROR: argument 1 should be either 32 or 64"
-  exit 1
-fi
-
-ROOTDIR=`pwd`/../
-WINDIR="$ROOTDIR/win$1"
-MISCDIR="$ROOTDIR/misc"
-# We assume that compilation is performed on 64 bit system, thus a separate flag is added only for 32-bit case
-if [ $1 -eq "32" ]; then
-  XFL="EXTRA_FLAGS=-m32"
-fi
-
-# Build misc tools
-cd "$ROOTDIR/misc"
-# update misc
-svn update
-if [ $? -ne 0 ]; then
-  echo "ERROR: svn update of misc/ failed"
-  exit 1
-fi
-for dir in `ls $MISCDIR`; do
-  MAKEFILE="$MISCDIR/$dir/Makefile"
-  if [ -f "$MAKEFILE" ]; then
-    echo "Processing misc/$dir"
-    # This is specific for Windows, ideally should be replaced by something like
-    # make install
-    dest="$WINDIR/misc/$dir"
-    if [ ! -d "$dest" ]; then
-        mkdir "$dest"
-    fi
-    cd "$dest"
-    make $XFL -f "$MAKEFILE" srcdir="$MISCDIR/$dir"
-    if [ $? -ne 0 ]; then
-      echo "ERROR: compilation in 'misc/$dir' failed"
-      exit 1
-    fi
-  fi
-done
diff --git a/devtools/win_commit.sh b/devtools/win_commit.sh
index 6c0fa269..22466d40 100644
--- a/devtools/win_commit.sh
+++ b/devtools/win_commit.sh
@@ -1,7 +1,10 @@
 #!/bin/bash
-# Runs a sample simulation and commits Windows executables, sample output, and docs
-# Accepts exactly one argument - version number (like 1.0)
+# Runs a sample simulation and commits Windows executables and sample output
+# Accepts exactly one argument - version number (like 1.4.0)
 # Should be executed on Windows after finishing all builds
+#
+# Copyright (C) ADDA contributors
+# GNU General Public License version 3
 
 # Test arguments
 if [ $# -ne 1 ]; then
@@ -9,7 +12,7 @@ if [ $# -ne 1 ]; then
   exit 1
 fi
 
-ROOTDIR=`pwd`/../
+ROOTDIR="`pwd`/../"
 EXECDIR="$ROOTDIR/win64"
 RUNDIR="run000_sphere_g16_m1.5"
 SAMPLEDIR="$ROOTDIR/sample"
@@ -28,8 +31,8 @@ mv $RUNDIR/* "$SAMPLEDIR/$RUNDIR"
 rm -r ExpCount "$RUNDIR"
 
 cd "$ROOTDIR"
-svn commit -m "Preparing files for release $1: Windows binaries, sample output, and docs" doc/ sample/ win32/ win64/
+git commit -m "Preparing files for release $1: Windows binaries and sample output" sample/ win64/
 if [ $? -ne 0 ]; then
-  echo "ERROR: svn commit failed"
+  echo "ERROR: git commit failed"
   exit 1
 fi
diff --git a/devtools/zip_packages b/devtools/zip_packages
index ef1d6971..e0e743b8 100755
--- a/devtools/zip_packages
+++ b/devtools/zip_packages
@@ -1,11 +1,10 @@
 #!/bin/bash
-# Assumes that a new tag in svn has been allready created.
-# Then creates the ADDA package excluding all the SVN entries. Win32 and Win64 files are zipped separately.
-# maximum compression, recursive
-# Single obligatory argument - name of version (like 1.0)
+# Assumes that a new tag in git has been allready created.
+# Then creates the ADDA package excluding all git and GitHub auxiliary files entries. Win64 files are zipped separately.
+# Single obligatory argument - name of version (like 1.4.0)
 #
-# !!! It is essential to run this script only on Unix system, since the package contain scripts, which has nontrivial
-# permission flags (+x) that should be preserved in the package.
+# !!! We are still struggling to produce correct LF line endings in the main source package, when archiving it on Windows
+# Until this is solved, it is recommende to run the last command on Unix system
 
 #Test arguments
 if [ $# -ne 1 ]; then
@@ -13,23 +12,7 @@ if [ $# -ne 1 ]; then
   exit 1
 fi
  
-# define variables
-URL="http://a-dda.googlecode.com/svn"
-target=\*
-tagname=rel_"$1"
-tagURL="$URL/tags/$tagname"
-filename="../adda_$1"
-# export a clean copy from tag
-svn export "$tagURL"
-if [ $? -ne 0 ]; then
-  echo "ERROR: svn export failed"
-  exit 1
-fi
-# zip general and win packages
-cd "$tagname"
-zip -9 -r "${filename}.zip" $target -x '*win*'
-zip -9 -r "${filename}_win32.zip" win32
-zip -9 -r "${filename}_win64.zip" win64
-# cleanup
 cd ..
-rm -r "$tagname"
+git archive -o adda_${1}_win64.zip v$1:win64 --prefix win64/
+git archive -o adda_${1}.zip v$1
+# -c core.eol=lf
\ No newline at end of file