-
Notifications
You must be signed in to change notification settings - Fork 1
/
2019_installation_notes.sh
1610 lines (1369 loc) · 96.4 KB
/
2019_installation_notes.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#1/31/2019
#running toast in interactive job
msub -I -A b1011 -q short -l nodes=1:ppn=8 -l walltime=01:00:00
source modules_source
export SCRATCH=/projects/b1011/fissel/runfiles/gls
cd $SCRATCH/mickey
module load boost
# dist_chan can't be more than number of cores
mpirun -np 8 -npernode 8 toast_mpi_map mickey_toast_new_test.bin --bin --diagntt --cov --gls --gls_maxiter 60 --gls_dump_iter -1 --rcond 0.01 --dist_chan 8 --out mickey_toast_new_gls_test_20190131
#converged after ~30 iterations
#Then we started rebuilding libraries in a new directory: /projects/b1011/blast-tng/
#First lapack:
module load gcc/4.6.3
module load cmake/3.1.0
mkdir /projects/b1011/blast-tng/lapack-3.6.0/build-2
cd /projects/b1011/blast-tng/lapack-3.6.0/build-2
cmake .. -DCMAKE_C_COMPILER=gcc -DCMAKE_INSTALL_PREFIX=/projects/b1011/blast-tng/software/lapack/3.6.0 -DCMAKE_Fortran_COMPILER=gfortran
#to check other options run ccmake
ccmake
make
make install
#fftw:
module load gcc/4.6.3
module load cmake/3.1.0
mkdir /projects/b1011/blast-tng/fftw-3.3.8/build
cd /projects/b1011/blast-tng/fftw-3.3.8/build
#were there other cmake options used here?
cmake .. -DCMAKE_INSTALL_PREFIX=/projects/b1011/blast-tng/software/fftw/3.3.8
make
make install
#had to repeat to get double and single precision libraries?
#do we need to repeat again to get mpi libraries? I don't see mpi libraries
#cfitsio (used 3.45, the most recent version. 3.37 was no longer available from developer):
module load gcc/4.6.3
module load cmake/3.1.0
mkdir /projects/b1011/blast-tng/cfitsio/build
cd /projects/b1011/blast-tng/cfitsio/build
#other options passed to cmake?
cmake .. -DCMAKE_INSTALL_PREFIX=/projects/b1011/blast-tng/software/cfitsio/3.45
make
make install
# wcslib (used the only available gmake, in /usr/bin/gmake, gmake v3.82):
# on quest make is gmake (realized later)
module load gcc/4.6.3
cd /projects/b1011/blast-tng/wcslib-5.15
./configure --prefix=/projects/b1011/blast-tng/software/wcslib/5.15
gmake
gmake check
gmake install
#install failed, tried to install in /usr/local/bin
#tried adding / to end of prefix
./configure --prefix=/projects/b1011/blast-tng/software/wcslib/5.15/
gmake
gmake check
gmake install
#worked!
#then found old config.log in /projects/b1011/fissel/pascal/wcslib-5.15/config.log, so I am going to use the options there for running configure
#removing previous installation
rm -rf /projects/b1011/blast-tng/software/wcslib/
module load gcc/4.6.3
cd /projects/b1011/blast-tng/wcslib-5.15
./configure --prefix=/projects/b1011/blast-tng/software/wcslib/5.15/ --with-cfitsiolib=/projects/b1011/blast-tng/software/cfitsio/3.45/lib --with-cfitsioinc=/projects/b1011/blast-tng/software/cfitsio/3.45/include --without-pgplot
gmake
gmake install
gmake check
#reports C/twcstab, C/tdis3, C/twcslint, Fortran/twcstab failed (these tests are not done if --with-cfitsiolib/inc options are not passed, in that case it doesn't find the cfitsio installation)
#these tests all report not finding libcfitsio.so.3, but that exists!
#permissions on libcfitsio.so.3 seem fine
#old wcslib passes all checks
#made a module for cfitsio, loaded it (now the libraries for cfitsio are in my LD_LIBRARY_PATH)
module use /projects/b1011/blast-tng/modules
module load cfitsio
./configure --prefix=/projects/b1011/blast-tng/software/wcslib/5.15/ --with-cfitsiolib=/projects/b1011/blast-tng/software/cfitsio/3.45/lib --with-cfitsioinc=/projects/b1011/blast-tng/software/cfitsio/3.45/include --without-pgplot
gmake
gmake check
#all checks passed!
gmake install
# MOAT
module load gcc/4.6.3
cd /projects/b1011/blast-tng/MOAT_3/MOAT
./configure --prefix=/projects/b1011/blast-tng/software/moat/
#part of output
##=========== Build Configuration ===========
##C++ Compiler : g++
##C++ Compile flags : -O3
##F77 Compiler : gfortran
##MPICXX Compiler : g++
##OpenMP : Detected (-fopenmp)
##OpenCL : Disabled
##ACML Vendor Lib : Disabled
##MASS Vendor Lib : Disabled
##MKL Vendor Lib : Disabled
##Apple Vendor Lib : Disabled
##===========================================
make
make install
#then I checked the old moat config.log in /projects/b1011/fissel/pascal/Experimental/MOAT_3/MOAT/config.log
#configure command was: ./configure --prefix=/projects/b1011/fissel/pascal/Experimental/moat_new --with-boost=/projects/b1011/fissel/pascal/Experimental/boost BOOST_ROOT=/projects/b1011/fissel/pascal/Experimental/boost MPICXX=mpic++ CC=gcc CXX=g++ F77=gfortran LIBS=-lgomp --with-fftw-libs=-L/software/FFTW/3.3.3/lib -lfftw3 CFLAGS=-I/software/FFTW/3.3.3/include CXXFLAGS=-I/software/FFTW/3.3.3/include -with-fftw-cpp=-I/software/FFTW/3.3.3/include
#so I need to wait until boost is built and installed to build moat (although make check seems to pass all tests right now, but we want to have boost).
#also need to load mpi, module load mpi/openmpi-1.6.3-gcc.4.6.3
# 2/21/2019
# Boost
module purge
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
cd /projects/b1011/blast-tng/boost_1_61_0
./bootstrap.sh --help
#saw that python is used, should load it so we are consistent
module load python
which python
#output: /software/anaconda2/bin/python
./bootstrap.sh --show-libraries
#lists libraries that require build and installation steps, that is the ones that you can list in --with-libraries
#this list includes mpi, I am going to build them all
./bootstrap.sh --with-libraries=all --prefix=/projects/b1011/blast-tng/software/boost/1.61.0
./b2
./b2 install
# 2/22/2019
# going to try starting over for fftw so I don't mess up what Erin already did, following the instructions here: http://www.fftw.org/fftw3_doc/Installation-on-Unix.html#Installation-on-Unix
# FFTW
cd /projects/b1011/blast-tng/fftw-3.3.8
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
#first double precision, try with mpi (I think it does both mpi and single thread)
./configure --prefix=/projects/b1011/blast-tng/software/fftw-paw/3.3.8 CC=gcc MPICC=mpicc --enable-mpi --enable-openmp
#no permission to write the config.log in this directory, Erin created it
cd /projects/b1011/blast-tng
mv fftw-3.3.8/ fftw-3.3.8-erin
mkdir fftw-3.3.8
cp -r fftw-3.3.8-erin/* fftw-3.3.8/
cd /projects/b1011/blast-tng/fftw-3.3.8
./configure --prefix=/projects/b1011/blast-tng/software/fftw-paw/3.3.8 CC=gcc MPICC=mpicc --enable-mpi --enable-openmp
make
make check
#passed all checks
make install
ls /projects/b1011/blast-tng/software/fftw-paw/3.3.8/lib/
#output: cmake libfftw3.a libfftw3.la libfftw3_mpi.a libfftw3_mpi.la libfftw3_omp.a libfftw3_omp.la pkgconfig
#only has static libraries, not shared need --enable-shared
rm -r /projects/b1011/blast-tng/software/fftw-paw/
./configure --prefix=/projects/b1011/blast-tng/software/fftw-paw/3.3.8 CC=gcc MPICC=mpicc --enable-mpi --enable-openmp --enable-shared
make
make install
make check
#passed all checks
ls /projects/b1011/blast-tng/software/fftw-paw/3.3.8/lib/
#output: cmake libfftw3.a libfftw3.la libfftw3_mpi.a libfftw3_mpi.la libfftw3_omp.a libfftw3_omp.la pkgconfig
#same as last time, I guess it's fine? But in the one Erin did, there are shared libraries
#try cmake way
module load cmake/3.1.0
rm -r /projects/b1011/blast-tng/fftw-3.3.8/build
mkdir build
cd /projects/b1011/blast-tng/fftw-3.3.8/build
rm -r /projects/b1011/blast-tng/software/fftw-paw
cmake .. -DCMAKE_C_COMPILER=gcc -DCMAKE_INSTALL_PREFIX=/projects/b1011/blast-tng/softare/fftw-paw
ccmake ..
#not making any changes
make
#error in Linking C executable bench
##collect2: ld returned 1 exit status
rm -r *
cmake .. -DCMAKE_C_COMPILER=gcc -DCMAKE_INSTALL_PREFIX=/projects/b1011/blast-tng/softare/fftw-paw
make
#same error
rm -r *
cmake .. -DCMAKE_C_COMPILER=gcc -DCMAKE_INSTALL_PREFIX=/projects/b1011/blast-tng/softare/fftw-paw
ccmake ..
#change ENABLE_OPENMP to ON
make
#same error
ccmake ..
#disable openmp, enable float
make
#same error
ccmake ..
#defaults, disable BUILD_TESTS
make
#built?
rm -r *
cmake .. -DCMAKE_C_COMPILER=gcc -DCMAKE_INSTALL_PREFIX=/projects/b1011/blast-tng/softare/fftw-paw
#defaults, disable BUILD_TESTS
make
#worked!
make install
#typo in install prefix, changing to:
cmake .. -DCMAKE_C_COMPILER=gcc -DCMAKE_INSTALL_PREFIX=/projects/b1011/blast-tng/software/fftw-paw/3.3.8/
ccmake ..
#disable BUILD_TESTS
make
make install
ccmake ..
#enable OPENMP
make
make install
ccmake ..
#enable OPENMP, enable FLOAT
make
#makes both serial and omp libraries
make install
ccmake ..
#enable long double, enable OPENMP
make
make install
#so we have fftw shared libraries for:
# double serial and openmp
# float serial and openmp
# long double serial and openmp
# trick was disabling the BUILD_TESTS
# 2/28/19
# building getdata 0.9.3 (latest is 0.10.0, maybe want to upgrade later?)
module load gcc/4.6.3
#previous configure for getdata in /home/paw663/fissel/getdata-0.9.3
#command: ./configure --disable-idl --disable-matlab --disable-perl --disable-python --disable-php --disable-zzslim --prefix=/projects/b1011/fissel/pascal/getdata
#I will leave python support this time
cd /projects/b1011/blast-tng/getdata-0.9.3
./configure --disable-idl --disable-matlab --disable-perl --disable-php --disable-zzslim --prefix=/projects/b1011/blast-tng/software/getdata/0.9.3
make
#Error:
##CDPATH="${ZSH_VERSION+.}:" && cd . && /bin/sh /projects/b1011/blast-tng/getdata-0.9.3/missing aclocal-1.15 -I m4
##/projects/b1011/blast-tng/getdata-0.9.3/missing: line 81: aclocal-1.15: command not found
##WARNING: 'aclocal-1.15' is missing on your system.
## You should only need it if you modified 'acinclude.m4' or
## 'configure.ac' or m4 files included by 'configure.ac'.
## The 'aclocal' program is part of the GNU Automake package:
## <http://www.gnu.org/software/automake>
## It also requires GNU Autoconf, GNU m4 and Perl in order to run:
## <http://www.gnu.org/software/autoconf>
## <http://www.gnu.org/software/m4/>
## <http://www.perl.org/>
##make: *** [aclocal.m4] Error 127
module load automake/1.15
./configure --disable-idl --disable-matlab --disable-perl --disable-php --disable-zzslim --prefix=/projects/b1011/blast-tng/software/getdata/0.9.3
make
#failed on python related things:
##building 'pygetdata' extension
##gcc -pthread -fno-strict-aliasing -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -DNDEBUG -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -fPIC -DHAVE_CONFIG_H=1 -I../../src -I../../src -I/usr/lib64/python2.7/site-packages/numpy/core/include -I/usr/include/python2.7 -c pydirfile.c -o build/temp.linux-x86_64-2.7/pydirfile.o
##cc1: error: unrecognised debug output level "record-gcc-switches"
##cc1: error: unrecognised debug output level "record-gcc-switches"
##cc1: error: unrecognized command line option ‘-fstack-protector-strong’
##cc1: error: unrecognized command line option ‘-fstack-protector-strong’
##error: command 'gcc' failed with exit status 1
##make[4]: *** [build/lib.linux-x86_64-2.7/pygetdata.so] Error 1
##make[4]: Leaving directory `/projects/b1011/blast-tng/getdata-0.9.3/bindings/python'
##make[3]: *** [all-recursive] Error 1
##make[3]: Leaving directory `/projects/b1011/blast-tng/getdata-0.9.3/bindings/python'
##make[2]: *** [all] Error 2
##make[2]: Leaving directory `/projects/b1011/blast-tng/getdata-0.9.3/bindings/python'
##make[1]: *** [all-recursive] Error 1
##make[1]: Leaving directory `/projects/b1011/blast-tng/getdata-0.9.3/bindings'
##make: *** [all-recursive] Error 1
module load python
./configure --disable-idl --disable-matlab --disable-perl --disable-php --disable-zzslim --prefix=/projects/b1011/blast-tng/software/getdata/0.9.3
make
make check
#all checks that ran were passed
make install
#made a module
#MOAT
#again, from /projects/b1011/blast-tng/MOAT_3/MOAT/config/log:
# ./configure --prefix=/projects/b1011/fissel/pascal/Experimental/moat_new --with-boost=/projects/b1011/fissel/pascal/Experimental/boost BOOST_ROOT=/projects/b1011/fissel/pascal/Experimental/boost MPICXX=mpic++ CC=gcc CXX=g++ F77=gfortran LIBS=-lgomp --with-fftw-libs=-L/software/FFTW/3.3.3/lib -lfftw3 CFLAGS=-I/software/FFTW/3.3.3/include CXXFLAGS=-I/software/FFTW/3.3.3/include -with-fftw-cpp=-I/software/FFTW/3.3.3/include
module use /projects/b1011/blast-tng/modules
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
module load python
module load fftw #using Erin and Pascal's fftw
module load lapack
module load boost
rm -r /projects/b1011/blast-tng/software/moat
cd /projects/b1011/blast-tng/MOAT_3/MOAT
./configure --prefix=/projects/b1011/blast-tng/software/moat --with-boost=/projects/b1011/blast-tng/software/boost/1.61.0 BOOST_ROOT=/projects/b1011/blast-tng/software/boost/1.61.0 MPICXX=mpic++ CC=gcc CXX=g++ F77=gfortran LIBS=-lgomp LIBS=-lfftw3 --with-fftw-libs=-L/projects/b1011/blast-tng/software/fftw/3.3.8/lib64 CFLAGS=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include CXXFLAGS=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include --with-fftw-cpp=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include
#output includes:
##configure: =========== Build Configuration ===========
##configure: C++ Compiler : g++
##configure: C++ Compile flags : -I/projects/b1011/blast-tng/software/fftw/3.3.8/include
##configure: F77 Compiler : gfortran
##configure: MPICXX Compiler : mpic++
##configure: OpenMP : Detected (-fopenmp)
##configure: OpenCL : Disabled
##configure: ACML Vendor Lib : Disabled
##configure: MASS Vendor Lib : Disabled
##configure: MKL Vendor Lib : Disabled
##configure: Apple Vendor Lib : Disabled
##configure: ===========================================
#but I noticed mpic++ is in the anaconda directory, think it should be from the mpi module instead
module purge
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
#now mpic++ is from this module
module load fftw
module load lapack
module load boost
./configure --prefix=/projects/b1011/blast-tng/software/moat --with-boost=/projects/b1011/blast-tng/software/boost/1.61.0 BOOST_ROOT=/projects/b1011/blast-tng/software/boost/1.61.0 MPICXX=mpic++ CC=gcc CXX=g++ F77=gfortran LIBS=-lgomp LIBS=-lfftw3 --with-fftw-libs=-L/projects/b1011/blast-tng/software/fftw/3.3.8/lib64 CFLAGS=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include CXXFLAGS=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include --with-fftw-cpp=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include
#output includes:
##configure: =========== Build Configuration ===========
##configure: C++ Compiler : g++
##configure: C++ Compile flags : -I/projects/b1011/blast-tng/software/fftw/3.3.8/include
##configure: F77 Compiler : gfortran
##configure: MPICXX Compiler : mpic++
##configure: OpenMP : Detected (-fopenmp)
##configure: OpenCL : Disabled
##configure: ACML Vendor Lib : Disabled
##configure: MASS Vendor Lib : Disabled
##configure: MKL Vendor Lib : Disabled
##configure: Apple Vendor Lib : Disabled
##configure: ===========================================
make
make check
#failing in a test of dense matrix ops
#terminate called after throwing an instance of 'moat::exception'
# what(): Exception at line 142 of file frameworks/03_la/test_mv.cpp: Fail on transposed output vector consistency
# this looks boost related, and ./configure --help does say to use boostv1.53, we have v1.61.0. But we used 1.61.0 before
# maybe because when I built boost I loaded the python module last, so we used its mpic++? I can try to re-build boost loading python first
# 3/1/19
# trying make check for old MOAT build
cd /projects/b1011/fissel/pascal/Experimental/MOAT_3/MOAT
make check
#same exact error as when I ran yesterday in my new moat build directory
#maybe this is okay?
# Boost
# trying boost again, load python module first this time
# putting it in software/boost2
cd /projects/b1011/blast-tng/boost_1_61_0
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
./bootstrap.sh --with-libraries=all --prefix=/projects/b1011/blast-tng/software/boost2/1.61.0
./b2
#error about not using mpi, have to add something to user-config.jam to get mpi support, maybe that is the problem?
./b2 install
#re-run ./b2 to see error message:
./b2
#warning:
##Graph library does not contain MPI-based parallel components.
##note: to enable them, add "using mpi ;" to your user-config.jam
## - zlib : yes (cached)
## - iconv (libc) : yes (cached)
## - icu : no (cached)
## - icu (lib64) : no (cached)
## - compiler-supports-visibility : yes (cached)
## - compiler-supports-ssse3 : yes (cached)
## - compiler-supports-avx2 : no (cached)
## - gcc visibility : yes (cached)
## - long double support : yes (cached)
##warning: skipping optional Message Passing Interface (MPI) library.
##note: to enable MPI support, add "using mpi ;" to user-config.jam.
##note: to suppress this message, pass "--without-mpi" to bjam.
##note: otherwise, you can safely ignore this message.
#made a boost2 module to point to the new boost installation
# try moat with the new boost
cd /projects/b1011/blast-tng/MOAT_3/MOAT
module purge
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
#now mpic++ is from this module
module use /projects/b1011/blast-tng/modules
module load fftw
module load lapack
module load boost2
./configure --prefix=/projects/b1011/blast-tng/software/moat --with-boost=/projects/b1011/blast-tng/software/boost2/1.61.0 BOOST_ROOT=/projects/b1011/blast-tng/software/boost2/1.61.0 MPICXX=mpic++ CC=gcc CXX=g++ F77=gfortran LIBS=-lgomp LIBS=-lfftw3 --with-fftw-libs=-L/projects/b1011/blast-tng/software/fftw/3.3.8/lib64 CFLAGS=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include CXXFLAGS=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include --with-fftw-cpp=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include
#output includes:
##configure: =========== Build Configuration ===========
##configure: C++ Compiler : g++
##configure: C++ Compile flags : -I/projects/b1011/blast-tng/software/fftw/3.3.8/include
##configure: F77 Compiler : gfortran
##configure: MPICXX Compiler : mpic++
##configure: OpenMP : Detected (-fopenmp)
##configure: OpenCL : Disabled
##configure: ACML Vendor Lib : Disabled
##configure: MASS Vendor Lib : Disabled
##configure: MKL Vendor Lib : Disabled
##configure: Apple Vendor Lib : Disabled
##configure: ===========================================
make
make check
#same error while testing dense matrix ops
# Boost again!
cd /projects/b1011/blast-tng/boost_1_61_0
vim ./tools/build/example/user-config.jam
#added "using mpi" on a new line at the end
module purge
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
rm -r ../software/boost2
./bootstrap.sh --with-libraries=all --prefix=/projects/b1011/blast-tng/software/boost2/1.61.0
./b2
#still got warning about mpi
#changed last line of user-config.jam to "using mpi ;"
./bootstrap.sh --with-libraries=all --prefix=/projects/b1011/blast-tng/software/boost2/1.61.0
./b2
#same warning
#maybe I need to change project-config.jam, which is created by boostrap.sh
./bootstrap.sh --with-libraries=all --prefix=/projects/b1011/blast-tng/software/boost2/1.61.0
vim project-config.jam
#added new final line "using mpi ;"
./b2
#no warning, building new things this time
./b2 install
#going to put it in the regular location (boost, not boost2)
./bootstrap.sh --with-libraries=all --prefix=/projects/b1011/blast-tng/software/boost/1.61.0
echo "using mpi ;" >> project-config.jam
./b2
./b2 install
# try moat again, with full mpi boost support
module purge
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
module use /projects/b1011/blast-tng/modules
module load fftw
module load lapack
module load boost
./configure --prefix=/projects/b1011/blast-tng/software/moat --with-boost=/projects/b1011/blast-tng/software/boost/1.61.0 BOOST_ROOT=/projects/b1011/blast-tng/software/boost/1.61.0 MPICXX=mpic++ CC=gcc CXX=g++ F77=gfortran LIBS=-lgomp LIBS=-lfftw3 --with-fftw-libs=-L/projects/b1011/blast-tng/software/fftw/3.3.8/lib64 CFLAGS=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include CXXFLAGS=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include --with-fftw-cpp=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include
#output includes:
##configure: =========== Build Configuration ===========
##configure: C++ Compiler : g++
##configure: C++ Compile flags : -I/projects/b1011/blast-tng/software/fftw/3.3.8/include
##configure: F77 Compiler : gfortran
##configure: MPICXX Compiler : mpic++
##configure: OpenMP : Detected (-fopenmp)
##configure: OpenCL : Disabled
##configure: ACML Vendor Lib : Disabled
##configure: MASS Vendor Lib : Disabled
##configure: MKL Vendor Lib : Disabled
##configure: Apple Vendor Lib : Disabled
##configure: ===========================================
make
make check
#still fails same test
make install
# maybe we don't need that operation for toast, after all the old moat build fails the same test
# could be an fftw issue, I have been using the less complete version for moat
# I made a module for fftw-paw, I will try using that next for moat
module purge
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
module use /projects/b1011/blast-tng/modules
module load fftw-paw
module load lapack
module load boost
./configure --prefix=/projects/b1011/blast-tng/software/moat --with-boost=/projects/b1011/blast-tng/software/boost/1.61.0 BOOST_ROOT=/projects/b1011/blast-tng/software/boost/1.61.0 MPICXX=mpic++ CC=gcc CXX=g++ F77=gfortran LIBS=-lgomp LIBS=-lfftw3 --with-fftw-libs=-L/projects/b1011/blast-tng/software/fftw-paw/3.3.8/lib64 CFLAGS=-I/projects/b1011/blast-tng/software/fftw-paw/3.3.8/include CXXFLAGS=-I/projects/b1011/blast-tng/software/fftw-paw/3.3.8/include --with-fftw-cpp=-I/projects/b1011/blast-tng/software/fftw-paw/3.3.8/include
#output includes:
##configure: =========== Build Configuration ===========
##configure: C++ Compiler : g++
##configure: C++ Compile flags : -I/projects/b1011/blast-tng/software/fftw-paw/3.3.8/include
##configure: F77 Compiler : gfortran
##configure: MPICXX Compiler : mpic++
##configure: OpenMP : Detected (-fopenmp)
##configure: OpenCL : Disabled
##configure: ACML Vendor Lib : Disabled
##configure: MASS Vendor Lib : Disabled
##configure: MKL Vendor Lib : Disabled
##configure: Apple Vendor Lib : Disabled
##configure: ===========================================
make
make check
#fails same test
#not installing this time, going to use the version with the less complete fftw
# changed the directory structure of the fftw installations slightly, now only one fftw directory but two versions
# same for the modules
# modules should still work though
# need to try toast next
# found the old config.log (/projects/b1011/fissel/pascal/Experimental/TOAST_5/TOAST/config.log)
#./configure --prefix=/projects/b1011/fissel/pascal/Experimental/toast_new --with-lapack=/projects/b1011/fissel/pascal/Experimental/lapack/lib64/liblapack.so --with-blas=/projects/b1011/fissel/pascal/Experimental/lapack/lib64/libblas.so --with-cfitsio=/software/supplemental/cfitsio/3.37 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1011/fissel/pascal/Experimental/moat_new/bin/moatconfig --with-wcslib=/projects/b1011/fissel/pascal/Experimental/wcslib --enable-exp-blast --with-getdata=/projects/b1011/fissel/pascal/Experimental/getdata CFLAGS=-I/projects/b1011/fissel/pascal/Experimental/wcslib/include/wcslib-5.15 -I/projects/b1011/fissel/pascal/Experimental/getdata/include/getdata -I/projects/b1011/fissel/pascal/Experimental/boost/include -I/projects/b1011/fissel/pascal/Experimental/boost/include/boost CXXFLAGS=-I/projects/b1011/fissel/pascal/Experimental/wcslib/include/wcslib-5.15 -I/projects/b1011/fissel/pascal/Experimental/getdata/include/getdata -I/projects/b1011/fissel/pascal/Experimental/boost/include -I/projects/b1011/fissel/pascal/Experimental/boost/include/boost CPPFLAGS=-I/projects/b1011/fissel/pascal/Experimental/wcslib/include/wcslib-5.15 -I/projects/b1011/fissel/pascal/Experimental/getdata/include/getdata -I/projects/b1011/fissel/pascal/Experimental/boost/include -I/projects/b1011/fissel/pascal/Experimental/boost/include/boost LDFLAGS=-L/software/anaconda2/lib -L/projects/b1011/fissel/pascal/Experimental/python/lib
# edited with new paths:
# Note: CFLAGS, CXXFLAGS, CPPFLAGS are all the same.
#./configure --prefix=/projects/b1011/blast-tng/software/toast --with-lapack=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/liblapack.a --with-blas=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/libblas.a --with-cfitsio=/projects/b1011/blast-tng/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1011/blast-tng/software/moat/bin/moatconfig --with-wcslib=/projects/b1011/blast-tng/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1011/blast-tng/software/getdata/0.9.3 CFLAGS=-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost CXXFLAGS=-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost CPPFLAGS=-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost LDFLAGS=-L/software/anaconda2/lib -L/projects/b1011/fissel/pascal/Experimental/python/lib
# potential problems and differences between new and old:
# old lapack used shared library, we only have static. – may require just changing an option in ccmake — did this
# in the old getdata/include/getdata, there is a link to ../getdata.h. New one has no link, should we make one? — didn't do this
# /projects/b1011/fissel/pascal/Experimental/python/lib contains a link to a library in the anaconda2 python, should we recreate that link? — didn't recreate link, just left that path in the LDFLAGS variable
# 3/11/19
# rebuilding lapack with shared libraries
cd /projects/b1011/blast-tng/lapack-3.6.0/build-2
module load gcc/4.6.3
module load cmake/3.1.0
ccmake ..
#changed BUILD_SHARED_LIBS to ON
make
#now there are shared libraries in addition to static ones, changing the toast configure to use shared lib
# new toast configure command
# using liblapack.so and libblas.so
#./configure --prefix=/projects/b1011/blast-tng/software/toast --with-lapack=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1011/blast-tng/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1011/blast-tng/software/moat/bin/moatconfig --with-wcslib=/projects/b1011/blast-tng/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1011/blast-tng/software/getdata/0.9.3 CFLAGS=-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost CXXFLAGS=-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost CPPFLAGS=-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost LDFLAGS=-L/software/anaconda2/lib -L/projects/b1011/fissel/pascal/Experimental/python/lib
# toast installation attempt:
module purge
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
module use /projects/b1011/blast-tng/modules
module load fftw
module load lapack
module load boost
module load moat
module load cfitsio
module load wcslib
module load getdata
#adding quotes around all flags (otherwise it sees the space and thinks -I/.../getdata is a new option)
./configure --prefix=/projects/b1011/blast-tng/software/toast --with-lapack=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1011/blast-tng/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1011/blast-tng/software/moat/bin/moatconfig --with-wcslib=/projects/b1011/blast-tng/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1011/blast-tng/software/getdata/0.9.3 CFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" LDFLAGS="-L/software/anaconda2/lib -L/projects/b1011/fissel/pascal/Experimental/python/lib"
#end of output:
##checking for the Boost serialization library... no
##configure: error: cannot find the flags to link with Boost serialization
# adding BOOST_ROOT to configure call
./configure --prefix=/projects/b1011/blast-tng/software/toast --with-lapack=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1011/blast-tng/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1011/blast-tng/software/moat/bin/moatconfig --with-wcslib=/projects/b1011/blast-tng/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1011/blast-tng/software/getdata/0.9.3 CFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" LDFLAGS="-L/software/anaconda2/lib -L/projects/b1011/fissel/pascal/Experimental/python/lib" BOOST_ROOT=/projects/b1011/blast-tng/software/boost/1.61.0/include/boost/
#end of output
##configure: Detected BOOST_ROOT; continuing with --with-boost=/projects/b1011/blast-tng/software/boost/1.61.0/include/boost/
##checking for Boost headers version >= 1.53.0... no
##configure: cannot find Boost headers version >= 1.53.0
##configure: error: Could not find BOOST library >= 1.53!
# changing BOOST_ROOT path slightly
./configure --prefix=/projects/b1011/blast-tng/software/toast --with-lapack=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1011/blast-tng/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1011/blast-tng/software/moat/bin/moatconfig --with-wcslib=/projects/b1011/blast-tng/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1011/blast-tng/software/getdata/0.9.3 CFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" LDFLAGS="-L/software/anaconda2/lib -L/projects/b1011/fissel/pascal/Experimental/python/lib" BOOST_ROOT=/projects/b1011/blast-tng/software/boost/1.61.0/include/
#end of output
##configure: Detected BOOST_ROOT; continuing with --with-boost=/projects/b1011/blast-tng/software/boost/1.61.0/include/
##checking for Boost headers version >= 1.53.0... /projects/b1011/blast-tng/software/boost/1.61.0/include/
##checking for Boost's header version... 1_61
##checking for the toolset name used by Boost for g++... gcc46 -gcc
##checking boost/archive/text_oarchive.hpp usability... yes
##checking boost/archive/text_oarchive.hpp presence... yes
##checking for boost/archive/text_oarchive.hpp... yes
##checking for the Boost serialization library... no
##configure: error: cannot find the flags to link with Boost serialization
#back where we started, so the first time I used BOOST_ROOT, configure got an error earlier
./configure --prefix=/projects/b1011/blast-tng/software/toast --with-lapack=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1011/blast-tng/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1011/blast-tng/software/moat/bin/moatconfig --with-wcslib=/projects/b1011/blast-tng/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1011/blast-tng/software/getdata/0.9.3 CFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" LDFLAGS="-L/software/anaconda2/lib -L/projects/b1011/fissel/pascal/Experimental/python/lib" BOOST_ROOT=/projects/b1011/blast-tng/software/boost/1.61.0/
#worked!
make
#example compile step (during make)
##g++ -DHAVE_CONFIG_H -I. -I../.. -I../../src/libtoast -I../../src/libtoast/map -I../../src/libtoast/IO -I../../src/libtoast/math -I../../src/libtoast/math/tpm -I../../src/tests -I/projects/b1011/fissel/pascal/Experimental/moat_new/include -I/projects/b1011/fissel/pascal/Experimental/boost/include -fopenmp -I/software/FFTW/3.3.3/include -I/projects/b1011/blast-tng/software/cfitsio/3.45/include -I/projects/b1011/blast-tng/software/boost/1.61.0//include -I/projects/b1011/blast-tng/software/getdata/0.9.3/include -I/projects/b1011/blast-tng/software/wcslib/5.15/include -I../../src/libtoast -I../../experiments/blast/IO -I../../experiments/blast/tests -fopenmp -I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost -MT toast_test_readwrite.o -MD -MP -MF .deps/toast_test_readwrite.Tpo -c -o toast_test_readwrite.o toast_test_readwrite.cpp
#-I/software/FFTW/3.3.3/include is here! Why? holdover from last time? How can we delete old settings?
#end of make output:
##In file included from ../../../src/libtoast/IO/../../../experiments/blast/IO/toast_blast.hpp:3:0,
## from ../../../src/libtoast/IO/experiments_IO.hpp:4,
## from ../../../src/libtoast/IO/toast_io_internal.hpp:16,
## from ../../../src/libtoast/toast_internal.hpp:12,
## from ../../../src/libtoast-mpi/toast_mpi_internal.hpp:8,
## from toast_mpi_run.cpp:3:
##/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata/dirfile.h:25:27: fatal error: getdata/types.h: No such file or directory
##compilation terminated.
#make link to one directory back up, types.h is in the same directory as dirfile.h (as are a bunch of other header files)
ln -s /projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata /projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata/getdata
#Problems:
# old FFTW seems to being used, don't see where in configure output it is being included
# maybe old moat is being used too, based on Build Configuration output?
# getdata header files, probably need to hack them so they look in the right place
# are things being cached? configure output says they are, but I don't see a config.cache
# saved most recent configure output in config.output so I can look later
# 4/9/2019
# rebuilding MOAT using make clean
cd /projects/b1011/blast-tng/MOAT_3/MOAT
module purge
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
module use /projects/b1011/blast-tng/modules
module load fftw
module load lapack
module load boost
./configure --prefix=/projects/b1011/blast-tng/software/moat --with-boost=/projects/b1011/blast-tng/software/boost/1.61.0 BOOST_ROOT=/projects/b1011/blast-tng/software/boost/1.61.0 MPICXX=mpic++ CC=gcc CXX=g++ F77=gfortran LIBS=-lgomp LIBS=-lfftw3 --with-fftw-libs=-L/projects/b1011/blast-tng/software/fftw/3.3.8/lib64 CFLAGS=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include CXXFLAGS=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include --with-fftw-cpp=-I/projects/b1011/blast-tng/software/fftw/3.3.8/include
make clean
make
make check
# fails same "testing dense matrix ops" test
make install
# TOAST again
cd /projects/b1011/blast-tng/TOAST_5/TOAST
module purge
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
module use /projects/b1011/blast-tng/modules
module load fftw
module load lapack
module load boost
module load moat
module load cfitsio
module load wcslib
module load getdata
./configure --prefix=/projects/b1011/blast-tng/software/toast --with-lapack=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1011/blast-tng/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1011/blast-tng/software/moat/bin/moatconfig --with-wcslib=/projects/b1011/blast-tng/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1011/blast-tng/software/getdata/0.9.3 CFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" LDFLAGS="-L/software/anaconda2/lib -L/projects/b1011/fissel/pascal/Experimental/python/lib" BOOST_ROOT=/projects/b1011/blast-tng/software/boost/1.61.0/
make clean
./configure --prefix=/projects/b1011/blast-tng/software/toast --with-lapack=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1011/blast-tng/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1011/blast-tng/software/moat/bin/moatconfig --with-wcslib=/projects/b1011/blast-tng/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1011/blast-tng/software/getdata/0.9.3 CFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" LDFLAGS="-L/software/anaconda2/lib -L/projects/b1011/fissel/pascal/Experimental/python/lib" BOOST_ROOT=/projects/b1011/blast-tng/software/boost/1.61.0/
make
# failed because of getdata header files in the wrong place, changing getdata path (in CFLAGS, CPPFLAGS, CXXFLAGS)
# and deleting the link at /projects/b1011/blast-tng/software/getdata/0.9.3/include/getdata/getdata
make clean
./configure --prefix=/projects/b1011/blast-tng/software/toast --with-lapack=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1011/blast-tng/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1011/blast-tng/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1011/blast-tng/software/moat/bin/moatconfig --with-wcslib=/projects/b1011/blast-tng/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1011/blast-tng/software/getdata/0.9.3 CFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1011/blast-tng/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1011/blast-tng/software/getdata/0.9.3/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include -I/projects/b1011/blast-tng/software/boost/1.61.0/include/boost" LDFLAGS="-L/software/anaconda2/lib -L/projects/b1011/fissel/pascal/Experimental/python/lib" BOOST_ROOT=/projects/b1011/blast-tng/software/boost/1.61.0
make
#error:
##In file included from ./experiments_IO.hpp:4:0,
## from ./toast_io_internal.hpp:16,
## from toast_fits.cpp:3:
##./../../../experiments/blast/IO/toast_blast.hpp:3:21: fatal error: dirfile.h: No such file or directory
# changed line 3 of /projects/b1011/blast-tng/TOAST_5/TOAST/experiments/blast/IO/toast_blast.hpp to:
#include <getdata/dirfile.h>
# next we need our own anaconda installation of python to avoid conflicts with libraries, like libbz2
#################################################################################
# Starting over on slurm cluster, going to make our own python:
# Also starting in the new project space, /projects/b1092
#
#################################################################################
# Anaconda python
cd /projects/b1092/
# Anaconda install script downloaded from anaconda's website
bash ./Anaconda2-2019.03-Linux-x86_64.sh -p /projects/b1092/software/anaconda
# the last thing the installer asks is "do you want to run conda init?"
# say no, we will load this python using a module instead
# made a module, now we need to always load this python by issuing "module use ..." before "module load python"
# Lapack
module purge
cd /projects/b1092/lapack-3.6.0
rm -rf build
rm -rf build-2
mkdir build
cd /projects/b1092/lapack-3.6.0
module use /projects/b1092/modules
module load python
module load gcc/4.6.3
module load cmake/3.1.0
cmake .. -DCMAKE_C_COMPILER=gcc -DCMAKE_INSTALL_PREFIX=/projects/b1092/software/lapack/3.6.0 -DCMAKE_Fortran_COMPILER=gfortran
ccmake ..
# turn BUILD_SHARED_LIBS ON
make
make install
# no static libraries this time, even though it is turned ON in ccmake
# only difference in Makefiles from this time and last time is paths for build directory
ccmake ..
# turn BUILD_SHARED_LIBS OFF
make
make install
# now we have both shared and static. Not sure if we need them all, but they are there
# Getdata
module purge
module use /projects/b1092/modules
module load python
module load gcc/4.6.3
module load automake/1.15
cd /projects/b1092/getdata-0.9.3
./configure --disable-idl --disable-matlab --disable-perl --disable-php --disable-zzslim --prefix=/projects/b1092/software/getdata/0.9.3
make clean
./configure --disable-idl --disable-matlab --disable-perl --disable-php --disable-zzslim --prefix=/projects/b1092/software/getdata/0.9.3
make
make check
# all checks that ran, passed (only 17 did not run)
make install
# made a module at /projects/b1092/modules/getdata/0.9.3
# Boost
module purge
module use /projects/b1092/modules
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
cd /projects/b1011/blast-tng/boost_1_61_0
./b2 clean
# not sure how much this helped, it says it performed configuration checks and built Boost C++ libraries
./bootstrap.sh --with-libraries=all --prefix=/projects/b1092/software/boost/1.61.0
echo "using mpi ;" >> project-config.jam
./b2
./b2 install
# Cfitsio
cd /projects/b1092/cfitsio/build/
module purge
module load gcc/4.6.3
cmake .. -DCMAKE_C_COMPILER=/hpc/software/gcc/4.6.3-rhel7/bin/gcc -DCMAKE_INSTALL_PREFIX=/projects/b1092/software/cfitsio/3.45 -DCMAKE_Fortran_COMPLIER=gfortran
#here I got an error about a path in CMakeCache.txt, so I edited the path names in that file and re-ran above command (seemed to work...)
make
make install
#things seemed to work without error, there is now cfitsio listed in the software directory
# FFTW
cd /projects/b1092/fftw-3.3.8/
module list (gcc/4.6.3)
module load mpi
module list
./configure --prefix=/projects/b1092/sofware/fftw/3.3.8 CC=gcc MPICC=mpicc --enable-mpi --enable-openmp --enable-shared
module load cmake/3.1.0
cd build/
cmake .. -DCMAKE_C_COMPILER=gcc -DCMAKE_INSTALL_PREFIX=/projects/b1092/software/fftw/3.3.8/
ccmake .. #(press c, q to save parameters)
#checking to make sure all flags are turned on or off from above (enable long double, enable OPENMP, enable FLOAT, disable BUILD_TESTS), everything was except one...can't remember which, oops
make
make install
# Wcslib
cd /projects/b1092/wcslib-5.15
module purge
module load gcc/4.6.3
module use /projects/b1092/modules/
module load cfitsio
./configure --prefix=/projects/b1092/software/wcslib/5.15/ --with-cfitsiolib=/projects/b1092/software/cfitsio/3.45/lib --with-cfitsioinc=/projects/b1092/software/cfitsio/3.45/include --without-pgplot
gmake clean
gmake
gmake check
#passes all tests
gmake install
#5/1/2019
#attempt to install moat (failed in same place as it did previously)
cd /projects/b1092
cp -r ../b1011/blast-tng/MOAT_3/ .
cd MOAT_3/MOAT
module use /projects/b1092/modules/
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
module load fftw
module load lapack
module load boost
./configure --prefix=/projects/b1092/software/moat --with-boost=/projects/b1092/software/boost/1.61.0 BOOST_ROOT=/projects/b1092/software/boost/1.61.0 MPICXX=mpic++ CC=gcc CXX=g++ F77=gfortran LIBS=-lgomp LIBS=-lfftw3 --with-fftw-libs=-L/projects/b1092/software/fftw/3.3.8/lib64 CFLAGS=-I/projects/b1092/software/fftw/3.3.8/include CXXFLAGS=-I/projects/b1092/software/fftw/3.3.8/include --with-fftw-cpp=-I/projects/b1092/software/fftw/3.3.8/include
make clean
make
make check
#output:
#Testing dense matrix ops...
#PROFILE: TEST_MV (Matrix-Vector test time) :
#PROFILE: Elapsed process time = 1.648071e-01 seconds
#PROFILE: Total thread time = 9.229197e+00 thread-seconds
#PROFILE: Active thread time = 1.648071e-01 thread-seconds
#PROFILE: Idle thread time = 9.064389e+00 thread-seconds
#PROFILE: Thread efficiency = 1.79%
#terminate called after throwing an instance of 'moat::exception'
# what(): Exception at line 142 of file frameworks/03_la/test_mv.cpp: Fail on transposed output vector consistency
#/bin/sh: line 5: 42667 Aborted ${dir}$tst
#FAIL: moat_test
make install
#did this because this is what was done previously. I must say that it is somewhat worrisome if this is failing in a matrix
#operation since the mapmaker relies on inverting matricies
#6/4/2019
#looks like when moat fails the check it stops and doesn't check anything else
#could remove references to 03_la directory in src/libmoat/frameworks/Makefile
#toast
module purge
module use /projects/b1092/modules
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
module load fftw
module load lapack
module load boost
module load moat
module load cfitsio
module load wcslib
module load getdata
# for some reason needed automake
module load automake/1.15
#changed all paths to point to b1092 installation of libraries except for the libboost_python library that we had to make a link to, that one doesn't exist in our anaconda
./configure --prefix=/projects/b1092/software/toast --with-lapack=/projects/b1092/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1092/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1092/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1092/software/moat/bin/moatconfig --with-wcslib=/projects/b1092/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1092/software/getdata/0.9.3 CFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" LDFLAGS="-L/projects/b1092/software/anaconda/lib" BOOST_ROOT=/projects/b1092/blast-tng/software/boost/1.61.0
#fixing BOOST_ROOT, still was wrong path
./configure --prefix=/projects/b1092/software/toast --with-lapack=/projects/b1092/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1092/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1092/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1092/software/moat/bin/moatconfig --with-wcslib=/projects/b1092/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1092/software/getdata/0.9.3 CFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" LDFLAGS="-L/projects/b1092/software/anaconda/lib" BOOST_ROOT=/projects/b1092/software/boost/1.61.0
make
#failed on some library conflicts:
#/usr/bin/ld: warning: libgfortran.so.3, needed by /projects/b1092/software/lapack/3.6.0/lib64/liblapack.so, may conflict with libgfortran.so.4
#/usr/bin/ld: warning: libbz2.so.1, needed by /projects/b1092/software/getdata/0.9.3/lib/libgetdata++.so, may conflict with libbz2.so.1.0
# making a conda environment with just basic python to avoid library conflicts
conda create --prefix /projects/b1092/software/toast-python python=2.7
conda activate /projects/b1092/software/toast-python
#try again to configure and make
./configure --prefix=/projects/b1092/software/toast --with-lapack=/projects/b1092/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1092/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1092/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1092/software/moat/bin/moatconfig --with-wcslib=/projects/b1092/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1092/software/getdata/0.9.3 CFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" LDFLAGS="-L/projects/b1092/software/anaconda/lib" BOOST_ROOT=/projects/b1092/software/boost/1.61.0
make
#/usr/bin/ld: warning: libgfortran.so.3, needed by /projects/b1092/software/lapack/3.6.0/lib64/liblapack.so, may conflict with libgfortran.so.4
#forgot to clean...
make clean
./configure --prefix=/projects/b1092/software/toast --with-lapack=/projects/b1092/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1092/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1092/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1092/software/moat/bin/moatconfig --with-wcslib=/projects/b1092/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1092/software/getdata/0.9.3 CFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" LDFLAGS="-L/projects/b1092/software/anaconda/lib" BOOST_ROOT=/projects/b1092/software/boost/1.61.0
#still had regular anaconda libs in the LDFLAGS, start again with configure
make clean
./configure --prefix=/projects/b1092/software/toast --with-lapack=/projects/b1092/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1092/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1092/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1092/software/moat/bin/moatconfig --with-wcslib=/projects/b1092/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1092/software/getdata/0.9.3 CFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" LDFLAGS="-L/projects/b1092/software/toast-python/lib" BOOST_ROOT=/projects/b1092/software/boost/1.61.0
make
#got an error about mpi conflicts, noticed we had the wrong version of fftw. we had the system one, we forgot to make our own module
#got to rebuild moat then toast, they are the only ones with fftw dependencies
#moat
#not in conda environment, never needed it before
cd /projects/b1092/MOAT_3/MOAT
module purge
module use /projects/b1092/modules/
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
module load fftw
module load lapack
module load boost
./configure --prefix=/projects/b1092/software/moat --with-boost=/projects/b1092/software/boost/1.61.0 BOOST_ROOT=/projects/b1092/software/boost/1.61.0 MPICXX=mpic++ CC=gcc CXX=g++ F77=gfortran LIBS=-lgomp LIBS=-lfftw3 --with-fftw-libs=-L/projects/b1092/software/fftw/3.3.8/lib64 CFLAGS=-I/projects/b1092/software/fftw/3.3.8/include CXXFLAGS=-I/projects/b1092/software/fftw/3.3.8/include --with-fftw-cpp=-I/projects/b1092/software/fftw/3.3.8/include
make clean
./configure --prefix=/projects/b1092/software/moat --with-boost=/projects/b1092/software/boost/1.61.0 BOOST_ROOT=/projects/b1092/software/boost/1.61.0 MPICXX=mpic++ CC=gcc CXX=g++ F77=gfortran LIBS=-lgomp LIBS=-lfftw3 --with-fftw-libs=-L/projects/b1092/software/fftw/3.3.8/lib64 CFLAGS=-I/projects/b1092/software/fftw/3.3.8/include CXXFLAGS=-I/projects/b1092/software/fftw/3.3.8/include --with-fftw-cpp=-I/projects/b1092/software/fftw/3.3.8/include
make
make check
#same dense matrix ops check fails like always
make install
#toast
cd /projects/b1092/TOAST_5/TOAST
module purge
module use /projects/b1092/modules
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
module load fftw
module load lapack
module load boost
module load moat
module load cfitsio
module load wcslib
module load getdata
module load automake/1.15
make clean
conda activate /projects/b1092/software/toast-python
./configure --prefix=/projects/b1092/software/toast --with-lapack=/projects/b1092/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1092/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1092/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1092/software/moat/bin/moatconfig --with-wcslib=/projects/b1092/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1092/software/getdata/0.9.3 CFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CXXFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CPPFLAGS="-I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" LDFLAGS="-L/projects/b1092/software/toast-python/lib" BOOST_ROOT=/projects/b1092/software/boost/1.61.0
make clean
make
#compile step that failed:
mpif90 -cpp -I../../src/libtoast -I../../src/libtoast/map -I../../src/libtoast/IO -I../../src/libtoast/math -I../../src/libtoast/math/tpm -I../../src/libtoast-mpi -I../../src/libtoast-mpi/math -I../../src/libtoast-mpi/IO -I../../src/libtoast-mpi/map -I../../src/tests-mpi -I/projects/b1092/software/moat/include -I/projects/b1092/software/boost/1.61.0/include -fopenmp -I/projects/b1092/software/fftw/3.3.8/include -I/projects/b1092/software/cfitsio/3.45/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/wcslib/5.15/include -I../../src/libtoast -I../../src/libtoast-mpi -g -O2 -c -o toast_mpi_fdist.o toast_mpi_fdist.f03
/bin/sh ../../libtool --tag=CXX --mode=link mpic++ -fopenmp -I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost -L/projects/b1092/software/toast-python/lib -o toast_mpi_fdist toast_mpi_fdist.o ../../src/libtoast-mpi/libftoast-mpi.la ../../src/libtoast/libftoast.la ../../src/libtoast-mpi/libtoast-mpi.la ../../src/libtoast/libtoast.la -lexpat -L/projects/b1092/software/moat/lib -lmoat-mpi -lmoat -lm -fopenmp -L/projects/b1092/software/fftw/3.3.8/lib64 -L/projects/b1092/software/boost/1.61.0/lib -Wl,-R,/projects/b1092/software/boost/1.61.0/lib -lboost_mpi -L/projects/b1092/software/moat/lib -lmoat -lm -fopenmp -L/projects/b1092/software/fftw/3.3.8/lib64 -L/projects/b1092/software/cfitsio/3.45/lib -lcfitsio -L/projects/b1092/software/getdata/0.9.3/lib -lgetdata++ -lgetdata -lbz2 -lz -L/projects/b1092/software/wcslib/5.15/lib -lwcs -lm -L/projects/b1092/software/boost/1.61.0/lib -Wl,-R,/projects/b1092/software/boost/1.61.0/lib -lboost_serialization -L/projects/b1092/software/boost/1.61.0/lib -Wl,-R,/projects/b1092/software/boost/1.61.0/lib -lboost_regex /projects/b1092/software/lapack/3.6.0/lib64/liblapack.so /projects/b1092/software/lapack/3.6.0/lib64/libblas.so -lm -L/projects/b1092/software/toast-python/lib -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc/x86_64-redhat-linux-gnu/4.6.3 -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc/x86_64-redhat-linux-gnu/4.6.3/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc/x86_64-redhat-linux-gnu/4.6.3/../../.. -lgfortran -lm -lquadmath -fopenmp -L/projects/b1092/software/toast-python/lib -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc/x86_64-redhat-linux-gnu/4.6.3 -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc/x86_64-redhat-linux-gnu/4.6.3/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc/x86_64-redhat-linux-gnu/4.6.3/../../.. -lgfortran -lm -lquadmath
libtool: link: mpic++ -fopenmp -I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost -o .libs/toast_mpi_fdist toast_mpi_fdist.o -fopenmp -Wl,-R -Wl,/projects/b1092/software/boost/1.61.0/lib -fopenmp -Wl,-R -Wl,/projects/b1092/software/boost/1.61.0/lib -Wl,-R -Wl,/projects/b1092/software/boost/1.61.0/lib /projects/b1092/software/lapack/3.6.0/lib64/liblapack.so /projects/b1092/software/lapack/3.6.0/lib64/libblas.so -fopenmp -L/projects/b1092/software/toast-python/lib ../../src/libtoast-mpi/.libs/libftoast-mpi.so ../../src/libtoast/.libs/libftoast.so ../../src/libtoast-mpi/.libs/libtoast-mpi.so ../../src/libtoast/.libs/libtoast.so -L/projects/b1092/TOAST_5/TOAST/src/libtoast/map -L/projects/b1092/TOAST_5/TOAST/src/libtoast/IO -L/projects/b1092/TOAST_5/TOAST/src/libtoast/math -L/projects/b1092/TOAST_5/TOAST/src/libtoast/math/tpm -lexpat -L/projects/b1092/software/moat/lib /projects/b1092/software/moat/lib/libmoat-mpi.so -L/projects/b1092/software/fftw/3.3.8/lib64 -L/projects/b1092/software/boost/1.61.0/lib -lboost_mpi /projects/b1092/software/moat/lib/libmoat.so -lfftw3 -L/projects/b1092/software/cfitsio/3.45/lib -lcfitsio -L/projects/b1092/software/getdata/0.9.3/lib /projects/b1092/software/getdata/0.9.3/lib/libgetdata++.so /software/gcc/4.6.3-rhel7/lib/../lib64/libstdc++.so /projects/b1092/software/getdata/0.9.3/lib/libgetdata.so -llzma -lbz2 -lz -L/projects/b1092/software/wcslib/5.15/lib -lwcs -lboost_serialization -lboost_regex -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc/x86_64-redhat-linux-gnu/4.6.3 -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc/x86_64-redhat-linux-gnu/4.6.3/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/hpc/software/gcc/4.6.3-rhel7/bin/../lib/gcc/x86_64-redhat-linux-gnu/4.6.3/../../.. /software/gcc/4.6.3-rhel7/lib/../lib64/libgfortran.so /software/gcc/4.6.3-rhel7/lib/../lib64/libquadmath.so -lm -lquadmath -Wl,-rpath -Wl,/projects/b1092/software/toast/lib -Wl,-rpath -Wl,/projects/b1092/software/moat/lib -Wl,-rpath -Wl,/projects/b1092/software/getdata/0.9.3/lib -Wl,-rpath -Wl,/software/gcc/4.6.3-rhel7/lib/../lib64
toast_mpi_fdist.o: In function 'MAIN__':
/projects/b1092/TOAST_5/TOAST/src/tests-mpi/toast_mpi_fdist.f03:76: undefined reference to 'mpi_init_'
/projects/b1092/TOAST_5/TOAST/src/tests-mpi/toast_mpi_fdist.f03:77: undefined reference to 'mpi_comm_rank_'
/projects/b1092/TOAST_5/TOAST/src/tests-mpi/toast_mpi_fdist.f03:78: undefined reference to 'mpi_comm_size_'
/projects/b1092/TOAST_5/TOAST/src/tests-mpi/toast_mpi_fdist.f03:205: undefined reference to 'mpi_barrier_'
/projects/b1092/TOAST_5/TOAST/src/tests-mpi/toast_mpi_fdist.f03:256: undefined reference to 'mpi_exscan_'
/projects/b1092/TOAST_5/TOAST/src/tests-mpi/toast_mpi_fdist.f03:266: undefined reference to 'mpi_barrier_'
/projects/b1092/TOAST_5/TOAST/src/tests-mpi/toast_mpi_fdist.f03:372: undefined reference to 'mpi_finalize_'
/projects/b1092/TOAST_5/TOAST/src/tests-mpi/toast_mpi_fdist.f03:87: undefined reference to 'mpi_abort_'
#it is the second step that has a problem, the /bin/sh one. We tried running the first two ourselves and the second one reproduced this error
#try adding include folder for mpi in configure in CFLAGS
#also adding -L/.../lib for mpi in LDFLAGS
#6/13/19
#toast
cd /projects/b1092/TOAST_5/TOAST
module purge
module use /projects/b1092/modules
module load python
module load gcc/4.6.3
module load mpi/openmpi-1.6.3-gcc-4.6.3
module load fftw
module load lapack
module load boost
module load moat
module load cfitsio
module load wcslib
module load getdata
module load automake/1.15
make clean
conda activate /projects/b1092/software/toast-python
#this configure has the mpi include in CFLAGS, CXXFLAGS, CPPFLAGS and has mpi lib64 in LDFLAGS
./configure --prefix=/projects/b1092/software/toast --with-lapack=/projects/b1092/software/lapack/3.6.0/lib64/liblapack.so --with-blas=/projects/b1092/software/lapack/3.6.0/lib64/libblas.so --with-cfitsio=/projects/b1092/software/cfitsio/3.45 --with-hdf5=no MPICC=mpicc MPICXX=mpic++ MPIFC=mpif90 CC=gcc CXX=g++ FC=gfortran --with-moatconfig=/projects/b1092/software/moat/bin/moatconfig --with-wcslib=/projects/b1092/software/wcslib/5.15 --enable-exp-blast --with-getdata=/projects/b1092/software/getdata/0.9.3 CFLAGS="-I/software/mpi/openmpi-1.6.3-gcc-4.6.3-RH7/include -I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CXXFLAGS="-I/software/mpi/openmpi-1.6.3-gcc-4.6.3-RH7/include -I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" CPPFLAGS="-I/software/mpi/openmpi-1.6.3-gcc-4.6.3-RH7/include -I/projects/b1092/software/wcslib/5.15/include/wcslib-5.15 -I/projects/b1092/software/getdata/0.9.3/include -I/projects/b1092/software/boost/1.61.0/include -I/projects/b1092/software/boost/1.61.0/include/boost" LDFLAGS="-I/software/mpi/openmpi-1.6.3-gcc-4.6.3-RH7/lib64 -L/projects/b1092/software/toast-python/lib" BOOST_ROOT=/projects/b1092/software/boost/1.61.0
make clean
make
#no errors!
make install
make check
#all (2) tests passed
#6/14/19
#made a module for toast that loads all the dependency modules, only have to `module use ...` and then `module load toast` now
module use /projects/b1092/modules
module load toast
#also noticed that the old toast module modified the python path to include pytoast, so I moved pytoast from the old project space to the new, and the toast module still adds this directory to the python path
#trying to use toast now:
#using old_toast_things/scripts/toast_mickey_500.sh
export SCRATCH=/projects/b1092/old_toast_things/runfiles