merge

bin/make_env_buildcache

@@ -54,6 +54,13 @@ make_reconstitute_script() {
     upstream=$(spack config get upstreams | 
                 grep install_tree |
                 sed -e 's/.*: *//' -e 's/  *$//')
+
+    # plain layout we have the setup-env script at $SPACK_ROOT
+    case "$upstream" in
+    *.opt/spack) 
+          upstream=$SPACK_ROOT
+          ;;
+     esac
 
     # get the environment name from $SPACK_ENV
     active=$(echo $SPACK_ENV | sed -e 's;.*/;;')

bin/make_spack

@@ -20,7 +20,7 @@ install_latest() {
         hash_pkg_ver=`spack -k buildcache list --long --allarch "$pkg $plat" | tail -1`
         echo "make_spack: info: latest $pkg is $hash_pkg_ver"
         hash=`echo $hash_pkg_ver | sed -e 's/ .*//'`
-        spack -k buildcache install -o -a -m "/$hash"
+        spack -k buildcache install -o -m "/$hash"
     else
         spack -k install "$pkg $plat"
     fi
@@ -146,12 +146,17 @@ create_spack() {
                ;;
             esac
             export plat="target=`uname -i` os=`spack arch --operating-system`"
-            spack mirror add --scope site fnal $binary_cache
-            spack buildcache update-index -k fnal
-            spack mirror add --scope site scisoft $binary_cache_bak
-            spack buildcache update-index -k scisoft
-            spack -k buildcache keys --install --trust --force
-            add_recipe_repos
+            if $minimal
+            then
+              :
+            else
+              spack mirror add --scope site fnal $binary_cache
+              spack buildcache update-index -k fnal
+              spack mirror add --scope site scisoft $binary_cache_bak
+              spack buildcache update-index -k scisoft
+              spack -k buildcache keys --install --trust --force
+            fi
+	    add_recipe_repos
             if $upgrading
             then
                 spack reindex

bin/make_subspack

@@ -20,6 +20,7 @@ usage() {
         echo "-v                  verbose"
         echo "-t                  traditional layout"
         echo "-u                  unified layout (default)"
+        echo "-p                  plain layout"
 }
 
 parse_args() {
@@ -29,8 +30,9 @@ parse_args() {
     spack_release=default
     padding=false
     verbose=false
+    plain=false
 
-    eval set : $(getopt --longoptions with_padding,spack_release:,spack_repo,local_env:,dev_pkgs: --options vtu -- "$@")
+    eval set : $(getopt --longoptions with_padding,spack_release:,spack_repo,local_env:,dev_pkgs: --options vtup -- "$@")
     shift
 
     while echo x$1 | grep x- > /dev/null
@@ -43,7 +45,8 @@ parse_args() {
         x--dev_pkgs)      dev_pkgs=$2; shift; shift;;
         x-t) unified=false; shift ;;
         x-u) unified=true; shift ;;
-        x-u) verbose=true; shift ;;
+        x-p) plain=true; unified=false; shift ;;
+        x-v) verbose=true; shift ;;
         x--) shift; break;;
         esac
     done
@@ -136,19 +139,35 @@ else
 fi
 
 # assume unified layout...
+if $plain
+then
+   SPACK_ROOT=$dst
+else
+
 if $unified
 then
-    install_tree_path=$src
     SPACK_ROOT="$dst/spack/$spack_release/NULL/"
-    src=`ls -d $src/spack/*/NULL/ | tail -1`
-    echo "src=$src"
 else
-    install_tree_path=$src/prd
     SPACK_ROOT="$dst/prd/spack/$spack_release/NULL/"
-    src="$src/prd/spack/$spack_release/NULL/"
 fi
+fi
+
+install_tree_path=$src
+src=`ls $src/spack/*/NULL/bin/spack $src/bin/spack $src/prd/spack/*/NULL/bin/spack 2>/dev/null | tail -1 | sed -e 's;/bin/spack;;'`
+echo "src=$src"
+
 install_tcl_path=$src/share/spack/modules
 
+if [ -d $install_tree_path/prd/ ]
+then
+    install_tree_path=$install_tree_path/prd/ 
+fi
+
+if [ -d $install_tree_path/opt/spack/.spack-db ]
+then
+    install_tree_path=$install_tree_path/opt/spack
+fi
+
 if [ -d "$install_tree_path/__spack_path_placeholder__" ]
 then
     message "Checking for padded .spack-db"
@@ -189,12 +208,17 @@ git clone $args $branchbits $spack_repo $SPACK_ROOTb
 cd $SPACK_ROOT
 
 echo "installing config.yaml..."
+if $plain
+then
+   printf "config:\n  deprecated: false\n" >> $SPACK_ROOT/etc/spack/config.yaml
+else
 if $unified
 then
 cp $spackbindir/../templates/config.yaml.unified${extra} $SPACK_ROOT/etc/spack/config.yaml
 else
 cp $spackbindir/../templates/config.yaml.traditional${extra} $SPACK_ROOT/etc/spack/config.yaml
 fi
+fi
 
 os=`$SPACK_ROOT/bin/spack arch --operating-system`
 
@@ -245,6 +269,7 @@ esac
 
 # copy compilers and packages
 test -d  $SPACK_ROOT/etc/spack/$os ||  mkdir -p  $SPACK_ROOT/etc/spack/$os
+[ -r  $src/etc/spack/compilers.yaml ] && cp $src/etc/spack/compilers.yaml $SPACK_ROOT/etc/spack/compilers.yaml
 [ -r  $src/etc/spack/$os/compilers.yaml ] && cp $src/etc/spack/$os/compilers.yaml $SPACK_ROOT/etc/spack/$os/compilers.yaml
 [ -r  $src/etc/spack/$os/packages.yaml ] && cp $src/etc/spack/$os/packages.yaml $SPACK_ROOT/etc/spack/$os/packages.yaml
 
@@ -287,7 +312,7 @@ do
 done
 
 # use the upstream's bootstrap area...
-$SPACK_ROOT/bin/spack bootstrap root --scope=site $install_tree_path/.bootstrap
+$SPACK_ROOT/bin/spack bootstrap root --scope=site $(SPACK_ROOT=$src spack bootstrap root)
 
 if [ "$local_env" != "" ]
 then

bin/sync_from_jenkins_local

@@ -0,0 +1,40 @@
+#!/bin/bash
+curl -OL https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/label1=$1/$2/artifact/copyBack/jenkins-build-spack-env-label1=$1-$2.tar.bz2
+mkdir $1-$2; tar -x  -C $1-$2 -f jenkins-build-spack-env-label1=$1-$2.tar.bz2
+#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-emergency-cache/ 2>/dev/null | grep -q 404 ); then
+	spack mirror add $1-$2e --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-emergency-cache/
+#fi
+#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-binary-cache/ 2>/dev/null | grep -q 404 ); then
+	spack mirror add $1-$2b --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-binary-cache/
+#fi
+#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-compiler-cache/ 2>/dev/null | grep -q 404 );then 
+	spack mirror add $1-$2c --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-compiler-cache
+#fi
+spack mirror list
+if [ -d $1-$2/spack_env/var/spack/environments ]; then
+	mkdir local-binary-plain
+	spack mirror add local-binary-plain $PWD/local-binary-plain
+	for env in $(ls -d $1-$2/spack_env/var/spack/environments/*);do
+		spack -e $env buildcache sync $1-$2b local-binary-plain
+		spack -e $env buildcache sync $1-$2e local-binary-plain
+	spack buildcache update-index local-binary-plain
+	done
+	mkdir local-compiler-plain
+	spack mirror add local-compiler-plain $PWD/local-compiler-plain
+	for env in $(ls -d $1-$2/spack_env/var/spack/environments/{gcc,clang}*);do
+		spack -e $env buildcache sync $1-$2c local-compiler-plain
+	done
+	spack buildcache update-index local-compiler-plain
+fi
+if [ -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments ]; then
+	for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/*);do
+		spack -e $env buildcache sync $1-$2b local-binary-unified
+		spack -e $env buildcache sync $1-$2e local-binary-unified
+	done
+	spack buildcache update-index local-binary-unified
+	for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/{gcc,clang}*);do
+		spack -e $env buildcache sync $1-$2b local-compiler-unified
+	done
+	spack buildcache update-index local-compiler-unified
+fi
+

bin/sync_from_jenkins_scisoft

@@ -0,0 +1,36 @@
+#!/bin/bash
+curl -OL https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/label1=$1/$2/artifact/copyBack/jenkins-build-spack-env-label1=$1-$2.tar.bz2
+mkdir $1-$2; tar -x  -C $1-$2 -f jenkins-build-spack-env-label1=$1-$2.tar.bz2
+#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-emergency-cache/ 2>/dev/null | grep -q 404 ); then
+	spack mirror add $1-$2e --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-emergency-cache/
+#fi
+#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-binary-cache/ 2>/dev/null | grep -q 404 ); then
+	spack mirror add $1-$2b --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-binary-cache/
+#fi
+#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-compiler-cache/ 2>/dev/null | grep -q 404 );then 
+	spack mirror add $1-$2c --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-compiler-cache
+#fi
+spack mirror list
+if [ -d $1-$2/spack_env/var/spack/environments ]; then
+	for env in $(ls -d $1-$2/spack_env/var/spack/environments/*);do
+		spack -e $env buildcache sync $1-$2b local-binary-plain
+		spack -e $env buildcache sync $1-$2e local-binary-plain
+	done
+        spack buildcache update index local-binary-plain
+	for env in $(ls -d $1-$2/spack_env/var/spack/environments/{gcc,clang}*);do
+		spack -e $env buildcache sync $1-$2c local-compiler-plain
+	done
+        spack buildcache update index local-compiler-plain
+fi
+if [ -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments ]; then
+	for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/*);do
+		spack -e $env buildcache sync $1-$2b local-binary-unified
+		spack -e $env buildcache sync $1-$2e local-binary-unified
+	done
+        spack buildcache update index local-binary-unified
+	for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/{gcc,clang}*);do
+		spack -e $env buildcache sync $1-$2b local-compiler-unified
+	done
+        spack buildcache update index local-compiler-unified
+fi
+

bin/ups_to_spack

@@ -457,11 +457,7 @@ class package:
             tf_2 = re.sub(".*-2.6", "scientificfermi5", tf_2)
             tf_2 = re.sub("^[0-9].*", "scientificfermi5", tf_2)
 
-        # excessive intel-centrism...
-        if f_os.find("64bit") > 0:
-            tf_3 = "x86_64_v2"
-        else:
-            tf_3 = "x86"
+        tf_3 = "x86_64_v2"
 
         logging.debug("spack_arch(%s) -> %s-%s-%s" % (self.flav, tf_1, tf_2, tf_3))
         return "%s-%s-%s" % (tf_1, tf_2, tf_3)
@@ -1941,7 +1937,7 @@ uts = ups_to_spack(pdr=pdr)
 
 if not os.path.exists("/tmp/empty.tar"):
     d = os.path.dirname(os.path.dirname(__file__))
-    os.system("cp {0}/templates/emtpy.tar /tmp/empty.tar .".format(d))
+    os.system("cp {0}/templates/emtpy.tar /tmp/empty.tar".format(d))
 
 # handle --use_spack args: prepopulate conversion cache
 for us in use_spack:

templates/compiler-projections.yaml.plain

@@ -0,0 +1,6 @@
+spack:
+  config:
+    install_tree:
+      projections:
+        gcc: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}"
+        llvm: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}"

templates/compiler-projections.yaml.unified

@@ -0,0 +1,6 @@
+spack:
+  config:
+    install_tree:
+      projections:
+        gcc: "{package.name}/{version}/{architecture}-{compiler.name}-{compiler.version}"
+        llvm: "{package.name}/{version}{architecture}-{compiler.name}-{compiler.version}"

templates/packagelist

@@ -54,6 +54,7 @@ libfontenc:libfontenc:$0-devel
 libICE:libice:$0-devel
 :libpciaccess:
 libSM:libsm:$0-devel
+libtirpc:libtirpc:$0-devel
 libtool:libtool:$0
 libuuid:libuuid:$0-devel
 libX11:libx11:$0-devel