Merged in static_data_update (pull request #5)

Static data update
ec-jrc · Nov 18, 2019 · d1d1f78 · d1d1f78
2 parents f72a700 + 8b1bdde
commit d1d1f78
Show file tree

Hide file tree

Showing 4 changed files with 175 additions and 3 deletions.
diff --git a/configuration/global/geopotentials.json b/configuration/global/geopotentials.json
@@ -36,5 +36,13 @@
     "336.5$62.5$1377$657$904689$regular_ll":"iconeu_904689.grb",
     "350$40.04$201$161$32361$regular_ll":"icon_32361.grb",
     "0$359.859$M$1280$2140702$reduced_gg":"N640_2140702.grb",
-    "0$359.296$M$256$88838$reduced_gg":"seas5.grb"
+    "0$359.296$M$256$88838$reduced_gg":"seas5.grb",
+    "0$359.93$M$2560$6599680$reduced_gg":"N640@O1280_6599680.grb",
+    "0$359.75$1440$721$1038240$regular_ll":"icon_1038240.grb",
+    "336.5$45$1097$657$720729$regular_ll":"iconeu_720729.grb",
+    "-40$80$M$427$164585$reduced_gg":"O640_p80_m40_p20_p80.grb",
+    "-40$80$M$214$41826$reduced_gg":"O320_p80_m40_p20_p80.grb",
+    "-40$80$M$214$58717$reduced_gg":"efhs_58717.grb",
+    "-40$80$M$106$14996$reduced_gg":"efhs_14996.grb",
+    "0$359.723$M$640$421120$reduced_gg":"O320_421120.grb"
 }
diff --git a/configuration/global/global_conf.json b/configuration/global/global_conf.json
@@ -1,5 +1,5 @@
 {
-    "geopotentials": "/perm/ma/ma9/efas/data/eefas/pyg2pData/configuration/geopotentials",
-    "intertables": "/perm/ma/ma9/efas/data/eefas/pyg2pData/configuration/intertables"
+    "geopotentials": "/gpfs/lxc/efas/emos/data/pyg2p/2.1.0/geopotentials",
+    "intertables": "/gpfs/lxc/efas/emos/data/pyg2p/2.1.0/intertables"
 }
 
diff --git a/configuration/global/intertables.json b/configuration/global/intertables.json
@@ -1023,5 +1023,115 @@
             1500,
             3600
         ]
+    },
+    "I0_359.929_M_2560_6599680_reduced_gg_-180_90_0_0_-59.95_89.95_-180_90_0_0_0.05_359.95scipy_nearest":{
+        "filename":"O1280_6599680_to_1500x3600_nearest.npy",
+        "method":"nearest",
+        "source_shape":[
+            6599680
+        ],
+        "target_shape":[
+            1500,
+            3600
+        ]
+    },
+    "I0_359.93_M_2560_6599680_reduced_gg_-180_90_0_0_-59.95_89.95_-180_90_0_0_0.05_359.95scipy_nearest":{
+        "filename":"N640@O1280_6599680_to_1500x3600_nearest.npy",
+        "method":"nearest",
+        "source_shape":[
+            6599680
+        ],
+        "target_shape":[
+            1500,
+            3600
+        ]
+    },
+    "I-15.75_16.125_511_415_212065_rotated_ll_2568000_4163000_1000_-1000_31.63_60.56_2568000_4163000_1000_-1000_-19.97_40.95scipy_nearest":{
+        "filename":"cos_212065_to_2981x3571_nearest.npy",
+        "method":"nearest",
+        "source_shape":[
+            212065
+        ],
+        "target_shape":[
+            2981,
+            3571
+        ]
+    },
+    "I336.5_45_1097_657_720729_regular_ll_2500000_5500000_5000_-5000_23.94_72.64_2500000_5500000_5000_-5000_-35.03_73.94scipy_nearest":{
+        "filename":"iconeu_720729_to_950x1000_nearest.npy",
+        "method":"nearest",
+        "source_shape":[
+            720729
+        ],
+        "target_shape":[
+            950,
+            1000
+        ]
+    },
+    "I0_359.75_1440_721_1038240_regular_ll_2500000_5500000_5000_-5000_23.94_72.64_2500000_5500000_5000_-5000_-35.03_73.94scipy_nearest":{
+        "filename":"icon_1038240_to_950x1000_nearest.npy",
+        "method":"nearest",
+        "source_shape":[
+            1038240
+        ],
+        "target_shape":[
+            950,
+            1000
+        ]
+    },
+    "I-40_80_M_427_164585_reduced_gg_2500000_5500000_5000_-5000_23.94_72.64_2500000_5500000_5000_-5000_-35.03_73.94scipy_nearest":{
+        "filename":"O640_164585_to_950x1000_nearest.npy",
+        "method":"nearest",
+        "source_shape":[
+            164585
+        ],
+        "target_shape":[
+            950,
+            1000
+        ]
+    },
+    "I-40_80_M_214_41826_reduced_gg_2500000_5500000_5000_-5000_23.94_72.64_2500000_5500000_5000_-5000_-35.03_73.94scipy_nearest":{
+        "filename":"O320_41826_to_950x1000_nearest.npy",
+        "method":"nearest",
+        "source_shape":[
+            41826
+        ],
+        "target_shape":[
+            950,
+            1000
+        ]
+    },
+    "I-40_80_M_106_14996_reduced_gg_2500000_5500000_5000_-5000_23.94_72.64_2500000_5500000_5000_-5000_-35.03_73.94scipy_nearest":{
+        "filename":"efhs_14996_950x1000_nearest.npy",
+        "method":"nearest",
+        "source_shape":[
+            14996
+        ],
+        "target_shape":[
+            950,
+            1000
+        ]
+    },
+    "I-40_80_M_214_58717_reduced_gg_2500000_5500000_5000_-5000_23.94_72.64_2500000_5500000_5000_-5000_-35.03_73.94scipy_nearest":{
+        "filename":"efhs_58717_950x1000_nearest.npy",
+        "method":"nearest",
+        "source_shape":[
+            58717
+        ],
+        "target_shape":[
+            950,
+            1000
+        ]
+    },
+    "I0_359.723_M_640_421120_reduced_gg_-180_90_0_0_-59.95_89.95_-180_90_0_0_0.05_359.95scipy_nearest":{
+        "filename":"O320_421120B_to_1500x3600_nearest.npy",
+        "method":"nearest",
+        "source_shape":[
+            421120
+        ],
+        "target_shape":[
+            1500,
+            3600
+        ]
     }
 }
diff --git a/ecmwf_install_as_module b/ecmwf_install_as_module
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -ex
+
+mod_version=${1:-2.1.1}
+assets_version=$mod_version
+
+# Where is pyg2p source code
+src_dir=$PWD
+src_host=$HOST
+
+# where to install it
+dest_dir=/usr/local/apps/pyg2p/$mod_version
+
+#for host in localhost lxc ecgb lxop cca ccb; do
+for dest_host in lxc lxop cca ccb; do
+
+    case $dest_host in
+        lxc* )  static_data_root=/gpfs/lxc/efas/emos/data/pyg2p/$assets_version ;;
+        lxop* ) static_data_root=/gpfs/lxop/efas/emos/data/pyg2p/$assets_version ;;
+        cca* )  static_data_root=/sc1/tcwork/emos/emos_data/efas/assets/pyg2p/$assets_version ;;
+        ccb* )  static_data_root=/sc2/tcwork/emos/emos_data/efas/assets/pyg2p/$assets_version ;;
+        *   )   "unexpected target host $dest_host"; false ;;
+    esac
+
+    echo installing pyg2p/$mod_version in $dest_host ....
+    if ssh $dest_host [[ -d $dest_dir ]]; then
+        case $mod_version in
+            dev | test ) echo "reinstalling pyg2p/$mod_version" ;;
+            * ) echo "module pyg2p/$mod_version is already installed on $dest_host. skipping."; continue ;;
+        esac
+    fi
+    ssh $dest_host bash -l << END
+        module unload python
+        module load python/2.7.15-01
+        umask 022
+        set -eux
+        echo \$TMPDIR
+        cd \$TMPDIR
+        rsync -avz --exclude='.git/' $src_host:$src_dir .
+        cd pyg2p
+        echo '{
+            "geopotentials": "$static_data_root/geopotentials",
+            "intertables": "$static_data_root/intertables"
+        }
+        ' > configuration/global/global_conf.json
+        export HTTPS_PROXY=http://proxy.ecmwf.int:3333
+        export PYTHONPATH=/usr/local/apps/pyg2p/$mod_version/lib/python2.7/site-packages
+        python setup.py clean --all
+        mkdir -p \$PYTHONPATH
+        python setup.py install --prefix=/usr/local/apps/pyg2p/$mod_version
+END
+    echo done
+done