Upgrade to PHP 8.1 when installing NC >= 24 (#1554)

* Update nextcloud to 24.0.4
* ncp-update-nc: Upgrade php to version 8.1 when installing NC >= 24
* ncp-update-nc: Use /etc/shadow workaround for installing systemd
* ncp-update-nc: Run nc-limits after php upgrade
* ncp-update-nc: Rollback after failed php upgrade
* ncp-update-nc: Add success message
* ncp-update-nc: Prevent installation of NC >= 24 on debian 10/PHP <= 7.3
* lamp.sh: Install php8.1 from sury.org
* lamp.sh: Use /etc/shadow workaround for installing systemd
* Dockerfile: Install wget, ca-certificates, lsb-release and procps before installing lamp.sh
* Dockerfile: Make sure, ncp-templates are available when installing lamp.sh
* Migrate all scripts to use template for writing opcache.ini and get_nc_config_value for retrieving datadir
* nc-nextcloud.sh Fix crash if nc-datadir has not been installed yet
* opcache.ini.sh: Don't try to get tmpl values from nc-datadir in containers

.github/workflows/build-docker.yml

@@ -35,14 +35,6 @@ jobs:
     steps:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
-      - name: debugging
-        run: |
-          mount
-          echo 'DOCKER_OPTS="--storage-driver=devicemapper"' | sudo tee -a /etc/default/docker
-          sudo apt-get update && sudo apt-get -y upgrade
-          sudo systemctl restart docker
-          sudo apt-get install -y qemu-user-static
-          docker run --rm -v /usr/bin/qemu-arm-static:/usr/bin/qemu-arm-static arm32v7/debian:bullseye-slim bash -c 'apt-get update && apt-get install -y apache2 && apache2ctl -V' || true
       - name: Setup Docker Buildx
         uses: docker/setup-buildx-action@v1
 

bin/ncp-diag

@@ -19,8 +19,7 @@ echo "NextCloudPi version|$( cat /usr/local/etc/ncp-version )"
 echo "OS|$(cat /etc/issue  | sed 's| \\n \\l||'). $(uname -r) ($(uname -m))"
 
 # Data
-DATADIR="$( grep datadirectory /var/www/nextcloud/config/config.php |
-	    awk '{ print $3 }' | grep -oP "[^']*[^']" | head -1 )"
+DATADIR="$( get_nc_config_value datadirectory )"
 test -d "$DATADIR" || DIRINFO=" (doesn't exist)"
 USBDEVS="$( lsblk -S -o  NAME,TRAN | awk '{ if ( $2 == "usb" ) print $1; }' | tr '\n' ' ' )"
 [[ "$USBDEVS" == "" ]] && USBDEVS="none"

bin/ncp-dist-upgrade

@@ -95,20 +95,6 @@ $APTINSTALL -t ${release_new} php${php_ver_new}-gmp
 apt-get autoremove -y
 apt-get clean
 
-# configure latest PHP version
-cat > /etc/php/${php_ver_new}/mods-available/opcache.ini <<EOF
-zend_extension=opcache.so
-opcache.enable=1
-opcache.enable_cli=1
-opcache.fast_shutdown=1
-opcache.interned_strings_buffer=8
-opcache.max_accelerated_files=10000
-opcache.memory_consumption=128
-opcache.save_comments=1
-opcache.revalidate_freq=1
-opcache.file_cache=/tmp;
-EOF
-
 cat > /etc/php/${php_ver_new}/fpm/conf.d/90-ncp.ini <<EOF
 ; disable .user.ini files for performance and workaround NC update bugs
 user_ini.filename =
@@ -136,6 +122,8 @@ is_active_app unattended-upgrades && run_app unattended-upgrades || true
 
 # mark as successful
 mv "${new_cfg}" "${old_cfg}"
+install_template "php/opcache.ini.sh" "/etc/php/${php_ver_new}/mods-available/opcache.ini" --defaults
+service "php${php_ver_new}-fpm" restart
 
 source /usr/local/etc/library.sh # refresh NCPCFG RELEASE PHPVER
 run_app nc-limits

bin/ncp-report

@@ -62,8 +62,9 @@ close_summary
 
 ##
 
-DATADIR="$( grep datadirectory /var/www/nextcloud/config/config.php |
-            awk '{ print $3 }' | grep -oP "[^']*[^']" | head -1 )"
+DATADIR="$( get_nc_config_value datadirectory ||
+            grep datadirectory /var/www/nextcloud/config/config.php |
+              awk '{ print $3 }' | grep -oP "[^']*[^']" | head -1 )"
 
 open_summary "Nextcloud logs"
 tail -20 "$DATADIR"/nextcloud.log

bin/ncp-update-nc

@@ -29,7 +29,7 @@ source /usr/local/etc/library.sh
 [[ "$VER" == "" ]] && { echo "Usage ${BIN} <version>"; exit 1; }
 [[ -f /.docker-image ]] && BASEDIR=/data || BASEDIR=/var/www
 cd "$BASEDIR"
-DATADIR="$( grep datadirectory nextcloud/config/config.php | awk '{ print $3 }' | grep -oP "[^']*[^']" | head -1 )"
+DATADIR="$( get_nc_config_value datadirectory )"
 ncc status &>/dev/null          || { echo "Nextcloud is currently down"; exit 1; }
 [[ -d /var/www/nextcloud-old ]] && { echo "Nextcloud backup directory found. Interrupted or already running installation?"; exit 1; }
 [[ -d /var/www/nextcloud     ]] || { echo "Nextcloud directory not found"      ; exit 1; }
@@ -48,13 +48,25 @@ if [[ $((MAJOR_NEW - MAJOR_CUR)) -gt 1 ]]; then
   exit 1
 fi
 
+if [[ "$MAJOR_NEW" -ge 24 ]] && [[ "$(lsb_release -r)" =~ .*10 ]]
+then
+  echo -e "Nextcloud version greater than 23 are not supported with Debian 10 (Buster). Please run ncp-dist-upgrade."
+  exit 1
+fi
+
 grep -qP "\d+\.\d+\.\d+" <<<"$CURRENT" || { echo "Malformed version $CURRENT"; exit 1; }
 grep -qP "\d+\.\d+\.\d+" <<<"$VER"     || { echo "Malformed version $VER"    ; exit 1; }
 
 echo "Current   Nextcloud version $CURRENT"
 echo "Available Nextcloud version $VER"
 is_more_recent_than "${VER}" "${CURRENT}" || { echo "Nothing to update"; exit 1; } # we want `exit 1` so the autoupdate doesn't notify success in this case
 
+if ! is_more_recent_than "24.0.0" "${VER}" && is_more_recent_than "7.4.0" "${PHPVER}.0"
+then
+  echo -e "Upgrading to Nextcloud versions > 23 requires the latest debian and PHP versions. Please run \`ncp-dist-upgrade\` and then run the update again."
+  exit 1
+fi
+
 # make sure that cron.php is not running and there are no pending jobs
 # https://github.com/nextcloud/server/issues/10949
 pgrep -cf cron.php &>/dev/null && { pkill -f cron.php; sleep 3; }
@@ -179,7 +191,6 @@ $ncc | grep -q db:add-missing-columns && $ncc db:add-missing-columns -n
 $ncc | grep -q db:add-missing-primary-keys && $ncc db:add-missing-primary-keys -n
 $ncc | grep -q db:convert-filecache-bigint && $ncc db:convert-filecache-bigint -n
 
-
 # use the correct version for custom apps
 NCVER="$(ncc status | grep "version:" | awk '{ print $3 }')"
 if is_more_recent_than "21.0.0" "${NCVER}"; then
@@ -189,7 +200,7 @@ else
   if ! is_app_enabled notify_push; then
     ncc app:install notify_push
     ncc app:enable  notify_push
-    bash /usr/local/etc/ncp-templates/nextcloud.conf.sh > /etc/apache2/sites-available/nextcloud.conf
+    install_template nextcloud.conf.sh /etc/apache2/sites-available/nextcloud.conf
     a2enmod proxy proxy_http proxy_wstunnel
     apachectl -k graceful
     ## make sure the notify_push daemon is runnnig
@@ -224,9 +235,80 @@ fi
 rm -rf /var/www/nextcloud/apps/previewgenerator
 ln -snf "${NCPREV}" /var/www/nextcloud/apps/previewgenerator
 
+
+if ! is_more_recent_than "24.0.0" "${NCVER}" && is_more_recent_than "8.1.0" "${PHPVER}.0"
+then
+  (
+  echo "Upgrading PHP..."
+  export DEBIAN_FRONTEND=noninteractive
+  PHPVER_OLD="$PHPVER"
+  PHPVER_NEW="8.1"
+  PHP_PACKAGES_OLD=(php-{common,igbinary,redis} "php${PHPVER_OLD}" \
+    "php${PHPVER_OLD}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,json,common,readline,mysql,bcmath,gmp})
+  PHP_PACKAGES_NEW=("php${PHPVER_NEW}" php-json \
+    "php${PHPVER_NEW}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,mysql,bcmath,gmp,redis,common})
+
+  php_restore() {
+    trap "" INT TERM HUP ERR
+    echo "Something went wrong while upgrading PHP. Rolling back to version ${PHPVER_OLD}..."
+    set +e
+    service "php${PHPVER_NEW}-fpm" stop
+    a2disconf php${PHPVER_NEW}-fpm
+    rm /etc/apt/sources.list.d/php.list
+    apt-get update
+    apt-get remove --purge -y "${PHP_PACKAGES_NEW[@]}" systemd
+    apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_OLD[@]}"
+    set_ncpcfg "php_version" "${PHPVER_OLD}"
+    install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini"
+    run_app nc-limits
+    a2enconf "php${PHPVER_OLD}-fpm"
+    service "php${PHPVER_OLD}-fpm" start
+    service apache2 restart
+    echo "PHP upgrade has been successfully reverted"
+    set -e
+  }
+
+  trap php_restore INT TERM HUP ERR
+
+  # Setup apt repository for php 8
+  wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg
+  echo "deb https://packages.sury.org/php/ ${RELEASE%-security} main" > /etc/apt/sources.list.d/php.list
+  apt-get update
+
+  echo "Stopping apache and  php-fpm..."
+  service "php${PHPVER_OLD}-fpm" stop
+  service apache2 stop
+
+  echo "Remove old PHP (${PHPVER_OLD})..."
+  a2disconf "php${PHPVER_OLD}-fpm"
+
+  apt-get remove --purge -y "${PHP_PACKAGES_OLD[@]}"
+
+  echo "Install PHP ${PHPVER_NEW}..."
+  install_with_shadow_workaround --no-install-recommends systemd
+  apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_NEW[@]}"
+
+  set_ncpcfg "php_version" "${PHPVER_NEW}"
+  install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini"
+  ( export PHPVER="${PHPVER_NEW}"; run_app nc-limits )
+  a2enconf php${PHPVER_NEW}-fpm
+
+  echo "Starting apache and php-fpm..."
+  service "php${PHPVER_NEW}-fpm" start
+  service apache2 start
+  ncc status
+  )
+
+  # Reload library.sh to reset PHPVER
+  source /usr/local/etc/library.sh
+
+fi
+
+
 # refresh completions
 ncc _completion -g --shell-type bash -p ncc | sed 's|/var/www/nextcloud/occ|ncc|g' > /usr/share/bash-completion/completions/ncp
 
+echo "Update completed successfully."
 # done
 ####################
 mkdir -p  "$DATADIR"/ncp-update-backups

bin/ncp/BACKUPS/nc-backup.sh

@@ -43,7 +43,7 @@ occ="sudo -u www-data php /var/www/nextcloud/occ"
 
 [[ "$compress" == "yes" ]] && destfile="$destfile".gz
 
-datadir=$( $occ config:system:get datadirectory ) || {
+datadir=$( get_nc_config_value datadirectory ) || {
   echo "Error reading data directory. Is NextCloud running and configured?";
   exit 1;
 }

bin/ncp/BACKUPS/nc-restore-snapshot.sh

@@ -16,7 +16,7 @@ configure()
   [[ -d "$SNAPSHOT" ]] || { echo "$SNAPSHOT doesn't exist"; return 1; }
 
   local datadir mountpoint
-  datadir=$( ncc config:system:get datadirectory ) || {
+  datadir=$( get_nc_config_value datadirectory ) || {
     echo -e "Error reading data directory. Is NextCloud running?";
     return 1;
   }

bin/ncp/BACKUPS/nc-restore.sh

@@ -96,7 +96,7 @@ mysql -u root nextcloud <  "$TMPDIR"/nextcloud-sqlbkp_*.bak || { echo "Error res
 if is_docker; then
   DATADIR=/data/nextcloud/data
 else
-  DATADIR="$(grep datadirectory "$NCDIR"/config/config.php | awk '{ print $3 }' | grep -oP "[^']*[^']" | head -1)"
+  DATADIR="$(get_nc_config_value datadirectory)"
 fi
 [[ "$DATADIR" == "" ]] && { echo "Error reading data directory"; exit 1; }
 
@@ -107,7 +107,7 @@ cd "$NCDIR"
 NUMFILES=2
 if [[ $( ls "$TMPDIR" | wc -l ) -eq $NUMFILES ]]; then
 
-  [[ -e "$DATADIR" ]] && { 
+  [[ -e "$DATADIR" ]] && {
     echo "backing up existing $DATADIR to $DATADIR-$( date "+%m-%d-%y" )..."
     mv "$DATADIR" "$DATADIR-$( date "+%m-%d-%y" )" || exit 1
   }
@@ -149,7 +149,7 @@ fi
 sed -i "s|'datadirectory' =>.*|'datadirectory' => '${DATADIR}',|" "${NCDIR}"/config/config.php
 
 # Just in case we moved the opcache dir
-sed -i "s|^opcache.file_cache=.*|opcache.file_cache=$DATADIR/.opcache|" /etc/php/${PHPVER}/mods-available/opcache.ini
+install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini"
 
 # tmp upload dir
 mkdir -p "$DATADIR/tmp"

bin/ncp/BACKUPS/nc-rsync-auto.sh

@@ -16,14 +16,14 @@ install()
 
 configure()
 {
-  [[ $ACTIVE != "yes" ]] && { 
+  [[ $ACTIVE != "yes" ]] && {
     rm -f /etc/cron.d/ncp-rsync-auto
     echo "automatic rsync disabled"
     return 0
   }
 
   local DATADIR
-  DATADIR=$( ncc config:system:get datadirectory ) || {
+  DATADIR=$( get_nc_config_value datadirectory ) || {
     echo -e "Error reading data directory. Is NextCloud running and configured?";
     return 1;
   }

bin/ncp/BACKUPS/nc-rsync.sh

@@ -19,7 +19,7 @@ configure()
   save_maintenance_mode
 
   local DATADIR
-  DATADIR=$( sudo -u www-data php /var/www/nextcloud/occ config:system:get datadirectory ) || {
+  DATADIR=$( get_nc_config_value datadirectory ) || {
     echo -e "Error reading data directory. Is NextCloud running and configured?";
     return 1;
   }

bin/ncp/BACKUPS/nc-snapshot-auto.sh

@@ -26,7 +26,7 @@ configure()
 
   cat > /etc/cron.hourly/btrfs-snp <<EOF
 #!/bin/bash
-DATADIR=\$(ncc config:system:get datadirectory) || {
+DATADIR=\$(get_nc_config_value datadirectory) || {
   echo -e "Error reading data directory. Is NextCloud running and configured?";
   exit 1;
 }

bin/ncp/BACKUPS/nc-snapshot.sh

@@ -20,7 +20,7 @@ configure()
   save_maintenance_mode
 
   local DATADIR MOUNTPOINT
-  DATADIR=$( ncc config:system:get datadirectory ) || {
+  DATADIR=$( get_nc_config_value datadirectory ) || {
     echo -e "Error reading data directory. Is NextCloud running?";
     return 1;
   }

bin/ncp/CONFIG/nc-database.sh

@@ -15,6 +15,12 @@ is_active()
   [[ "$SRCDIR" != "/var/lib/mysql" ]]
 }
 
+tmpl_db_dir() {
+  if is_active_app nc-database; then
+    find_app_param nc-database DBDIR
+  fi
+}
+
 configure()
 {
   local SRCDIR=$( grep datadir /etc/mysql/mariadb.conf.d/90-ncp.cnf | awk -F "= " '{ print $2 }' )
@@ -25,14 +31,14 @@ configure()
       echo "$DBDIR is not empty"
       return 1
     }
-    rmdir "$DBDIR" 
+    rmdir "$DBDIR"
   }
 
   local BASEDIR=$( dirname "$DBDIR" )
   mkdir -p "$BASEDIR"
 
   grep -q -e ext -e btrfs <( stat -fc%T "$BASEDIR" ) || { echo -e "Only ext/btrfs filesystems can hold the data directory"; return 1; }
-  
+
   sudo -u mysql test -x "$BASEDIR" || { echo -e "ERROR: the user mysql does not have access permissions over $BASEDIR"; return 1; }
 
   [[ $( stat -fc%d / ) == $( stat -fc%d "$BASEDIR" ) ]] && \
@@ -42,9 +48,9 @@ configure()
 
   echo "moving database to $DBDIR..."
   service mysql stop
-  mv "$SRCDIR" "$DBDIR" && \
-    sed -i "s|^datadir.*|datadir = $DBDIR|" /etc/mysql/mariadb.conf.d/90-ncp.cnf
-  service mysql start 
+  mv "$SRCDIR" "$DBDIR"
+  install_template "mysql/90-ncp.cnf.sh" "/etc/mysql/mariadb.conf.d/90-ncp.cnf"
+  service mysql start
 
   restore_maintenance_mode
 }

bin/ncp/CONFIG/nc-datadir.sh

@@ -20,14 +20,45 @@ install()
   apt_install btrfs-progs
 }
 
+tmpl_opcache_dir() {
+  DATADIR="$(get_nc_config_value datadirectory)"
+  echo -n "${DATADIR}/.opcache"
+  #[[ $( stat -fc%d / ) == $( stat -fc%d "$DATADIR" ) ]] && echo "/tmp" || echo "${DATADIR}/.opcache"
+}
+
+tmpl_tmp_upload_dir() {
+  DATADIR="$(get_nc_config_value datadirectory)"
+  echo -n "${DATADIR}/tmp"
+}
+
+create_opcache_dir() {
+  OPCACHE_DIR="$(tmpl_opcache_dir)"
+  mkdir -p "$OPCACHE_DIR"
+  chown -R www-data:www-data "$OPCACHE_DIR"
+  if [[ "$(stat -fc%T "${BASEDIR}")" == "btrfs" ]]
+  then
+    chattr -R +C "$OPCACHE_DIR"
+  fi
+}
+
+create_tmp_upload_dir() {
+  UPLOAD_DIR="$(tmpl_tmp_upload_dir)"
+  mkdir -p "${UPLOAD_DIR}"
+  chown www-data:www-data "${UPLOAD_DIR}"
+  if [[ "$(stat -fc%T "${BASEDIR}")" == "btrfs" ]]
+  then
+    chattr +C "${UPLOAD_DIR}"
+  fi
+}
+
 configure()
 {
   set -e -o pipefail
   shopt -s dotglob # includes dot files
 
   ## CHECKS
   local SRCDIR BASEDIR ENCDIR
-  SRCDIR=$( cd /var/www/nextcloud; ncc config:system:get datadirectory ) || {
+  SRCDIR=$( get_nc_config_value datadirectory ) || {
     echo -e "Error reading data directory. Is NextCloud running and configured?";
     return 1;
   }
@@ -98,15 +129,15 @@ configure()
   set_ncpcfg datadir "${DATADIR}"
 
   # tmp upload dir
-  mkdir -p "${DATADIR}/tmp"
-  chown www-data:www-data "${DATADIR}/tmp"
+  create_tmp_upload_dir
   ncc config:system:set tempdirectory --value "$DATADIR/tmp"
   sed -i "s|^;\?upload_tmp_dir =.*$|uploadtmp_dir = ${DATADIR}/tmp|"  /etc/php/"${PHPVER?}"/cli/php.ini
   sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = ${DATADIR}/tmp|" /etc/php/"${PHPVER}"/fpm/php.ini
   sed -i "s|^;\?sys_temp_dir =.*$|sys_temp_dir = ${DATADIR}/tmp|"     /etc/php/"${PHPVER}"/fpm/php.ini
 
   # opcache dir
-  sed -i "s|^opcache.file_cache=.*|opcache.file_cache=${DATADIR}/.opcache|" /etc/php/"${PHPVER}"/mods-available/opcache.ini
+  create_opcache_dir
+  install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini"
 
   # update fail2ban logpath
   [[ -f /etc/fail2ban/jail.local ]] && \