Merge branch 'develop'

Preparing for release

.editorconfig

@@ -8,6 +8,11 @@ charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
 
+; applies only to Makefiles
+[Makefile]
+indent_style = tab
+tab_width = 4
+
 ; applies only to JS, TS, and YML files
 [*.{js,ts,yml}]
 tab_width = 2

.github/workflows/integrate-and-deploy.yml

@@ -51,6 +51,11 @@ jobs:
           echo ::set-output name=TAG_APP::${{ inputs.image-tag-app }}
           echo ::set-output name=TAG_PROXY::${{ inputs.image-tag-proxy }}
           echo ::set-output name=TAG_NEXT_APP::${{ inputs.image-tag-next-app }}
+          echo ::set-output name=LFMERGE_NAMESPACE::ghcr.io/sillsdev/lfmerge
+          # Get LfMerge tag from Dockerfile, fallback to "latest" as default if that fails
+          TAG_LFMERGE=$(head -n 1 lfmerge/Dockerfile | cut -d: -f2)
+          TAG_LFMERGE=${TAG_LFMERGE:-latest}
+          echo ::set-output name=TAG_LFMERGE::${TAG_LFMERGE}
       -
         uses: actions/setup-node@v2
         with:
@@ -98,6 +103,7 @@ jobs:
       IMAGE_APP: ${{ steps.image.outputs.NAMESPACE }}:${{ steps.image.outputs.TAG_APP }}
       IMAGE_PROXY: ${{ steps.image.outputs.NAMESPACE }}:${{ steps.image.outputs.TAG_PROXY }}
       IMAGE_NEXT_APP: ${{ steps.image.outputs.NAMESPACE }}:${{ steps.image.outputs.TAG_NEXT_APP }}
+      IMAGE_LFMERGE: ${{ steps.image.outputs.LFMERGE_NAMESPACE }}:${{ steps.image.outputs.TAG_LFMERGE }}
 
   deploy:
     runs-on: [self-hosted, languageforge]
@@ -112,3 +118,4 @@ jobs:
           kubectl --context ${{ secrets.kube-context }} set image deployment/app app=${{ needs.integrate.outputs.IMAGE_APP }}
           kubectl --context ${{ secrets.kube-context }} set image deployment/next-proxy next-proxy=${{ needs.integrate.outputs.IMAGE_PROXY }}
           kubectl --context ${{ secrets.kube-context }} set image deployment/next-app next-app=${{ needs.integrate.outputs.IMAGE_NEXT_APP }}
+          kubectl --context ${{ secrets.kube-context }} set image deployment/lfmerge lfmerge=${{ needs.integrate.outputs.IMAGE_LFMERGE }}

docker/Makefile

@@ -49,7 +49,7 @@ unit-tests-ci:
 .PHONY: build
 build:
 	npm install
-	docker-compose build mail app ld-api next-proxy next-app
+	docker-compose build mail app lfmerge ld-api next-proxy next-app
 
 .PHONY: scan
 # https://docs.docker.com/engine/scan

docker/app/Dockerfile

@@ -75,8 +75,6 @@ COPY --from=ui-builder /data/src/dist /var/www/html/dist
 RUN    chown -R www-data:www-data /var/www/html/assets /var/www/html/cache \
     && chmod -R g+ws /var/www/html/assets /var/www/html/cache
 
-COPY docker/app/entrypoint.sh /
-
 COPY --from=composer-builder /composer/vendor /var/www/html/vendor
 
 # patch exception handling from Symfony to actually show exceptions instead of swallowing them
@@ -86,5 +84,4 @@ RUN patch -p4 -i /symfony-exceptions.patch
 RUN echo "${BUILD_VERSION}" > /var/www/html/build-version.txt \
  && sed -i /var/www/html/version.php -e "s/^\\(define('VERSION', '\\).*;\$/\\1${BUILD_VERSION}'\\);/"
 
-ENTRYPOINT [ "/entrypoint.sh" ]
 CMD [ "apache2-foreground" ]

docker/base-php/Dockerfile

@@ -3,38 +3,14 @@ FROM php:7.3.28-apache
 # install apt packages
 # p7zip-full - used by LF application for unzipping lexicon uploads
 # unzip - used by LF application for unzipping lexicon uploads
-# gnupg2 - necessary for LFMerge package installation via SIL sources (will be uninstalled in production)
 # curl - used by LF application
-RUN apt-get update && apt-get -y install p7zip-full unzip gnupg2 curl && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get -y install p7zip-full unzip curl && rm -rf /var/lib/apt/lists/*
 
 # see https://github.com/mlocati/docker-php-extension-installer
 # PHP extensions required by the LF application
 COPY --from=mlocati/php-extension-installer /usr/bin/install-php-extensions /usr/local/bin/
 RUN install-php-extensions gd mongodb intl
 
-# install and configure LFMerge
-# LFMerge required apt packages
-# python - required by Mercurial (written in Python), which is bundled in the LFMerge apt package
-# lfmerge - main package, from SIL sources
-# rsyslog - lfmerge logs to rsyslog and expects this to exist
-# logrotate - TODO: is this required?
-# iputils-ping - Chorus (part of LFMerge) requires the "ping" command to be available on the command line
-RUN curl -L http://linux.lsdev.sil.org/downloads/sil-testing.gpg | apt-key add - \
-&& echo "deb http://linux.lsdev.sil.org/ubuntu bionic main" > /etc/apt/sources.list.d/linux-lsdev-sil-org.list \
-&& apt-get update \
-&& apt-get install --yes --no-install-recommends python lfmerge rsyslog logrotate iputils-ping \
-&& rm -rf /var/lib/apt/lists/*
-COPY docker/base-php/lfmerge.conf /etc/languageforge/conf/sendreceive.conf
-COPY docker/base-php/lfmergeqm-background.sh /
-RUN adduser www-data fieldworks \
-&& chown -R www-data:www-data /var/lib/languageforge \
-&& chmod 0755 /var/lib/languageforge \
-&& mkdir -m 02775 -p /var/www/.local \
-&& chown www-data:www-data /var/www/.local
-
-# rsyslog customizations (imklog reads kernel messages, which isn't allowed or desired in Docker containers)
-RUN sed -i '/load="imklog"/s/^/#/' /etc/rsyslog.conf
-
 # php customizations
 COPY docker/base-php/customizations.php.ini $PHP_INI_DIR/conf.d/
 

docker/deployment/Makefile

@@ -10,9 +10,11 @@ list-contexts:
 list-config:
 	cat ~/.kube/config
 
-logs: logs-app logs-db logs-mail logs-next-app logs-next-proxy
+logs: logs-app logs-lfmerge logs-db logs-mail logs-next-app logs-next-proxy
 logs-app:
 	kubectl logs deploy/app
+logs-lfmerge:
+	kubectl logs deploy/lfmerge
 logs-next-app:
 	kubectl logs deploy/next-app
 logs-next-proxy:
@@ -24,6 +26,8 @@ logs-mail:
 
 shell-app:
 	kubectl exec -it deploy/app -- sh
+shell-lfmerge:
+	kubectl exec -it deploy/lfmerge -- sh
 shell-next-app:
 	kubectl exec -it deploy/next-app -- sh
 shell-next-proxy:
@@ -35,23 +39,27 @@ init-secrets:
 create-new-deployment-mail:
 	kubectl create deployment mail --image=juanluisbaptiste/postfix:1.0.0 --dry-run=client -o yaml > mail-deployment-new.yaml
 
-deploy-staging: deploy-db deploy-mail-staging deploy-app-staging deploy-next-proxy-staging deploy-next-app-staging
+deploy-staging: deploy-db deploy-mail-staging deploy-app-staging deploy-lfmerge-staging deploy-next-proxy-staging deploy-next-app-staging
 deploy-mail-staging:
 	sed -e s/{{SERVER_HOSTNAME}}/qa.languageforge.org/ mail-deployment.yaml | kubectl apply -f -
 deploy-app-staging:
 	sed -e s/{{WEBSITE}}/qa.languageforge.org/ app-deployment.yaml \
   | sed -e s/{{VERSION}}/$(VERSION_APP)/ | kubectl apply -f -
+deploy-lfmerge-staging:
+	sed -e s/{{VERSION_LFMERGE}}/$(VERSION_LFMERGE)/ lfmerge-deployment.yaml | kubectl apply -f -
 deploy-next-proxy-staging:
 	sed -e s/{{WEBSITE}}/qa.languageforge.org/ next-proxy-deployment.yaml \
   | sed -e s/{{VERSION}}/$(VERSION_PROXY)/ | kubectl apply -f -
 deploy-next-app-staging:
 	sed -e s/{{VERSION}}/$(VERSION_NEXT_APP)/ next-app-deployment.yaml | kubectl apply -f -
-deploy-prod: deploy-db deploy-mail-prod deploy-app-prod deploy-next-proxy-prod deploy-next-app-prod
+deploy-prod: deploy-db deploy-mail-prod deploy-app-prod deploy-lfmerge-prod deploy-next-proxy-prod deploy-next-app-prod
 deploy-mail-prod:
 	sed -e s/{{SERVER_HOSTNAME}}/languageforge.org/ mail-deployment.yaml | kubectl apply -f -
 deploy-app-prod:
 	sed -e s/{{WEBSITE}}/languageforge.org/ app-deployment.yaml \
   | sed -e s/{{VERSION}}/$(VERSION_APP)/ | kubectl apply -f -
+deploy-lfmerge-prod:
+	sed -e s/{{VERSION_LFMERGE}}/$(VERSION_LFMERGE)/ lfmerge-deployment.yaml | kubectl apply -f -
 deploy-next-proxy-prod:
 	sed -e s/{{WEBSITE}}/languageforge.org/ next-proxy-deployment.yaml \
   | sed -e s/{{VERSION}}/$(VERSION_PROXY)/ | kubectl apply -f -
@@ -60,7 +68,7 @@ deploy-next-app-prod:
 deploy-db:
 	kubectl apply -f db-deployment.yaml
 
-delete: delete-app delete-mail delete-db delete-next-proxy delete-next-app
+delete: delete-app delete-lfmerge delete-mail delete-db delete-next-proxy delete-next-app
 delete-db: # does NOT delete the volume, i.e., the data in the database
 	kubectl delete deployment,service db
 delete-db-data:
@@ -73,6 +81,8 @@ delete-app-assets:
 	kubectl delete pvc lf-project-assets
 delete-app-sendreceive-data:
 	kubectl delete pvc lfmerge-sendreceive-data
+delete-lfmerge: # does NOT delete the volumes, e.g., the send/receive webwork folder
+	kubectl delete deployment,service lfmerge
 delete-next-proxy:
 	kubectl delete deployment,service next-proxy
 	kubectl delete ingress languageforge-app

docker/deployment/app-deployment.yaml

@@ -146,7 +146,7 @@ spec:
               secretKeyRef:
                 key: GOOGLE_CLIENT_SECRET
                 name: oauth
-          - name: LD_TRUST_TOKEN
+          - name: LANGUAGE_DEPOT_TRUST_TOKEN
             valueFrom:
               secretKeyRef:
                 key: LD_TRUST_TOKEN

docker/deployment/lfmerge-deployment.yaml

@@ -0,0 +1,126 @@
+# https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: lfmerge
+  name: lfmerge
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+  - name: http
+    protocol: TCP
+    port: 80
+    targetPort: 80
+  selector:
+    app: lfmerge
+
+---
+
+# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: lf-project-assets
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
+
+---
+
+# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: lfmerge-sendreceive-data
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 50Gi
+  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
+
+---
+
+# https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#writing-a-deployment-spec
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: lfmerge
+  name: lfmerge
+spec:
+  selector:
+    matchLabels:
+      app: lfmerge
+  template:
+    # https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates
+    metadata:
+      labels:
+        app: lfmerge
+    spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - app
+            topologyKey: kubernetes.io/hostname
+      volumes:
+      - name: sendreceive-data
+        persistentVolumeClaim:
+          claimName: lfmerge-sendreceive-data
+      initContainers:
+      - name: initialize-volume-ownership
+        image: busybox:1.33
+        command:
+          - 'sh'
+          - '-c'
+          - |-
+            mkdir -m 02775 -p /var/lib/languageforge/lexicon/sendreceive/state /var/lib/languageforge/lexicon/sendreceive/syncqueue /var/lib/languageforge/lexicon/sendreceive/webwork /var/lib/languageforge/lexicon/sendreceive/Templates
+            chown www-data:www-data /var/lib/languageforge/lexicon/sendreceive /var/lib/languageforge/lexicon/sendreceive/state /var/lib/languageforge/lexicon/sendreceive/syncqueue /var/lib/languageforge/lexicon/sendreceive/webwork /var/lib/languageforge/lexicon/sendreceive/Templates
+        volumeMounts:
+          - mountPath: /var/lib/languageforge/lexicon/sendreceive
+            name: sendreceive-data
+      containers:
+      - name: lfmerge
+        image: ghcr.io/sillsdev/lfmerge:{{VERSION_LFMERGE}}
+        imagePullPolicy: Always
+        volumeMounts:
+          - mountPath: /var/lib/languageforge/lexicon/sendreceive
+            name: sendreceive-data
+        env:
+          - name: ENVIRONMENT
+            value: production
+          - name: LFMERGE_LOGGING_DEST
+            value: syslog
+          - name: LFMERGE_BASE_DIR
+            value: /var/lib/languageforge/lexicon/sendreceive
+          - name: LFMERGE_WEBWORK_DIR
+            value: webwork
+          - name: LFMERGE_TEMPLATES_DIR
+            value: Templates
+          - name: LFMERGE_MONGO_HOSTNAME
+            value: db
+          - name: LFMERGE_MONGO_PORT
+            value: "27017"
+          - name: LFMERGE_MONGO_MAIN_DB_NAME
+            value: scriptureforge
+          - name: LFMERGE_MONGO_DB_NAME_PREFIX
+            value: sf_
+          - name: LFMERGE_VERBOSE_PROGRESS
+            value: "true"
+          - name: LANGUAGE_DEPOT_TRUST_TOKEN
+            valueFrom:
+              secretKeyRef:
+                key: LD_TRUST_TOKEN
+                name: ld-trust-token