feat: apple-silicon support

Makefile

@@ -8,8 +8,8 @@
 #     make dev.attach.credentials
 #     make dev.pull.registrar+studio
 #     make dev.up.lms
-#     make dev.up.without-deps.lms+forum+discovery+mysql57+elasticsearch+memcached
-#     make dev.restart-container.mysql57+lms
+#     make dev.up.without-deps.lms+forum+discovery+mysql80+elasticsearch+memcached
+#     make dev.restart-container.mysql80+lms
 
 # There are also "prefix-form" targets, which are simply an alternate way to spell
 # the 'dev.' targets.
@@ -230,17 +230,17 @@ impl-dev.provision.%: dev.check-memory ## Provision specified services.
 dev.provision.%: ## Provision specified services.
 	@scripts/send_metrics.py wrap "dev.provision.$*"
 
-dev.backup: dev.up.mysql57+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Write all data volumes to the host.
-	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql57) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql57.tar.gz /var/lib/mysql
+dev.backup: dev.up.mysql80+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Write all data volumes to the host.
+	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql80) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql80.tar.gz /var/lib/mysql
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch7) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch7.tar.gz /usr/share/elasticsearch/data
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch710) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch710.tar.gz /usr/share/elasticsearch/data
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.opensearch12) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/opensearch12.tar.gz /usr/share/opensearch/data
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.coursegraph) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/coursegraph.tar.gz /data
 
-dev.restore: dev.up.mysql57+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Restore all data volumes from the host. WILL OVERWRITE ALL EXISTING DATA!
-	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql57) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql57.tar.gz
+dev.restore: dev.up.mysql80+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Restore all data volumes from the host. WILL OVERWRITE ALL EXISTING DATA!
+	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql80) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql80.tar.gz
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch7) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch7.tar.gz
@@ -270,7 +270,7 @@ dev.migrate.%: ## Run migrations on a service.
 dev.drop-db: _expects-database.dev.drop-db
 
 dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mysql container.
-	docker-compose exec -T mysql57 bash -c "mysql --execute=\"DROP DATABASE $*;\""
+	docker-compose exec -T mysql80 bash -c "mysql --execute=\"DROP DATABASE $*;\""
 
 
 ########################################################################################
@@ -457,10 +457,10 @@ dev.shell.%: ## Run a shell on the specified service's container.
 	docker-compose exec $* /bin/bash
 
 dev.dbshell:
-	docker-compose exec mysql57 bash -c "mysql"
+	docker-compose exec mysql80 bash -c "mysql"
 
 dev.dbshell.%: ## Run a SQL shell on the given database.
-	docker-compose exec mysql57 bash -c "mysql $*"
+	docker-compose exec mysql80 bash -c "mysql $*"
 
 # List of Makefile targets to run static asset generation, in the form dev.static.$(service)
 # Services will only have their asset generation added here

README.rst

@@ -65,7 +65,7 @@ You can also browse all the documentation in `Read the Docs`_.
 Notices
 -------
 
-**NOTE:** LMS is now using MySql 5.7 by default. You have to run ``make dev.pull.lms`` and ``make dev.provision.lms`` (more details in `Getting Started`_) to fetch latest images and reprovision local copies of databases in order for an existing devstack setup to keep working.
+**NOTE:** LMS is now using MySql 8.0 by default. You have to run ``make dev.pull.lms`` and ``make dev.provision.lms`` (more details in `Getting Started`_) to fetch latest images and reprovision local copies of databases in order for an existing devstack setup to keep working.
 
 
 Getting Started

docker-compose.yml

@@ -237,22 +237,23 @@ services:
     volumes:
       - mongo_data:/data/db
 
-  mysql57:
+  mysql80:
     command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci
-    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql57"
-    hostname: mysql57.devstack.edx
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql80"
+    hostname: mysql80.devstack.edx
     environment:
       MYSQL_ROOT_PASSWORD: ""
       MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
-    image: mysql:5.7
+    image: mysql:8.0.28-oracle
     networks:
       default:
         aliases:
+          - edx.devstack.mysql80
           - edx.devstack.mysql57
     ports:
       - "3506:3306"
     volumes:
-      - mysql57_data:/var/lib/mysql
+      - mysql80_data:/var/lib/mysql
 
   redis:
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.redis"
@@ -309,13 +310,13 @@ services:
     depends_on:
       - lms
       - memcached
-      - mysql57
+      - mysql80
     # Allows attachment to the credentials service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
     environment:
       CACHE_LOCATION: edx.devstack.memcached:11211
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       SOCIAL_AUTH_EDX_OIDC_URL_ROOT: http://edx.devstack.lms:18000/oauth2
       ENABLE_DJANGO_TOOLBAR: 1
       DJANGO_WATCHMAN_TIMEOUT: 30
@@ -334,14 +335,14 @@ services:
     depends_on:
       - elasticsearch710
       - memcached
-      - mysql57
+      - mysql80
     # Allows attachment to the discovery service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
     environment:
       # This next DB_MIGRATION_HOST line can be removed once edx/configuration has been updated with this value for
       # a while and most people have had a chance to do a "make pull" to get the latest images.
-      DB_MIGRATION_HOST: edx.devstack.mysql57
+      DB_MIGRATION_HOST: edx.devstack.mysql80
       TEST_ELASTICSEARCH_URL: "edx.devstack.elasticsearch710"
       ENABLE_DJANGO_TOOLBAR: 1
       DJANGO_WATCHMAN_TIMEOUT: 30
@@ -363,7 +364,7 @@ services:
       - discovery
       - lms
       - memcached
-      - mysql57
+      - mysql80
     # Allows attachment to the ecommerce service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
@@ -387,7 +388,7 @@ services:
       - devpi
       - elasticsearch710
       - lms
-      - mysql57
+      - mysql80
     image: edxops/notes:${OPENEDX_RELEASE:-latest}
     networks:
       default:
@@ -397,7 +398,7 @@ services:
       - "18120:18120"
     environment:
       DB_ENGINE: "django.db.backends.mysql"
-      DB_HOST: "edx.devstack.mysql57"
+      DB_HOST: "edx.devstack.mysql80"
       DB_NAME: "notes"
       DB_PASSWORD: "password"
       DB_PORT: "3306"
@@ -437,7 +438,7 @@ services:
       - forum
       - memcached
       - mongo
-      - mysql57
+      - mysql80
       - opensearch12
     # Allows attachment to the LMS service using 'docker attach <containerID>'.
     stdin_open: true
@@ -469,14 +470,14 @@ services:
     hostname: insights.devstack.edx
     depends_on:
       - analyticsapi
-      - mysql57
+      - mysql80
       - lms
       - memcached
     # Allows attachment to the insights service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_NAME: dashboard
       DB_PORT: 3306
       DB_USER: analytics001
@@ -498,13 +499,13 @@ services:
     container_name: edx.devstack.analyticsapi
     hostname: analyticsapi
     depends_on:
-      - mysql57
+      - mysql80
       - elasticsearch710
     command: bash -c 'source /edx/app/analytics_api/analytics_api_env && while true;  do python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:19001 --settings analyticsdataserver.settings.devstack; sleep 2; done'
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_PORT: 3306
       DB_USER: analytics001
       DB_PASSWORD: password
@@ -521,15 +522,15 @@ services:
     depends_on:
       - discovery
       - lms
-      - mysql57
+      - mysql80
       - memcached
       - redis
       - registrar-worker
     # Allows attachment to the registrar service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_NAME: registrar
       DB_PORT: 3306
       DB_USER: registrar001
@@ -559,12 +560,12 @@ services:
     hostname: registrar-worker.devstack.edx
     depends_on:
       - lms
-      - mysql57
+      - mysql80
       - redis
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_NAME: registrar
       DB_PORT: 3306
       DB_USER: registrar001
@@ -597,7 +598,7 @@ services:
       - lms
       - memcached
       - mongo
-      - mysql57
+      - mysql80
       - opensearch12
     # Allows attachment to the Studio service using 'docker attach <containerID>'.
     stdin_open: true
@@ -630,7 +631,7 @@ services:
     volumes:
       - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue
     depends_on:
-      - mysql57
+      - mysql80
     networks:
       default:
         aliases:
@@ -645,7 +646,7 @@ services:
     volumes:
       - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue
     depends_on:
-      - mysql57
+      - mysql80
     networks:
       default:
         aliases:
@@ -837,4 +838,4 @@ volumes:
   elasticsearch710_data:
   mongo_data:
   opensearch12_data:
-  mysql57_data:
+  mysql80_data:

docs/developing_on_named_release_branches.rst

@@ -47,7 +47,7 @@ Switch between your Devstack releases by doing the following:
 Examples of Docker Service Names After Setting the ``COMPOSE_PROJECT_NAME`` variable. Notice that the **devstack-juniper.master** name represents the ``COMPOSE_PROJECT_NAME``.
 
 -  edx.devstack-juniper.master.lms
--  edx.devstack-juniper.master.mysql57
+-  edx.devstack-juniper.master.mysql80
 
 Each instance has an isolated set of databases. This could, for example, be used to quickly switch between versions of Open edX without hitting as many issues with migrations, data integrity, etc.
 

docs/devstack_faq.rst

@@ -134,7 +134,7 @@ How do I connect to the databases from an outside editor?
 ---------------------------------------------------------
 
 To connect to the databases from an outside editor (such as MySQLWorkbench),
-first uncomment these lines from ``docker-compose.yml``'s ``mysql57`` section
+first uncomment these lines from ``docker-compose.yml``'s ``mysql80`` section
 
 .. code:: yaml
 
@@ -145,8 +145,8 @@ Then bring your mysql container down and back up by running:
 
 .. code:: sh
 
-  docker-compose stop mysql57
-  docker-compose up -d mysql57
+  docker-compose stop mysql80
+  docker-compose up -d mysql80
 
 Then connect using the values below. Note that the username and password will
 vary depending on the database. For all of the options, see ``provision.sql``.
@@ -158,7 +158,7 @@ vary depending on the database. For all of the options, see ``provision.sql``.
 
 If you have trouble connecting, ensure the port was mapped successfully by
 running ``make dev.ps`` and looking for a line like this:
-``edx.devstack.mysql57 docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``.
+``edx.devstack.mysql80 docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``.
 
 How do I build the service images myself?
 -----------------------------------------
@@ -209,7 +209,7 @@ To access the MySQL shell for a particular database, run:
 
 .. code:: sh
 
-   make dev.shell.mysql57
+   make dev.shell.mysql80
    mysql
    use <database>;
 

load-db.sh

@@ -16,6 +16,6 @@ then
 fi
 
 echo "Loading the $1 database..."
-mysql_container=$(make --silent --no-print-directory dev.print-container.mysql57)
+mysql_container=$(make --silent --no-print-directory dev.print-container.mysql80)
 docker exec -i "$mysql_container" mysql -uroot $1 < $1.sql
 echo "Finished loading the $1 database!"

options.mk

@@ -90,4 +90,4 @@ credentials+discovery+ecommerce+insights+lms+registrar+studio
 # All third-party services.
 # Separated by plus signs. Listed in alphabetical order for clarity.
 THIRD_PARTY_SERVICES ?= \
-chrome+coursegraph+devpi+elasticsearch+elasticsearch7+elasticsearch710+firefox+memcached+mongo+mysql57+opensearch12+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica
+chrome+coursegraph+devpi+elasticsearch+elasticsearch7+elasticsearch710+firefox+memcached+mongo+mysql80+opensearch12+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica

provision.sh

@@ -123,14 +123,14 @@ fi
 echo -e "${GREEN}Will provision the following:\n  ${to_provision_ordered}${NC}"
 
 # Bring the databases online.
-docker-compose up -d mysql57
+docker-compose up -d mysql80
 if needs_mongo "$to_provision_ordered"; then
 	docker-compose up -d mongo
 fi
 
 # Ensure the MySQL server is online and usable
-echo "${GREEN}Waiting for MySQL 5.7.${NC}"
-until docker-compose exec -T mysql57 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null
+echo "${GREEN}Waiting for MySQL 8.0.${NC}"
+until docker-compose exec -T mysql80 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null
 do
   printf "."
   sleep 1
@@ -140,8 +140,8 @@ done
 # See https://github.com/docker-library/mysql/issues/245 for why this is necessary.
 sleep 10
 
-echo "${GREEN}Waiting for MySQL 5.7 to restart.${NC}"
-until docker-compose exec -T mysql57 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null
+echo "${GREEN}Waiting for MySQL 8.0 to restart.${NC}"
+until docker-compose exec -T mysql80 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null
 do
   printf "."
   sleep 1
@@ -152,8 +152,8 @@ echo -e "${GREEN}MySQL ready.${NC}"
 
 # Ensure that the MySQL databases and users are created for all IDAs.
 # (A no-op for databases and users that already exist).
-echo -e "${GREEN}Ensuring MySQL 5.7 databases and users exist...${NC}"
-docker-compose exec -T mysql57 bash -e -c "mysql -uroot mysql" < provision.sql
+echo -e "${GREEN}Ensuring MySQL 8.0 databases and users exist...${NC}"
+docker-compose exec -T mysql80 bash -e -c "mysql -uroot mysql" < provision.sql
 
 # If necessary, ensure the MongoDB server is online and usable
 # and create its users.