From c75ff03efb6527d6c6ce06839867a06b6fa9387d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 5 Nov 2021 08:30:59 +1100 Subject: [PATCH] MDEV-26961: add 10.8 branch --- .github/workflows/ci.yml | 2 +- 10.8/Dockerfile | 138 ++++++++++++ 10.8/docker-entrypoint.sh | 400 ++++++++++++++++++++++++++++++++++ generate-stackbrew-library.sh | 1 + update.sh | 3 + 5 files changed, 543 insertions(+), 1 deletion(-) create mode 100644 10.8/Dockerfile create mode 100755 10.8/docker-entrypoint.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43b75ce..f2657cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: name: Generate Jobs run: | git clone --depth 1 https://github.com/docker-library/bashbrew.git -b master ~/bashbrew - strategy="$(~/bashbrew/scripts/github-actions/generate.sh | jq -c '.matrix.include |= map(.meta.entries[0].tags[0] as $tag | .runs.mariadbtest = "./.test/run.sh " + $tag)')" + strategy="$(~/bashbrew/scripts/github-actions/generate.sh | jq -c '.matrix.include |= map(.meta.entries[0].tags[0] as $tag | select(.name != "10.8-focal") | .runs.mariadbtest = "./.test/run.sh " + $tag)')" jq . <<<"$strategy" # sanity check / debugging aid echo "::set-output name=strategy::$strategy" diff --git a/10.8/Dockerfile b/10.8/Dockerfile new file mode 100644 index 0000000..3e9167c --- /dev/null +++ b/10.8/Dockerfile @@ -0,0 +1,138 @@ +# vim:set ft=dockerfile: +FROM ubuntu:focal + +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN groupadd -r mysql && useradd -r -g mysql mysql + +# https://bugs.debian.org/830696 (apt uses gpgv by default in newer releases, rather than gpg) +RUN set -ex; \ + apt-get update; \ + if ! which gpg; then \ + apt-get install -y --no-install-recommends gnupg; \ + fi; \ + if ! gpg --version | grep -q '^gpg (GnuPG) 1\.'; then \ +# Ubuntu includes "gnupg" (not "gnupg2", but still 2.x), but not dirmngr, and gnupg 2.x requires dirmngr +# so, if we're not running gnupg 1.x, explicitly install dirmngr too + apt-get install -y --no-install-recommends dirmngr; \ + fi; \ + rm -rf /var/lib/apt/lists/* + +# add gosu for easy step-down from root +# https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.14 +RUN set -eux; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get install -y --no-install-recommends wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true + +RUN mkdir /docker-entrypoint-initdb.d + +# install "libjemalloc2" as it offers better performance in some cases. Use with LD_PRELOAD +# install "pwgen" for randomizing passwords +# install "tzdata" for /usr/share/zoneinfo/ +# install "xz-utils" for .sql.xz docker-entrypoint-initdb.d files +# install "zstd" for .sql.zst docker-entrypoint-initdb.d files +RUN set -ex; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libjemalloc2 \ + pwgen \ + tzdata \ + xz-utils \ + zstd \ + ; \ + rm -rf /var/lib/apt/lists/* + +ARG GPG_KEYS=177F4010FE56CA3336300305F1656F24C74CD1D8 +# pub rsa4096 2016-03-30 [SC] +# 177F 4010 FE56 CA33 3630 0305 F165 6F24 C74C D1D8 +# uid [ unknown] MariaDB Signing Key +# sub rsa4096 2016-03-30 [E] + +RUN set -ex; \ + export GNUPGHOME="$(mktemp -d)"; \ + for key in $GPG_KEYS; do \ + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key"; \ + done; \ + gpg --batch --export $GPG_KEYS > /etc/apt/trusted.gpg.d/mariadb.gpg; \ + command -v gpgconf > /dev/null && gpgconf --kill all || :; \ + rm -fr "$GNUPGHOME"; \ + apt-key list + +# bashbrew-architectures: amd64 arm64v8 ppc64le s390x +ARG MARIADB_MAJOR=10.8 +ENV MARIADB_MAJOR $MARIADB_MAJOR +ARG MARIADB_VERSION=1:10.8.0+maria~focal +ENV MARIADB_VERSION $MARIADB_VERSION +# release-status:Alpha +# (https://downloads.mariadb.org/mariadb/+releases/) + +# Allowing overriding of REPOSITORY, a URL that includes suite and component for testing and Enterprise Versions +ARG REPOSITORY="http://archive.mariadb.org/mariadb-10.8.0/repo/ubuntu/ focal main" + +RUN set -e;\ + echo "deb ${REPOSITORY}" > /etc/apt/sources.list.d/mariadb.list; \ + { \ + echo 'Package: *'; \ + echo 'Pin: release o=MariaDB'; \ + echo 'Pin-Priority: 999'; \ + } > /etc/apt/preferences.d/mariadb +# add repository pinning to make sure dependencies from this MariaDB repo are preferred over Debian dependencies +# libmariadbclient18 : Depends: libmysqlclient18 (= 5.5.42+maria-1~wheezy) but 5.5.43-0+deb7u1 is to be installed + +# the "/var/lib/mysql" stuff here is because the mysql-server postinst doesn't have an explicit way to disable the mysql_install_db codepath besides having a database already "configured" (ie, stuff in /var/lib/mysql/mysql) +# also, we set debconf keys to make APT a little quieter +RUN set -ex; \ + { \ + echo "mariadb-server-$MARIADB_MAJOR" mysql-server/root_password password 'unused'; \ + echo "mariadb-server-$MARIADB_MAJOR" mysql-server/root_password_again password 'unused'; \ + } | debconf-set-selections; \ + apt-get update; \ + apt-get install -y \ + "mariadb-server=$MARIADB_VERSION" \ +# mariadb-backup is installed at the same time so that `mysql-common` is only installed once from just mariadb repos + mariadb-backup \ + socat \ + ; \ + rm -rf /var/lib/apt/lists/*; \ +# purge and re-create /var/lib/mysql with appropriate ownership + rm -rf /var/lib/mysql; \ + mkdir -p /var/lib/mysql /var/run/mysqld; \ + chown -R mysql:mysql /var/lib/mysql /var/run/mysqld; \ +# ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime + chmod 777 /var/run/mysqld; \ +# comment out a few problematic configuration values + find /etc/mysql/ -name '*.cnf' -print0 \ + | xargs -0 grep -lZE '^(bind-address|log|user\s)' \ + | xargs -rt -0 sed -Ei 's/^(bind-address|log|user\s)/#&/'; \ +# don't reverse lookup hostnames, they are usually another container +# Issue #327 Correct order of reading directories /etc/mysql/mariadb.conf.d before /etc/mysql/conf.d (mount-point per documentation) + if [ ! -L /etc/mysql/my.cnf ]; then sed -i -e '/includedir/i[mariadb]\nskip-host-cache\nskip-name-resolve\n' /etc/mysql/my.cnf; \ +# 10.5+ + else sed -i -e '/includedir/ {N;s/\(.*\)\n\(.*\)/[mariadbd]\nskip-host-cache\nskip-name-resolve\n\n\2\n\1/}' \ + /etc/mysql/mariadb.cnf; fi + + +VOLUME /var/lib/mysql + +COPY docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] + +EXPOSE 3306 +CMD ["mariadbd"] diff --git a/10.8/docker-entrypoint.sh b/10.8/docker-entrypoint.sh new file mode 100755 index 0000000..8774deb --- /dev/null +++ b/10.8/docker-entrypoint.sh @@ -0,0 +1,400 @@ +#!/bin/bash +set -eo pipefail +shopt -s nullglob + +# logging functions +mysql_log() { + local type="$1"; shift + printf '%s [%s] [Entrypoint]: %s\n' "$(date --rfc-3339=seconds)" "$type" "$*" +} +mysql_note() { + mysql_log Note "$@" +} +mysql_warn() { + mysql_log Warn "$@" >&2 +} +mysql_error() { + mysql_log ERROR "$@" >&2 + exit 1 +} + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + mysql_error "Both $var and $fileVar are set (but are exclusive)" + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# set MARIADB_xyz from MYSQL_xyz when MARIADB_xyz is unset +# and make them the same value (so user scripts can use either) +_mariadb_file_env() { + local var="$1"; shift + local maria="MARIADB_${var#MYSQL_}" + file_env "$var" "$@" + file_env "$maria" "${!var}" + if [ "${!maria:-}" ]; then + export "$var"="${!maria}" + fi +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions +docker_process_init_files() { + # mysql here for backwards compatibility "${mysql[@]}" + mysql=( docker_process_sql ) + + echo + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + mysql_note "$0: running $f" + "$f" + else + mysql_note "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) mysql_note "$0: running $f"; docker_process_sql < "$f"; echo ;; + *.sql.gz) mysql_note "$0: running $f"; gunzip -c "$f" | docker_process_sql; echo ;; + *.sql.xz) mysql_note "$0: running $f"; xzcat "$f" | docker_process_sql; echo ;; + *.sql.zst) mysql_note "$0: running $f"; zstd -dc "$f" | docker_process_sql; echo ;; + *) mysql_warn "$0: ignoring $f" ;; + esac + echo + done +} + +# arguments necessary to run "mysqld --verbose --help" successfully (used for testing configuration validity and for extracting default/configured values) +_verboseHelpArgs=( + --verbose --help + --log-bin-index="$(mktemp -u)" # https://github.com/docker-library/mysql/issues/136 +) + +mysql_check_config() { + local toRun=( "$@" "${_verboseHelpArgs[@]}" ) errors + if ! errors="$("${toRun[@]}" 2>&1 >/dev/null)"; then + mysql_error $'mysqld failed while attempting to check config\n\tcommand was: '"${toRun[*]}"$'\n\t'"$errors" + fi +} + +# Fetch value from server config +# We use mysqld --verbose --help instead of my_print_defaults because the +# latter only show values present in config files, and not server defaults +mysql_get_config() { + local conf="$1"; shift + "$@" "${_verboseHelpArgs[@]}" 2>/dev/null \ + | awk -v conf="$conf" '$1 == conf && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }' + # match "datadir /some/path with/spaces in/it here" but not "--xyz=abc\n datadir (xyz)" +} + +# Do a temporary startup of the MariaDB server, for init purposes +docker_temp_server_start() { + "$@" --skip-networking --default-time-zone=SYSTEM --socket="${SOCKET}" --wsrep_on=OFF & + mysql_note "Waiting for server startup" + # only use the root password if the database has already been initializaed + # so that it won't try to fill in a password file when it hasn't been set yet + extraArgs=() + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + extraArgs+=( '--dont-use-mysql-root-password' ) + fi + local i + for i in {30..0}; do + if docker_process_sql "${extraArgs[@]}" --database=mysql <<<'SELECT 1' &> /dev/null; then + break + fi + sleep 1 + done + if [ "$i" = 0 ]; then + mysql_error "Unable to start server." + fi +} + +# Stop the server. When using a local socket file mysqladmin will block until +# the shutdown is complete. +docker_temp_server_stop() { + if ! MYSQL_PWD=$MARIADB_ROOT_PASSWORD mysqladmin shutdown -uroot --socket="${SOCKET}"; then + mysql_error "Unable to shut down server." + fi +} + +# Verify that the minimally required password settings are set for new databases. +docker_verify_minimum_env() { + if [ -z "$MARIADB_ROOT_PASSWORD" -a -z "$MARIADB_ALLOW_EMPTY_ROOT_PASSWORD" -a -z "$MARIADB_RANDOM_ROOT_PASSWORD" ]; then + mysql_error $'Database is uninitialized and password option is not specified\n\tYou need to specify one of MARIADB_ROOT_PASSWORD, MARIADB_ALLOW_EMPTY_ROOT_PASSWORD and MARIADB_RANDOM_ROOT_PASSWORD' + fi +} + +# creates folders for the database +# also ensures permission for user mysql of run as root +docker_create_db_directories() { + local user; user="$(id -u)" + + # TODO other directories that are used by default? like /var/lib/mysql-files + # see https://github.com/docker-library/mysql/issues/562 + mkdir -p "$DATADIR" + + if [ "$user" = "0" ]; then + # this will cause less disk access than `chown -R` + find "$DATADIR" \! -user mysql -exec chown mysql '{}' + + # See https://github.com/MariaDB/mariadb-docker/issues/363 + find "${SOCKET%/*}" -maxdepth 0 \! -user mysql -exec chown mysql '{}' \; + fi +} + +# initializes the database directory +docker_init_database_dir() { + mysql_note "Initializing database files" + installArgs=( --datadir="$DATADIR" --rpm --auth-root-authentication-method=normal ) + if { mysql_install_db --help || :; } | grep -q -- '--skip-test-db'; then + # 10.3+ + installArgs+=( --skip-test-db ) + fi + # "Other options are passed to mysqld." (so we pass all "mysqld" arguments directly here) + mysql_install_db "${installArgs[@]}" "${@:2}" --default-time-zone=SYSTEM --enforce-storage-engine= + mysql_note "Database files initialized" +} + +# Loads various settings that are used elsewhere in the script +# This should be called after mysql_check_config, but before any other functions +docker_setup_env() { + # Get config + declare -g DATADIR SOCKET + DATADIR="$(mysql_get_config 'datadir' "$@")" + SOCKET="$(mysql_get_config 'socket' "$@")" + + + # Initialize values that might be stored in a file + _mariadb_file_env 'MYSQL_ROOT_HOST' '%' + _mariadb_file_env 'MYSQL_DATABASE' + _mariadb_file_env 'MYSQL_USER' + _mariadb_file_env 'MYSQL_PASSWORD' + _mariadb_file_env 'MYSQL_ROOT_PASSWORD' + + # set MARIADB_ from MYSQL_ when it is unset and then make them the same value + : "${MARIADB_ALLOW_EMPTY_ROOT_PASSWORD:=${MYSQL_ALLOW_EMPTY_PASSWORD:-}}" + export MYSQL_ALLOW_EMPTY_PASSWORD="$MARIADB_ALLOW_EMPTY_ROOT_PASSWORD" MARIADB_ALLOW_EMPTY_ROOT_PASSWORD + : "${MARIADB_RANDOM_ROOT_PASSWORD:=${MYSQL_RANDOM_ROOT_PASSWORD:-}}" + export MYSQL_RANDOM_ROOT_PASSWORD="$MARIADB_RANDOM_ROOT_PASSWORD" MARIADB_RANDOM_ROOT_PASSWORD + : "${MARIADB_INITDB_SKIP_TZINFO:=${MYSQL_INITDB_SKIP_TZINFO:-}}" + export MYSQL_INITDB_SKIP_TZINFO="$MARIADB_INITDB_SKIP_TZINFO" MARIADB_INITDB_SKIP_TZINFO + + declare -g DATABASE_ALREADY_EXISTS + if [ -d "$DATADIR/mysql" ]; then + DATABASE_ALREADY_EXISTS='true' + fi +} + +# Execute the client, use via docker_process_sql to handle root password +docker_exec_client() { + # args sent in can override this db, since they will be later in the command + if [ -n "$MYSQL_DATABASE" ]; then + set -- --database="$MYSQL_DATABASE" "$@" + fi + mysql --protocol=socket -uroot -hlocalhost --socket="${SOCKET}" "$@" +} + +# Execute sql script, passed via stdin +# usage: docker_process_sql [--dont-use-mysql-root-password] [mysql-cli-args] +# ie: docker_process_sql --database=mydb <<<'INSERT ...' +# ie: docker_process_sql --dont-use-mysql-root-password --database=mydb /dev/null + + docker_init_database_dir "$@" + + mysql_note "Starting temporary server" + docker_temp_server_start "$@" + mysql_note "Temporary server started." + + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + mysql_note "Stopping temporary server" + docker_temp_server_stop + mysql_note "Temporary server stopped" + + echo + mysql_note "MariaDB init process done. Ready for start up." + echo + fi + fi + exec "$@" +} + +# If we are sourced from elsewhere, don't perform any further actions +if ! _is_sourced; then + _main "$@" +fi diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index 81a9b0e..f0ae0a9 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -14,6 +14,7 @@ cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" source '.architectures-lib' +GLOBIGNORE=10.8/:.*/:test/ versions=( */ ) versions=( "${versions[@]%/}" ) diff --git a/update.sh b/update.sh index db2521c..c539fe2 100755 --- a/update.sh +++ b/update.sh @@ -40,6 +40,9 @@ fi versions=( "${versions[@]%/}" ) for version in "${versions[@]}"; do + if [ "$version" == 10.8 ]; then + version=10.8.0 + fi if [ ! -d "$version" ]; then # assume full version and trim this to major version ExpectedFullVersion=$version