From 32e900d0de441eb680e6fb240b57727751630161 Mon Sep 17 00:00:00 2001 From: Matthew Baggett Date: Sun, 28 Aug 2022 11:38:02 +0200 Subject: [PATCH] Working mysql build. --- Dockerfile.mariadb | 54 +++++++++++++ Dockerfile => Dockerfile.postgres | 14 ++-- bake.hcl | 74 ++++++++++++++++-- composer.json | 2 +- composer.lock | 66 ++++++++-------- docker-compose.yml | 29 ++++++- mysql.runit | 6 ++ healthcheck => postgres_healthcheck | 0 sync-pull.runit | 5 +- syncer/MysqlSyncer.php | 77 +++++++++++++++++++ ...sAbstractSyncer.php => PostgresSyncer.php} | 2 +- syncer/Sync.php | 7 +- 12 files changed, 285 insertions(+), 51 deletions(-) create mode 100644 Dockerfile.mariadb rename Dockerfile => Dockerfile.postgres (78%) create mode 100644 mysql.runit rename healthcheck => postgres_healthcheck (100%) create mode 100644 syncer/MysqlSyncer.php rename syncer/{PostgresAbstractSyncer.php => PostgresSyncer.php} (97%) diff --git a/Dockerfile.mariadb b/Dockerfile.mariadb new file mode 100644 index 0000000..b9c41d2 --- /dev/null +++ b/Dockerfile.mariadb @@ -0,0 +1,54 @@ +ARG MARIADB_VERSION=10.9 +FROM mariadb:$MARIADB_VERSION AS mysql +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN echo "APT::Acquire::Retries \"5\";" > /etc/apt/apt.conf.d/80-retries && \ + echo "Acquire::http::No-Cache=true;" > /etc/apt/apt.conf.d/80-no-cache && \ + echo "Acquire::http::Pipeline-Depth=0;" > /etc/apt/apt.conf.d/80-no-pipeline && \ + apt-get -qq update && \ + apt-get -yqq upgrade && \ + apt-get -yqq install --no-install-recommends \ + xz-utils \ + dos2unix \ + runit \ + python3-software-properties \ + software-properties-common \ + && \ + add-apt-repository -y ppa:ondrej/php && \ + apt-get -qq update && \ + apt-get -yqq install --no-install-recommends \ + #php8.1-apcu \ + php8.1-bcmath \ + #php8.1-bz2 \ + php8.1-cli \ + php8.1-curl \ + php8.1-mysql \ + #php8.1-opcache \ + #php8.1-phpdbg \ + php8.1-xml \ + php8.1-zip \ + && \ + apt-get remove -yqq \ + software-properties-common \ + python-apt-common \ + python3-software-properties \ + python3.5 python3.5-minimal libpython3.5-minimal \ + && \ + apt-get autoremove -yqq && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/lib/dpkg/status.old /var/cache/debconf/templates.dat /var/log/dpkg.log /var/log/lastlog /var/log/apt/*.log + +COPY start.sh /usr/local/bin/start.sh +COPY mysql.runit /etc/service/mysql/run +COPY sync-pull.runit /etc/service/sync-pull/run +COPY sync-push.runit /etc/service/sync-push/run +VOLUME /dumps +WORKDIR /sync +COPY vendor /sync/vendor +COPY syncer /sync/syncer +COPY sync /sync/sync +COPY start.sh /sync/start.sh +ENV PATH="/sync:${PATH}" +RUN chmod +x /sync/sync /etc/service/*/run +CMD ["start.sh"] +HEALTHCHECK --start-period=30s CMD /usr/local/bin/healthcheck.sh +RUN ln -s /sync/vendor/bin/wait-for-mysql /usr/local/bin/wait-for-database \ No newline at end of file diff --git a/Dockerfile b/Dockerfile.postgres similarity index 78% rename from Dockerfile rename to Dockerfile.postgres index bf1e0df..f756692 100644 --- a/Dockerfile +++ b/Dockerfile.postgres @@ -1,7 +1,7 @@ ARG PGSQL_VERSION FROM postgres:$PGSQL_VERSION-alpine AS postgres -RUN apk add --no-cache runit -RUN apk --no-cache --repository https://dl-cdn.alpinelinux.org/alpine/edge/main add \ +RUN apk add --no-cache runit && \ + apk --no-cache --repository https://dl-cdn.alpinelinux.org/alpine/edge/main add \ icu-libs \ &&\ apk --no-cache --repository https://dl-cdn.alpinelinux.org/alpine/edge/community add \ @@ -49,8 +49,12 @@ COPY sync-pull.runit /etc/service/sync-pull/run COPY sync-push.runit /etc/service/sync-push/run VOLUME /dumps WORKDIR /sync -COPY . /sync +COPY vendor /sync/vendor +COPY syncer /sync/syncer +COPY sync /sync/sync +COPY start.sh /sync/start.sh ENV PATH="/sync:${PATH}" -RUN chmod +x /sync/sync /etc/service/*/run /sync/healthcheck CMD ["start.sh"] -HEALTHCHECK --start-period=30s CMD /sync/healthcheck \ No newline at end of file +COPY postgres_healthcheck /sync/postgres_healthcheck +HEALTHCHECK --start-period=30s CMD /sync/postgres_healthcheck +RUN ln -s /sync/vendor/bin/wait-for-postgresql /usr/local/bin/wait-for-database diff --git a/bake.hcl b/bake.hcl index e9ac6c0..576c4af 100644 --- a/bake.hcl +++ b/bake.hcl @@ -10,7 +10,7 @@ group "default" { target "postgres-14" { context = "." - dockerfile = "Dockerfile" + dockerfile = "Dockerfile.postgres" platforms = ["arm64","amd64"] tags = ["benzine/postgres:14", "benzine/postgres:latest"] args = { @@ -20,7 +20,7 @@ target "postgres-14" { target "postgres-13" { context = "." - dockerfile = "Dockerfile" + dockerfile = "Dockerfile.postgres" platforms = ["arm64","amd64"] tags = ["benzine/postgres:13"] args = { @@ -30,7 +30,7 @@ target "postgres-13" { target "postgres-12" { context = "." - dockerfile = "Dockerfile" + dockerfile = "Dockerfile.postgres" platforms = ["arm64","amd64"] tags = ["benzine/postgres:12"] args = { @@ -40,7 +40,7 @@ target "postgres-12" { target "postgres-11" { context = "." - dockerfile = "Dockerfile" + dockerfile = "Dockerfile.postgres" platforms = ["arm64","amd64"] tags = ["benzine/postgres:11"] args = { @@ -50,10 +50,74 @@ target "postgres-11" { target "postgres-10" { context = "." - dockerfile = "Dockerfile" + dockerfile = "Dockerfile.postgres" platforms = ["arm64","amd64"] tags = ["benzine/postgres:10"] args = { PGSQL_VERSION = 10 } } + +target "mariadb-10.9" { + context = "." + dockerfile = "Dockerfile.mariadb" + platforms = ["arm64", "amd64"] + tags = ["benzine/mariadb:10.9", "benzine/mariadb:latest"] + args = { + MARIADB_VERSION=10.9 + } +} +target "mariadb-10.8" { + context = "." + dockerfile = "Dockerfile.mariadb" + platforms = ["arm64", "amd64"] + tags = ["benzine/mariadb:10.8"] + args = { + MARIADB_VERSION=10.8 + } +} +target "mariadb-10.7" { + context = "." + dockerfile = "Dockerfile.mariadb" + platforms = ["arm64", "amd64"] + tags = ["benzine/mariadb:10.7"] + args = { + MARIADB_VERSION=10.7 + } +} +target "mariadb-10.6" { + context = "." + dockerfile = "Dockerfile.mariadb" + platforms = ["arm64", "amd64"] + tags = ["benzine/mariadb:10.6"] + args = { + MARIADB_VERSION=10.6 + } +} +target "mariadb-10.5" { + context = "." + dockerfile = "Dockerfile.mariadb" + platforms = ["arm64", "amd64"] + tags = ["benzine/mariadb:10.5"] + args = { + MARIADB_VERSION=10.5 + } +} +target "mariadb-10.4" { + context = "." + dockerfile = "Dockerfile.mariadb" + platforms = ["arm64", "amd64"] + tags = ["benzine/mariadb:10.4"] + args = { + MARIADB_VERSION=10.4 + } +} +target "mariadb-10.3" { + context = "." + dockerfile = "Dockerfile.mariadb" + platforms = ["arm64", "amd64"] + tags = ["benzine/mariadb:10.3"] + args = { + MARIADB_VERSION=10.3 + } +} \ No newline at end of file diff --git a/composer.json b/composer.json index f22d221..5dd40b0 100644 --- a/composer.json +++ b/composer.json @@ -15,7 +15,7 @@ "spatie/emoji": "^2.3", "rych/bytesize": "^1.0", "jimmiw/php-time-ago": "^3.2", - "matthewbaggett/wait-for-mysql": "^1.0" + "matthewbaggett/wait-for-mysql": "dev-main" }, "require-dev": { "friendsofphp/php-cs-fixer": "^3.0" diff --git a/composer.lock b/composer.lock index 0ed4741..c9d9a7d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9e33df3b624d4c411ab342f127b432cd", + "content-hash": "519bdadff1be790c998c5adf50f9384c", "packages": [ { "name": "adambrett/shell-wrapper", @@ -108,16 +108,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.234.0", + "version": "3.234.4", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "d2113f1e5ec9f7f19de2472f5063333b39a55280" + "reference": "8d56ddb99632200273bb933cbf82b758ab9cde2a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/d2113f1e5ec9f7f19de2472f5063333b39a55280", - "reference": "d2113f1e5ec9f7f19de2472f5063333b39a55280", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/8d56ddb99632200273bb933cbf82b758ab9cde2a", + "reference": "8d56ddb99632200273bb933cbf82b758ab9cde2a", "shasum": "" }, "require": { @@ -194,9 +194,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.234.0" + "source": "https://github.com/aws/aws-sdk-php/tree/3.234.4" }, - "time": "2022-08-22T18:20:42+00:00" + "time": "2022-08-26T18:20:48+00:00" }, { "name": "bramus/ansi-php", @@ -955,18 +955,19 @@ }, { "name": "matthewbaggett/wait-for-mysql", - "version": "v1.1", + "version": "dev-main", "source": { "type": "git", "url": "https://github.com/matthewbaggett/wait-for-mysql.git", - "reference": "fa8786e752aa77aacf79a8e40df8b84eacde7770" + "reference": "729ef9518a8df1c4685654ffda034e766047ae10" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matthewbaggett/wait-for-mysql/zipball/fa8786e752aa77aacf79a8e40df8b84eacde7770", - "reference": "fa8786e752aa77aacf79a8e40df8b84eacde7770", + "url": "https://api.github.com/repos/matthewbaggett/wait-for-mysql/zipball/729ef9518a8df1c4685654ffda034e766047ae10", + "reference": "729ef9518a8df1c4685654ffda034e766047ae10", "shasum": "" }, + "default-branch": true, "bin": [ "wait-for-mysql", "wait-for-postgresql" @@ -984,9 +985,9 @@ ], "support": { "issues": "https://github.com/matthewbaggett/wait-for-mysql/issues", - "source": "https://github.com/matthewbaggett/wait-for-mysql/tree/v1.1" + "source": "https://github.com/matthewbaggett/wait-for-mysql/tree/main" }, - "time": "2021-09-22T09:25:08+00:00" + "time": "2022-08-28T09:25:52+00:00" }, { "name": "monolog/monolog", @@ -2395,16 +2396,16 @@ }, { "name": "symfony/console", - "version": "v6.1.3", + "version": "v6.1.4", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "43fcb5c5966b43c56bcfa481368d90d748936ab8" + "reference": "7fccea8728aa2d431a6725b02b3ce759049fc84d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/43fcb5c5966b43c56bcfa481368d90d748936ab8", - "reference": "43fcb5c5966b43c56bcfa481368d90d748936ab8", + "url": "https://api.github.com/repos/symfony/console/zipball/7fccea8728aa2d431a6725b02b3ce759049fc84d", + "reference": "7fccea8728aa2d431a6725b02b3ce759049fc84d", "shasum": "" }, "require": { @@ -2471,7 +2472,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v6.1.3" + "source": "https://github.com/symfony/console/tree/v6.1.4" }, "funding": [ { @@ -2487,7 +2488,7 @@ "type": "tidelift" } ], - "time": "2022-07-22T14:17:57+00:00" + "time": "2022-08-26T10:32:31+00:00" }, { "name": "symfony/event-dispatcher", @@ -2653,16 +2654,16 @@ }, { "name": "symfony/filesystem", - "version": "v6.1.3", + "version": "v6.1.4", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "c780e677cddda78417fa5187a7c6cd2f21110db9" + "reference": "3f39c04d2630c34019907b02f85672dac99f8659" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/c780e677cddda78417fa5187a7c6cd2f21110db9", - "reference": "c780e677cddda78417fa5187a7c6cd2f21110db9", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/3f39c04d2630c34019907b02f85672dac99f8659", + "reference": "3f39c04d2630c34019907b02f85672dac99f8659", "shasum": "" }, "require": { @@ -2696,7 +2697,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v6.1.3" + "source": "https://github.com/symfony/filesystem/tree/v6.1.4" }, "funding": [ { @@ -2712,7 +2713,7 @@ "type": "tidelift" } ], - "time": "2022-07-20T14:45:06+00:00" + "time": "2022-08-02T16:17:38+00:00" }, { "name": "symfony/finder", @@ -3464,16 +3465,16 @@ }, { "name": "symfony/string", - "version": "v6.1.3", + "version": "v6.1.4", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "f35241f45c30bcd9046af2bb200a7086f70e1d6b" + "reference": "290972cad7b364e3befaa74ba0ec729800fb161c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/f35241f45c30bcd9046af2bb200a7086f70e1d6b", - "reference": "f35241f45c30bcd9046af2bb200a7086f70e1d6b", + "url": "https://api.github.com/repos/symfony/string/zipball/290972cad7b364e3befaa74ba0ec729800fb161c", + "reference": "290972cad7b364e3befaa74ba0ec729800fb161c", "shasum": "" }, "require": { @@ -3529,7 +3530,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.1.3" + "source": "https://github.com/symfony/string/tree/v6.1.4" }, "funding": [ { @@ -3545,13 +3546,14 @@ "type": "tidelift" } ], - "time": "2022-07-27T15:50:51+00:00" + "time": "2022-08-12T18:05:43+00:00" } ], "aliases": [], "minimum-stability": "stable", "stability-flags": { - "adambrett/shell-wrapper": 20 + "adambrett/shell-wrapper": 20, + "matthewbaggett/wait-for-mysql": 20 }, "prefer-stable": false, "prefer-lowest": false, diff --git a/docker-compose.yml b/docker-compose.yml index cd0b7a5..6c1fc8c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,7 @@ services: image: benzine/postgres:14 build: context: . + dockerfile: Dockerfile.postgres target: postgres args: PGSQL_VERSION: 14 @@ -42,4 +43,30 @@ services: depends_on: - minio volumes: - - ./:/sync \ No newline at end of file + - ./:/sync + + mariadb-10.9: + image: benzine/mysql:10.9 + build: + context: . + target: mysql + dockerfile: Dockerfile.mariadb + args: + MARIADB_VERSION: 10.9 + environment: + MARIADB_RANDOM_ROOT_PASSWORD: "yes" + MARIADB_USER: example + MARIADB_PASSWORD: changeme + MARIADB_DATABASE: s3db + S3_ENDPOINT: http://minio:9000/ + S3_API_KEY: *s3_key + S3_API_SECRET: *s3_secret + S3_USE_PATH_STYLE_ENDPOINT: "yes" + S3_BUCKET: "s3db" + S3_PREFIX: "test/mariadb/" + ports: + - "127.0.0.127:3306:3306" + depends_on: + - minio + volumes: + - ./:/sync diff --git a/mysql.runit b/mysql.runit new file mode 100644 index 0000000..38adcb8 --- /dev/null +++ b/mysql.runit @@ -0,0 +1,6 @@ +#!/bin/bash + +echo "Running docker-entrypoint" +/usr/local/bin/docker-entrypoint.sh mariadbd + +sleep 60 diff --git a/healthcheck b/postgres_healthcheck similarity index 100% rename from healthcheck rename to postgres_healthcheck diff --git a/sync-pull.runit b/sync-pull.runit index 114f280..6d5743a 100644 --- a/sync-pull.runit +++ b/sync-pull.runit @@ -1,8 +1,9 @@ #!/bin/bash sleep 10 -echo "Running sync-pull" -vendor/bin/wait-for-postgres +echo "Waiting for database before running sync-pull.." +/usr/local/bin/wait-for-database +echo "Database read, running sync-pull." /sync/sync --pull sleep infinity \ No newline at end of file diff --git a/syncer/MysqlSyncer.php b/syncer/MysqlSyncer.php new file mode 100644 index 0000000..8d7d355 --- /dev/null +++ b/syncer/MysqlSyncer.php @@ -0,0 +1,77 @@ + /dumps/%s', $dumpFile); + passthru($command); + + // Verify the dump worked + if (!$this->localFilesystem->fileExists($dumpFile)) { + $this->logger->critical('Database dump failed'); + + exit; + } + $this->logger->debug(sprintf( + 'Dump file was made, and is %s uncompressed', + ByteSize::formatMetric( + $this->localFilesystem->fileSize($dumpFile) + ) + )); + + // Checksum dump and don't upload if the checksum is the same as last time. + $hash = sha1_file("/dumps/{$dumpFile}"); + if ($this->localFilesystem->has('previous_hash') && $hash == $this->localFilesystem->read('previous_hash')) { + $this->logger->debug(sprintf( + '%s Dump of %s matches previous dump (%s), not uploading the same file again.', + Emoji::abacus(), + $dumpFile, + substr($hash, 0, 7) + )); + + exit; + } + $this->localFilesystem->write('previous_hash', $hash); + + // XZ compress dump + $compressedDumpFile = $this->compress($dumpFile); + + // Upload + $storageFile = sprintf('s3db-%s.sql.xz', date('Ymd-His')); + $this->upload($storageFile, $compressedDumpFile); + + // Cleanup + $this->cleanup([$compressedDumpFile]); + } + + public function pull(): void + { + // Download latest dumpfile + $localDownloadedFile = $this->download(); + + // Decompress + $localDecompressedFile = $this->decompress($localDownloadedFile); + + // Push into MySQL + $startImport = microtime(true); + $command = sprintf('mysql -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE < /dumps/%s', $localDecompressedFile); + exec($command); + $this->logger->info(sprintf( + '%s Imported %s to MySQL in %s seconds', + Emoji::accordion(), + $localDecompressedFile, + number_format(microtime(true) - $startImport, 3) + )); + + // Cleanup + $this->cleanup([$localDecompressedFile]); + } +} diff --git a/syncer/PostgresAbstractSyncer.php b/syncer/PostgresSyncer.php similarity index 97% rename from syncer/PostgresAbstractSyncer.php rename to syncer/PostgresSyncer.php index 2b79338..884767b 100644 --- a/syncer/PostgresAbstractSyncer.php +++ b/syncer/PostgresSyncer.php @@ -5,7 +5,7 @@ namespace S3DB\Sync; use Rych\ByteSize\ByteSize; use Spatie\Emoji\Emoji; -class PostgresAbstractSyncer extends AbstractSyncer +class PostgresSyncer extends AbstractSyncer { public function push(): void { diff --git a/syncer/Sync.php b/syncer/Sync.php index 6b0ef48..a9d74dc 100644 --- a/syncer/Sync.php +++ b/syncer/Sync.php @@ -46,11 +46,10 @@ class Sync // Postgres mode is enabled if --postgres is set, or PG_VERSION envvar is set, // which it is when we're built ontop of the postgres docker container $this->logger->debug(sprintf('%s Starting in postgres mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE)); - $this->syncer = new PostgresAbstractSyncer($this->logger, $this->storageFilesystem, $this->localFilesystem); - } elseif ($this->args->hasOpt('mysql')) { + $this->syncer = new PostgresSyncer($this->logger, $this->storageFilesystem, $this->localFilesystem); + } elseif ($this->args->hasOpt('mysql') || isset($environment['MARIADB_VERSION'])) { $this->logger->debug(sprintf('%s Starting in mysql mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE)); - - exit('Not implemented yet'); + $this->syncer = new MysqlSyncer($this->logger, $this->storageFilesystem, $this->localFilesystem); } else { $this->logger->critical(sprintf('%s Must be started in either --mysql or --postgres mode!', Emoji::CHARACTER_NERD_FACE));