Benzine/Docker-S3DB
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Working mysql build.

Browse source
This commit is contained in:
Greyscale 2022-08-28 11:38:02 +02:00
parent 94e4fce2df
commit 32e900d0de
No known key found for this signature in database
GPG key ID: 74BAFF55434DA4B2
12 changed files with 285 additions and 51 deletions

54
Dockerfile.mariadb Normal file
View file

@ -0,0 +1,54 @@
ARG MARIADB_VERSION=10.9
FROM mariadb:$MARIADB_VERSION AS mysql
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN echo "APT::Acquire::Retries \"5\";" > /etc/apt/apt.conf.d/80-retries && \
echo "Acquire::http::No-Cache=true;" > /etc/apt/apt.conf.d/80-no-cache && \
echo "Acquire::http::Pipeline-Depth=0;" > /etc/apt/apt.conf.d/80-no-pipeline && \
apt-get -qq update && \
apt-get -yqq upgrade && \
apt-get -yqq install --no-install-recommends \
xz-utils \
dos2unix \
runit \
python3-software-properties \
software-properties-common \
&& \
add-apt-repository -y ppa:ondrej/php && \
apt-get -qq update && \
apt-get -yqq install --no-install-recommends \
#php8.1-apcu \
php8.1-bcmath \
#php8.1-bz2 \
php8.1-cli \
php8.1-curl \
php8.1-mysql \
#php8.1-opcache \
#php8.1-phpdbg \
php8.1-xml \
php8.1-zip \
&& \
apt-get remove -yqq \
software-properties-common \
python-apt-common \
python3-software-properties \
python3.5 python3.5-minimal libpython3.5-minimal \
&& \
apt-get autoremove -yqq && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/lib/dpkg/status.old /var/cache/debconf/templates.dat /var/log/dpkg.log /var/log/lastlog /var/log/apt/*.log
COPY start.sh /usr/local/bin/start.sh
COPY mysql.runit /etc/service/mysql/run
COPY sync-pull.runit /etc/service/sync-pull/run
COPY sync-push.runit /etc/service/sync-push/run
VOLUME /dumps
WORKDIR /sync
COPY vendor /sync/vendor
COPY syncer /sync/syncer
COPY sync /sync/sync
COPY start.sh /sync/start.sh
ENV PATH="/sync:${PATH}"
RUN chmod +x /sync/sync /etc/service/*/run
CMD ["start.sh"]
HEALTHCHECK --start-period=30s CMD /usr/local/bin/healthcheck.sh
RUN ln -s /sync/vendor/bin/wait-for-mysql /usr/local/bin/wait-for-database

14
Dockerfile → Dockerfile.postgres
View file

@ -1,7 +1,7 @@
ARG PGSQL_VERSION
FROM postgres:$PGSQL_VERSION-alpine AS postgres
RUN apk add --no-cache runit
RUN apk --no-cache --repository https://dl-cdn.alpinelinux.org/alpine/edge/main add \
RUN apk add --no-cache runit && \
apk --no-cache --repository https://dl-cdn.alpinelinux.org/alpine/edge/main add \
icu-libs \
&&\
apk --no-cache --repository https://dl-cdn.alpinelinux.org/alpine/edge/community add \
@ -49,8 +49,12 @@ COPY sync-pull.runit /etc/service/sync-pull/run
COPY sync-push.runit /etc/service/sync-push/run
VOLUME /dumps
WORKDIR /sync
COPY . /sync
COPY vendor /sync/vendor
COPY syncer /sync/syncer
COPY sync /sync/sync
COPY start.sh /sync/start.sh
ENV PATH="/sync:${PATH}"
RUN chmod +x /sync/sync /etc/service/*/run /sync/healthcheck
CMD ["start.sh"]
HEALTHCHECK --start-period=30s CMD /sync/healthcheck
COPY postgres_healthcheck /sync/postgres_healthcheck
HEALTHCHECK --start-period=30s CMD /sync/postgres_healthcheck
RUN ln -s /sync/vendor/bin/wait-for-postgresql /usr/local/bin/wait-for-database

74
bake.hcl
View file

@ -10,7 +10,7 @@ group "default" {
target "postgres-14" {
context = "."
dockerfile = "Dockerfile"
dockerfile = "Dockerfile.postgres"
platforms = ["arm64","amd64"]
tags = ["benzine/postgres:14", "benzine/postgres:latest"]
args = {
@ -20,7 +20,7 @@ target "postgres-14" {
target "postgres-13" {
context = "."
dockerfile = "Dockerfile"
dockerfile = "Dockerfile.postgres"
platforms = ["arm64","amd64"]
tags = ["benzine/postgres:13"]
args = {
@ -30,7 +30,7 @@ target "postgres-13" {
target "postgres-12" {
context = "."
dockerfile = "Dockerfile"
dockerfile = "Dockerfile.postgres"
platforms = ["arm64","amd64"]
tags = ["benzine/postgres:12"]
args = {
@ -40,7 +40,7 @@ target "postgres-12" {
target "postgres-11" {
context = "."
dockerfile = "Dockerfile"
dockerfile = "Dockerfile.postgres"
platforms = ["arm64","amd64"]
tags = ["benzine/postgres:11"]
args = {
@ -50,10 +50,74 @@ target "postgres-11" {
target "postgres-10" {
context = "."
dockerfile = "Dockerfile"
dockerfile = "Dockerfile.postgres"
platforms = ["arm64","amd64"]
tags = ["benzine/postgres:10"]
args = {
PGSQL_VERSION = 10
}
}
target "mariadb-10.9" {
context = "."
dockerfile = "Dockerfile.mariadb"
platforms = ["arm64", "amd64"]
tags = ["benzine/mariadb:10.9", "benzine/mariadb:latest"]
args = {
MARIADB_VERSION=10.9
}
}
target "mariadb-10.8" {
context = "."
dockerfile = "Dockerfile.mariadb"
platforms = ["arm64", "amd64"]
tags = ["benzine/mariadb:10.8"]
args = {
MARIADB_VERSION=10.8
}
}
target "mariadb-10.7" {
context = "."
dockerfile = "Dockerfile.mariadb"
platforms = ["arm64", "amd64"]
tags = ["benzine/mariadb:10.7"]
args = {
MARIADB_VERSION=10.7
}
}
target "mariadb-10.6" {
context = "."
dockerfile = "Dockerfile.mariadb"
platforms = ["arm64", "amd64"]
tags = ["benzine/mariadb:10.6"]
args = {
MARIADB_VERSION=10.6
}
}
target "mariadb-10.5" {
context = "."
dockerfile = "Dockerfile.mariadb"
platforms = ["arm64", "amd64"]
tags = ["benzine/mariadb:10.5"]
args = {
MARIADB_VERSION=10.5
}
}
target "mariadb-10.4" {
context = "."
dockerfile = "Dockerfile.mariadb"
platforms = ["arm64", "amd64"]
tags = ["benzine/mariadb:10.4"]
args = {
MARIADB_VERSION=10.4
}
}
target "mariadb-10.3" {
context = "."
dockerfile = "Dockerfile.mariadb"
platforms = ["arm64", "amd64"]
tags = ["benzine/mariadb:10.3"]
args = {
MARIADB_VERSION=10.3
}
}

2
composer.json
View file

@ -15,7 +15,7 @@
"spatie/emoji": "^2.3",
"rych/bytesize": "^1.0",
"jimmiw/php-time-ago": "^3.2",
"matthewbaggett/wait-for-mysql": "^1.0"
"matthewbaggett/wait-for-mysql": "dev-main"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "^3.0"

66
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "9e33df3b624d4c411ab342f127b432cd",
"content-hash": "519bdadff1be790c998c5adf50f9384c",
"packages": [
{
"name": "adambrett/shell-wrapper",
@ -108,16 +108,16 @@
},
{
"name": "aws/aws-sdk-php",
"version": "3.234.0",
"version": "3.234.4",
"source": {
"type": "git",
"url": "https://github.com/aws/aws-sdk-php.git",
"reference": "d2113f1e5ec9f7f19de2472f5063333b39a55280"
"reference": "8d56ddb99632200273bb933cbf82b758ab9cde2a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/d2113f1e5ec9f7f19de2472f5063333b39a55280",
"reference": "d2113f1e5ec9f7f19de2472f5063333b39a55280",
"url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/8d56ddb99632200273bb933cbf82b758ab9cde2a",
"reference": "8d56ddb99632200273bb933cbf82b758ab9cde2a",
"shasum": ""
},
"require": {
@ -194,9 +194,9 @@
"support": {
"forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80",
"issues": "https://github.com/aws/aws-sdk-php/issues",
"source": "https://github.com/aws/aws-sdk-php/tree/3.234.0"
"source": "https://github.com/aws/aws-sdk-php/tree/3.234.4"
},
"time": "2022-08-22T18:20:42+00:00"
"time": "2022-08-26T18:20:48+00:00"
},
{
"name": "bramus/ansi-php",
@ -955,18 +955,19 @@
},
{
"name": "matthewbaggett/wait-for-mysql",
"version": "v1.1",
"version": "dev-main",
"source": {
"type": "git",
"url": "https://github.com/matthewbaggett/wait-for-mysql.git",
"reference": "fa8786e752aa77aacf79a8e40df8b84eacde7770"
"reference": "729ef9518a8df1c4685654ffda034e766047ae10"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/matthewbaggett/wait-for-mysql/zipball/fa8786e752aa77aacf79a8e40df8b84eacde7770",
"reference": "fa8786e752aa77aacf79a8e40df8b84eacde7770",
"url": "https://api.github.com/repos/matthewbaggett/wait-for-mysql/zipball/729ef9518a8df1c4685654ffda034e766047ae10",
"reference": "729ef9518a8df1c4685654ffda034e766047ae10",
"shasum": ""
},
"default-branch": true,
"bin": [
"wait-for-mysql",
"wait-for-postgresql"
@ -984,9 +985,9 @@
],
"support": {
"issues": "https://github.com/matthewbaggett/wait-for-mysql/issues",
"source": "https://github.com/matthewbaggett/wait-for-mysql/tree/v1.1"
"source": "https://github.com/matthewbaggett/wait-for-mysql/tree/main"
},
"time": "2021-09-22T09:25:08+00:00"
"time": "2022-08-28T09:25:52+00:00"
},
{
"name": "monolog/monolog",
@ -2395,16 +2396,16 @@
},
{
"name": "symfony/console",
"version": "v6.1.3",
"version": "v6.1.4",
"source": {
"type": "git",
"url": "https://github.com/symfony/console.git",
"reference": "43fcb5c5966b43c56bcfa481368d90d748936ab8"
"reference": "7fccea8728aa2d431a6725b02b3ce759049fc84d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/console/zipball/43fcb5c5966b43c56bcfa481368d90d748936ab8",
"reference": "43fcb5c5966b43c56bcfa481368d90d748936ab8",
"url": "https://api.github.com/repos/symfony/console/zipball/7fccea8728aa2d431a6725b02b3ce759049fc84d",
"reference": "7fccea8728aa2d431a6725b02b3ce759049fc84d",
"shasum": ""
},
"require": {
@ -2471,7 +2472,7 @@
"terminal"
],
"support": {
"source": "https://github.com/symfony/console/tree/v6.1.3"
"source": "https://github.com/symfony/console/tree/v6.1.4"
},
"funding": [
{
@ -2487,7 +2488,7 @@
"type": "tidelift"
}
],
"time": "2022-07-22T14:17:57+00:00"
"time": "2022-08-26T10:32:31+00:00"
},
{
"name": "symfony/event-dispatcher",
@ -2653,16 +2654,16 @@
},
{
"name": "symfony/filesystem",
"version": "v6.1.3",
"version": "v6.1.4",
"source": {
"type": "git",
"url": "https://github.com/symfony/filesystem.git",
"reference": "c780e677cddda78417fa5187a7c6cd2f21110db9"
"reference": "3f39c04d2630c34019907b02f85672dac99f8659"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/filesystem/zipball/c780e677cddda78417fa5187a7c6cd2f21110db9",
"reference": "c780e677cddda78417fa5187a7c6cd2f21110db9",
"url": "https://api.github.com/repos/symfony/filesystem/zipball/3f39c04d2630c34019907b02f85672dac99f8659",
"reference": "3f39c04d2630c34019907b02f85672dac99f8659",
"shasum": ""
},
"require": {
@ -2696,7 +2697,7 @@
"description": "Provides basic utilities for the filesystem",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/filesystem/tree/v6.1.3"
"source": "https://github.com/symfony/filesystem/tree/v6.1.4"
},
"funding": [
{
@ -2712,7 +2713,7 @@
"type": "tidelift"
}
],
"time": "2022-07-20T14:45:06+00:00"
"time": "2022-08-02T16:17:38+00:00"
},
{
"name": "symfony/finder",
@ -3464,16 +3465,16 @@
},
{
"name": "symfony/string",
"version": "v6.1.3",
"version": "v6.1.4",
"source": {
"type": "git",
"url": "https://github.com/symfony/string.git",
"reference": "f35241f45c30bcd9046af2bb200a7086f70e1d6b"
"reference": "290972cad7b364e3befaa74ba0ec729800fb161c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/string/zipball/f35241f45c30bcd9046af2bb200a7086f70e1d6b",
"reference": "f35241f45c30bcd9046af2bb200a7086f70e1d6b",
"url": "https://api.github.com/repos/symfony/string/zipball/290972cad7b364e3befaa74ba0ec729800fb161c",
"reference": "290972cad7b364e3befaa74ba0ec729800fb161c",
"shasum": ""
},
"require": {
@ -3529,7 +3530,7 @@
"utf8"
],
"support": {
"source": "https://github.com/symfony/string/tree/v6.1.3"
"source": "https://github.com/symfony/string/tree/v6.1.4"
},
"funding": [
{
@ -3545,13 +3546,14 @@
"type": "tidelift"
}
],
"time": "2022-07-27T15:50:51+00:00"
"time": "2022-08-12T18:05:43+00:00"
}
],
"aliases": [],
"minimum-stability": "stable",
"stability-flags": {
"adambrett/shell-wrapper": 20
"adambrett/shell-wrapper": 20,
"matthewbaggett/wait-for-mysql": 20
},
"prefer-stable": false,
"prefer-lowest": false,

29
docker-compose.yml
View file

@ -25,6 +25,7 @@ services:
image: benzine/postgres:14
build:
context: .
dockerfile: Dockerfile.postgres
target: postgres
args:
PGSQL_VERSION: 14
@ -42,4 +43,30 @@ services:
depends_on:
- minio
volumes:
- ./:/sync
- ./:/sync
mariadb-10.9:
image: benzine/mysql:10.9
build:
context: .
target: mysql
dockerfile: Dockerfile.mariadb
args:
MARIADB_VERSION: 10.9
environment:
MARIADB_RANDOM_ROOT_PASSWORD: "yes"
MARIADB_USER: example
MARIADB_PASSWORD: changeme
MARIADB_DATABASE: s3db
S3_ENDPOINT: http://minio:9000/
S3_API_KEY: *s3_key
S3_API_SECRET: *s3_secret
S3_USE_PATH_STYLE_ENDPOINT: "yes"
S3_BUCKET: "s3db"
S3_PREFIX: "test/mariadb/"
ports:
- "127.0.0.127:3306:3306"
depends_on:
- minio
volumes:
- ./:/sync

6
mysql.runit Normal file
View file

@ -0,0 +1,6 @@
#!/bin/bash
echo "Running docker-entrypoint"
/usr/local/bin/docker-entrypoint.sh mariadbd
sleep 60

0
healthcheck → postgres_healthcheck
View file

5
sync-pull.runit
View file

@ -1,8 +1,9 @@
#!/bin/bash
sleep 10
echo "Running sync-pull"
vendor/bin/wait-for-postgres
echo "Waiting for database before running sync-pull.."
/usr/local/bin/wait-for-database
echo "Database read, running sync-pull."
/sync/sync --pull
sleep infinity

77
syncer/MysqlSyncer.php Normal file
View file

@ -0,0 +1,77 @@
<?php
namespace S3DB\Sync;
use Rych\ByteSize\ByteSize;
use Spatie\Emoji\Emoji;
class MysqlSyncer extends AbstractSyncer
{
public function push(): void
{
// Dump file from Postgres
$dumpFile = 'dump.sql';
$command = sprintf('mysqldump -u $MARIADB_USER -p$MARIADB_PASSWORD --extended-insert --quick --add-locks --add-drop-database --add-drop-table --add-drop-trigger $MARIADB_DATABASE > /dumps/%s', $dumpFile);
passthru($command);
// Verify the dump worked
if (!$this->localFilesystem->fileExists($dumpFile)) {
$this->logger->critical('Database dump failed');
exit;
}
$this->logger->debug(sprintf(
'Dump file was made, and is %s uncompressed',
ByteSize::formatMetric(
$this->localFilesystem->fileSize($dumpFile)
)
));
// Checksum dump and don't upload if the checksum is the same as last time.
$hash = sha1_file("/dumps/{$dumpFile}");
if ($this->localFilesystem->has('previous_hash') && $hash == $this->localFilesystem->read('previous_hash')) {
$this->logger->debug(sprintf(
'%s Dump of %s matches previous dump (%s), not uploading the same file again.',
Emoji::abacus(),
$dumpFile,
substr($hash, 0, 7)
));
exit;
}
$this->localFilesystem->write('previous_hash', $hash);
// XZ compress dump
$compressedDumpFile = $this->compress($dumpFile);
// Upload
$storageFile = sprintf('s3db-%s.sql.xz', date('Ymd-His'));
$this->upload($storageFile, $compressedDumpFile);
// Cleanup
$this->cleanup([$compressedDumpFile]);
}
public function pull(): void
{
// Download latest dumpfile
$localDownloadedFile = $this->download();
// Decompress
$localDecompressedFile = $this->decompress($localDownloadedFile);
// Push into MySQL
$startImport = microtime(true);
$command = sprintf('mysql -u $MARIADB_USER -p$MARIADB_PASSWORD $MARIADB_DATABASE < /dumps/%s', $localDecompressedFile);
exec($command);
$this->logger->info(sprintf(
'%s Imported %s to MySQL in %s seconds',
Emoji::accordion(),
$localDecompressedFile,
number_format(microtime(true) - $startImport, 3)
));
// Cleanup
$this->cleanup([$localDecompressedFile]);
}
}

2
syncer/PostgresAbstractSyncer.php → syncer/PostgresSyncer.php
View file

@ -5,7 +5,7 @@ namespace S3DB\Sync;
use Rych\ByteSize\ByteSize;
use Spatie\Emoji\Emoji;
class PostgresAbstractSyncer extends AbstractSyncer
class PostgresSyncer extends AbstractSyncer
{
public function push(): void
{

7
syncer/Sync.php
View file

@ -46,11 +46,10 @@ class Sync
// Postgres mode is enabled if --postgres is set, or PG_VERSION envvar is set,
// which it is when we're built ontop of the postgres docker container
$this->logger->debug(sprintf('%s Starting in postgres mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE));
$this->syncer = new PostgresAbstractSyncer($this->logger, $this->storageFilesystem, $this->localFilesystem);
} elseif ($this->args->hasOpt('mysql')) {
$this->syncer = new PostgresSyncer($this->logger, $this->storageFilesystem, $this->localFilesystem);
} elseif ($this->args->hasOpt('mysql') || isset($environment['MARIADB_VERSION'])) {
$this->logger->debug(sprintf('%s Starting in mysql mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE));
exit('Not implemented yet');
$this->syncer = new MysqlSyncer($this->logger, $this->storageFilesystem, $this->localFilesystem);
} else {
$this->logger->critical(sprintf('%s Must be started in either --mysql or --postgres mode!', Emoji::CHARACTER_NERD_FACE));