From 09ad28584f7f239e4ebb057316474335e3f70a4f Mon Sep 17 00:00:00 2001 From: Matthew Baggett <matthew@baggett.me> Date: Tue, 23 Aug 2022 14:19:05 +0200 Subject: [PATCH] Initial working version. --- .gitignore | 1 + .php-cs-fixer.php | 5 +- Dockerfile | 10 +- composer.json | 5 +- composer.lock | 150 +++++++++++++++++++++-- docker-compose.yml | 20 ++- postgres.runit | 2 +- sync | 2 +- sync-pull.runit | 8 ++ sync-push.runit | 4 + syncer/AbstractSyncer.php | 121 +++++++++++++++++- syncer/Filesystems/LocalFilesystem.php | 16 +++ syncer/Filesystems/StorageFilesystem.php | 30 +++++ syncer/PostgresAbstractSyncer.php | 49 ++++++++ syncer/Sync.php | 21 +++- 15 files changed, 418 insertions(+), 26 deletions(-) mode change 100644 => 100755 sync create mode 100644 sync-pull.runit create mode 100644 sync-push.runit create mode 100644 syncer/Filesystems/LocalFilesystem.php create mode 100644 syncer/Filesystems/StorageFilesystem.php diff --git a/.gitignore b/.gitignore index 20351e1..bca31cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .php-cs-fixer.cache .idea /vendor/ +.minio \ No newline at end of file diff --git a/.php-cs-fixer.php b/.php-cs-fixer.php index 46a948c..56cfebf 100644 --- a/.php-cs-fixer.php +++ b/.php-cs-fixer.php @@ -1,8 +1,9 @@ <?php + $finder = PhpCsFixer\Finder::create(); $finder->in(__DIR__); -return (new PhpCsFixer\Config) +return (new PhpCsFixer\Config()) ->setRiskyAllowed(true) ->setHideProgress(false) ->setRules([ @@ -19,4 +20,4 @@ return (new PhpCsFixer\Config) 'yoda_style' => false, ]) ->setFinder($finder) - ; +; diff --git a/Dockerfile b/Dockerfile index 0b69eb1..7aa6b80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,12 +36,20 @@ RUN apk --no-cache --repository https://dl-cdn.alpinelinux.org/alpine/edge/main php81-fpm \ php81-sodium \ php81-tokenizer \ + php81-fileinfo \ + php81-simplexml \ # Iconv Fix php81-pecl-apcu \ + ncurses \ + xz \ && ln -s /usr/bin/php81 /usr/bin/php COPY start.sh /usr/local/bin/start.sh COPY postgres.runit /etc/service/postgres/run +COPY sync-pull.runit /etc/service/sync-pull/run +COPY sync-push.runit /etc/service/sync-push/run +VOLUME /dumps WORKDIR /sync COPY . /sync -RUN chmod +x /sync/sync +ENV PATH="/sync:${PATH}" +RUN chmod +x /sync/sync /etc/service/*/run CMD ["start.sh"] \ No newline at end of file diff --git a/composer.json b/composer.json index 13ec9c4..f22d221 100644 --- a/composer.json +++ b/composer.json @@ -12,7 +12,10 @@ "monolog/monolog": "^2.2", "bramus/monolog-colored-line-formatter": "~3.0", "adambrett/shell-wrapper": "dev-master", - "spatie/emoji": "^2.3" + "spatie/emoji": "^2.3", + "rych/bytesize": "^1.0", + "jimmiw/php-time-ago": "^3.2", + "matthewbaggett/wait-for-mysql": "^1.0" }, "require-dev": { "friendsofphp/php-cs-fixer": "^3.0" diff --git a/composer.lock b/composer.lock index c6659b3..0ed4741 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cb65968236e13f317b1a5db79f853c52", + "content-hash": "9e33df3b624d4c411ab342f127b432cd", "packages": [ { "name": "adambrett/shell-wrapper", @@ -108,16 +108,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.233.6", + "version": "3.234.0", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "cd9019a38cee8cabfd42dc7692e54106c93c1e6a" + "reference": "d2113f1e5ec9f7f19de2472f5063333b39a55280" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/cd9019a38cee8cabfd42dc7692e54106c93c1e6a", - "reference": "cd9019a38cee8cabfd42dc7692e54106c93c1e6a", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/d2113f1e5ec9f7f19de2472f5063333b39a55280", + "reference": "d2113f1e5ec9f7f19de2472f5063333b39a55280", "shasum": "" }, "require": { @@ -194,9 +194,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.233.6" + "source": "https://github.com/aws/aws-sdk-php/tree/3.234.0" }, - "time": "2022-08-19T18:16:02+00:00" + "time": "2022-08-22T18:20:42+00:00" }, { "name": "bramus/ansi-php", @@ -611,6 +611,58 @@ ], "time": "2022-06-20T21:43:11+00:00" }, + { + "name": "jimmiw/php-time-ago", + "version": "3.2.3", + "source": { + "type": "git", + "url": "https://github.com/jimmiw/php-time-ago.git", + "reference": "2e0da84d3bd35344f44582ea78c85c75d937c457" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/jimmiw/php-time-ago/zipball/2e0da84d3bd35344f44582ea78c85c75d937c457", + "reference": "2e0da84d3bd35344f44582ea78c85c75d937c457", + "shasum": "" + }, + "require": { + "php": ">=7.0.0" + }, + "require-dev": { + "phpmd/phpmd": "@stable", + "phpunit/phpunit": "^6", + "squizlabs/php_codesniffer": "^3.3" + }, + "type": "library", + "autoload": { + "psr-4": { + "Westsworld\\": "src/Westsworld/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jimmi Westerberg", + "homepage": "http://www.westsworld.dk", + "role": "Developer" + } + ], + "description": "Simple module, that displays the date in a \"time ago\" format", + "homepage": "https://github.com/jimmiw/php-time-ago", + "keywords": [ + "distance of time", + "time ago", + "time ago in words" + ], + "support": { + "issues": "https://github.com/jimmiw/php-time-ago/issues", + "source": "https://github.com/jimmiw/php-time-ago/tree/3.2.3" + }, + "time": "2022-06-10T10:15:42+00:00" + }, { "name": "kint-php/kint", "version": "3.3", @@ -901,6 +953,41 @@ ], "time": "2022-04-17T13:12:02+00:00" }, + { + "name": "matthewbaggett/wait-for-mysql", + "version": "v1.1", + "source": { + "type": "git", + "url": "https://github.com/matthewbaggett/wait-for-mysql.git", + "reference": "fa8786e752aa77aacf79a8e40df8b84eacde7770" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/matthewbaggett/wait-for-mysql/zipball/fa8786e752aa77aacf79a8e40df8b84eacde7770", + "reference": "fa8786e752aa77aacf79a8e40df8b84eacde7770", + "shasum": "" + }, + "bin": [ + "wait-for-mysql", + "wait-for-postgresql" + ], + "type": "library", + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-3.0-or-later" + ], + "authors": [ + { + "name": "Matthew Baggett", + "email": "matthew@baggett.me" + } + ], + "support": { + "issues": "https://github.com/matthewbaggett/wait-for-mysql/issues", + "source": "https://github.com/matthewbaggett/wait-for-mysql/tree/v1.1" + }, + "time": "2021-09-22T09:25:08+00:00" + }, { "name": "monolog/monolog", "version": "2.8.0", @@ -1318,6 +1405,55 @@ }, "time": "2019-03-08T08:55:37+00:00" }, + { + "name": "rych/bytesize", + "version": "v1.0.0", + "source": { + "type": "git", + "url": "https://github.com/rchouinard/bytesize.git", + "reference": "297e16ea047461b91e8d7eb90aa46aaa52917824" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/rchouinard/bytesize/zipball/297e16ea047461b91e8d7eb90aa46aaa52917824", + "reference": "297e16ea047461b91e8d7eb90aa46aaa52917824", + "shasum": "" + }, + "require": { + "ext-bcmath": "*", + "php": ">=5.3.4" + }, + "require-dev": { + "phpunit/phpunit": "3.7.*" + }, + "type": "library", + "autoload": { + "psr-4": { + "Rych\\ByteSize\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ryan Chouinard", + "email": "rchouinard@gmail.com", + "homepage": "http://ryanchouinard.com" + } + ], + "description": "Utility component for nicely formatted file sizes.", + "homepage": "https://github.com/rchouinard/bytesize", + "keywords": [ + "filesize" + ], + "support": { + "issues": "https://github.com/rchouinard/bytesize/issues", + "source": "https://github.com/rchouinard/bytesize/tree/master" + }, + "time": "2014-04-04T18:06:18+00:00" + }, { "name": "spatie/emoji", "version": "2.3.1", diff --git a/docker-compose.yml b/docker-compose.yml index b156956..cd0b7a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,11 +2,14 @@ version: "3.7" services: minio: - image: quay.io/minio/minio:RELEASE.2022-08-13T21-54-44Z - command: server --console-address ":9001" http://minio{1...4}/data{1...2} + image: minio/minio + command: server --console-address ":9001" /data + ports: + - "127.0.0.127:9000:9000" + - "127.0.0.127:9001:9001" expose: - - "9000" - - "9001" + - 9000 + - 9001 environment: MINIO_ROOT_USER: &s3_key minio MINIO_ROOT_PASSWORD: &s3_secret changeme @@ -15,6 +18,8 @@ services: interval: 30s timeout: 20s retries: 3 + volumes: + - ./.minio/data:/data postgres-14: image: benzine/postgres:14 @@ -29,7 +34,12 @@ services: S3_ENDPOINT: http://minio:9000/ S3_API_KEY: *s3_key S3_API_SECRET: *s3_secret + S3_USE_PATH_STYLE_ENDPOINT: "yes" + S3_BUCKET: "s3db" + S3_PREFIX: "test/postgres/" ports: - "127.0.0.127:5432:5432" depends_on: - - minio \ No newline at end of file + - minio + volumes: + - ./:/sync \ No newline at end of file diff --git a/postgres.runit b/postgres.runit index 1e0e8d2..01bf8f6 100644 --- a/postgres.runit +++ b/postgres.runit @@ -3,4 +3,4 @@ echo "Running docker-entrypoint" /usr/local/bin/docker-entrypoint.sh postgres -sleep 60 \ No newline at end of file +sleep 60 diff --git a/sync b/sync old mode 100644 new mode 100755 index 00ac99f..214772a --- a/sync +++ b/sync @@ -5,4 +5,4 @@ use S3DB\Sync\Sync; require_once 'vendor/autoload.php'; -(new Sync())->run(); \ No newline at end of file +(new Sync())->run(); diff --git a/sync-pull.runit b/sync-pull.runit new file mode 100644 index 0000000..114f280 --- /dev/null +++ b/sync-pull.runit @@ -0,0 +1,8 @@ +#!/bin/bash + +sleep 10 +echo "Running sync-pull" +vendor/bin/wait-for-postgres +/sync/sync --pull + +sleep infinity \ No newline at end of file diff --git a/sync-push.runit b/sync-push.runit new file mode 100644 index 0000000..ada99cd --- /dev/null +++ b/sync-push.runit @@ -0,0 +1,4 @@ +#!/bin/bash + +sleep 600 +/sync/sync --push diff --git a/syncer/AbstractSyncer.php b/syncer/AbstractSyncer.php index 785a4fb..9ef4e2e 100644 --- a/syncer/AbstractSyncer.php +++ b/syncer/AbstractSyncer.php @@ -2,12 +2,20 @@ namespace S3DB\Sync; +use League\Flysystem\FileAttributes; use Monolog\Logger; +use Rych\ByteSize\ByteSize; +use S3DB\Sync\Filesystems\LocalFilesystem; +use S3DB\Sync\Filesystems\StorageFilesystem; +use Spatie\Emoji\Emoji; +use Westsworld\TimeAgo; abstract class AbstractSyncer { public function __construct( - protected Logger $logger + protected Logger $logger, + protected StorageFilesystem $storageFilesystem, + protected LocalFilesystem $localFilesystem ) { } @@ -15,7 +23,116 @@ abstract class AbstractSyncer abstract public function pull(); - public function uploadToS3(): void + protected function download(): string { + $filesInS3 = $this->storageFilesystem->listContents('/')->toArray(); + usort($filesInS3, function (FileAttributes $a, FileAttributes $b) { + return $a->lastModified() < $b->lastModified(); + }); + + /** @var FileAttributes $file */ + foreach ($filesInS3 as $file) { + $this->logger->debug(sprintf( + '%s Found %s. It is %s and was created %s', + Emoji::magnifyingGlassTiltedLeft(), + $file->path(), + ByteSize::formatMetric( + $file->fileSize() + ), + (new TimeAgo())->inWords((new \DateTime())->setTimestamp($file->lastModified())) + )); + } + + // Choose which we're downloadin' + $latest = $filesInS3[0]; + $this->logger->debug(sprintf( + '%s Selecting %s... Downloading %s...', + Emoji::downArrow(), + $latest->path(), + ByteSize::formatMetric($latest->fileSize()) + )); + + $localDownloadedFile = basename($latest->path()); + $this->localFilesystem->writeStream( + $localDownloadedFile, + $this->storageFilesystem->readStream( + $latest->path() + ) + ); + + return $localDownloadedFile; + } + protected function upload(string $remoteStorageFile, string $localCompressedDumpFile): void + { + $startUpload = microtime(true); + $this->storageFilesystem->writeStream( + $remoteStorageFile, + $this->localFilesystem->readStream($localCompressedDumpFile) + ); + $this->logger->debug(sprintf( + 'Uploaded %s as %s to S3 in %s seconds', + $localCompressedDumpFile, + $remoteStorageFile, + number_format(microtime(true) - $startUpload, 3) + )); + } + + protected function cleanup(array $files): void + { + $cumulativeBytes = 0; + foreach ($files as $file) { + $cumulativeBytes += $this->localFilesystem->fileSize($file); + $this->localFilesystem->delete($file); + } + $this->logger->debug(sprintf( + '%s Cleanup: Deleted %d files, freed %s', + Emoji::wastebasket(), + count($files), + ByteSize::formatMetric($cumulativeBytes) + )); + } + + protected function compress(string $file): string + { + $startCompression = microtime(true); + passthru(sprintf('xz -f -T0 -6 /dumps/%s', $file)); + $compressedFile = "{$file}.xz"; + $this->logger->debug(sprintf( + '%s Dump file was made, and is %s compressed in %s seconds', + Emoji::computerDisk(), + ByteSize::formatMetric( + $this->localFilesystem->fileSize($compressedFile) + ), + number_format(microtime(true) - $startCompression, 3) + )); + + return $compressedFile; + } + + protected function decompress(string $compressedFile): string + { + $startDecompression = microtime(true); + if (!substr($compressedFile, -3, 3) == '.xz') { + $this->logger->critical(sprintf( + '%s Compressed file %s does not end in .xz', + Emoji::explodingHead(), + $compressedFile + )); + + exit; + } + $uncompressedFile = substr($compressedFile, 0, -3); + passthru(sprintf('xz -d -f /dumps/%s', $compressedFile)); + + $this->logger->debug(sprintf( + '%s Dump file %s was uncompressed from %s to %s in %s seconds', + Emoji::computerDisk(), + $uncompressedFile, + ByteSize::formatMetric($this->storageFilesystem->fileSize($compressedFile)), + ByteSize::formatMetric($this->localFilesystem->fileSize($uncompressedFile)), + number_format(microtime(true) - $startDecompression, 3) + )); + + return $uncompressedFile; } } diff --git a/syncer/Filesystems/LocalFilesystem.php b/syncer/Filesystems/LocalFilesystem.php new file mode 100644 index 0000000..83d0a1a --- /dev/null +++ b/syncer/Filesystems/LocalFilesystem.php @@ -0,0 +1,16 @@ +<?php + +namespace S3DB\Sync\Filesystems; + +use League\Flysystem\Filesystem; +use League\Flysystem\Local\LocalFilesystemAdapter; + +class LocalFilesystem extends Filesystem +{ + public function __construct() + { + $environment = array_merge($_ENV, $_SERVER); + $localAdapter = new LocalFilesystemAdapter('/dumps/'); + parent::__construct($localAdapter); + } +} diff --git a/syncer/Filesystems/StorageFilesystem.php b/syncer/Filesystems/StorageFilesystem.php new file mode 100644 index 0000000..1922ac5 --- /dev/null +++ b/syncer/Filesystems/StorageFilesystem.php @@ -0,0 +1,30 @@ +<?php + +namespace S3DB\Sync\Filesystems; + +use Aws\S3\S3Client; +use League\Flysystem\AwsS3V3\AwsS3V3Adapter; +use League\Flysystem\Filesystem; + +class StorageFilesystem extends Filesystem +{ + public function __construct() + { + $environment = array_merge($_ENV, $_SERVER); + $s3Adapter = new AwsS3V3Adapter( + new S3Client([ + 'endpoint' => $environment['S3_ENDPOINT'], + 'use_path_style_endpoint' => isset($environment['S3_USE_PATH_STYLE_ENDPOINT']), + 'credentials' => [ + 'key' => $environment['S3_API_KEY'], + 'secret' => $environment['S3_API_SECRET'], + ], + 'region' => $environment['S3_REGION'] ?? 'us-east', + 'version' => 'latest', + ]), + $environment['S3_BUCKET'], + $environment['S3_PREFIX'] ?? null + ); + parent::__construct($s3Adapter); + } +} diff --git a/syncer/PostgresAbstractSyncer.php b/syncer/PostgresAbstractSyncer.php index 09df4a8..73760f4 100644 --- a/syncer/PostgresAbstractSyncer.php +++ b/syncer/PostgresAbstractSyncer.php @@ -2,13 +2,62 @@ namespace S3DB\Sync; +use Rych\ByteSize\ByteSize; +use Spatie\Emoji\Emoji; + class PostgresAbstractSyncer extends AbstractSyncer { public function push(): void { + // Dump file from Postgres + $dumpFile = 'dump.sql'; + $command = sprintf('PG_PASSWORD=$POSTGRESS_PASSWORD pg_dump -U $POSTGRES_USER --clean --inserts > /dumps/%s', $dumpFile); + passthru($command); + + // Verify the dump worked + if (!$this->localFilesystem->fileExists($dumpFile)) { + $this->logger->critical('Database dump failed'); + + exit; + } + $this->logger->debug(sprintf( + 'Dump file was made, and is %s uncompressed', + ByteSize::formatMetric( + $this->localFilesystem->fileSize($dumpFile) + ) + )); + + // XZ compress dump + $compressedDumpFile = $this->compress($dumpFile); + + // Upload + $storageFile = sprintf('s3db-%s.sql.xz', date('Ymd-His')); + $this->upload($storageFile, $compressedDumpFile); + + // Cleanup + $this->cleanup([$compressedDumpFile]); } public function pull(): void { + // Download latest dumpfile + $localDownloadedFile = $this->download(); + + // Decompress + $localDecompressedFile = $this->decompress($localDownloadedFile); + + // Push into postgres + $startImport = microtime(true); + $command = sprintf('PG_PASSWORD=$POSTGRESS_PASSWORD psql -U $POSTGRES_USER --quiet < /dumps/%s', $localDecompressedFile); + exec($command); + $this->logger->info(sprintf( + '%s Imported %s to postgres in %s seconds', + Emoji::accordion(), + $localDecompressedFile, + number_format(microtime(true) - $startImport, 3) + )); + + // Cleanup + $this->cleanup([$localDecompressedFile]); } } diff --git a/syncer/Sync.php b/syncer/Sync.php index 49f1e2c..6b0ef48 100644 --- a/syncer/Sync.php +++ b/syncer/Sync.php @@ -7,6 +7,8 @@ use Garden\Cli\Args; use Garden\Cli\Cli; use Monolog\Handler\StreamHandler; use Monolog\Logger; +use S3DB\Sync\Filesystems\LocalFilesystem; +use S3DB\Sync\Filesystems\StorageFilesystem; use Spatie\Emoji\Emoji; class Sync @@ -15,6 +17,8 @@ class Sync protected Cli $cli; protected Args $args; protected AbstractSyncer $syncer; + protected StorageFilesystem $storageFilesystem; + protected LocalFilesystem $localFilesystem; public function __construct( ) { @@ -35,11 +39,16 @@ class Sync $stdout->setFormatter(new ColoredLineFormatter(null, "%level_name%: %message% \n")); $this->logger->pushHandler($stdout); - if ($this->args->hasOpt('postgres')) { - $this->logger->debug(sprintf('%s Starting in postgres mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE)); - $this->syncer = new PostgresAbstractSyncer($this->logger); + $this->storageFilesystem = new StorageFilesystem(); + $this->localFilesystem = new LocalFilesystem(); + + if ($this->args->hasOpt('postgres') || isset($environment['PG_VERSION'])) { + // Postgres mode is enabled if --postgres is set, or PG_VERSION envvar is set, + // which it is when we're built ontop of the postgres docker container + $this->logger->debug(sprintf('%s Starting in postgres mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE)); + $this->syncer = new PostgresAbstractSyncer($this->logger, $this->storageFilesystem, $this->localFilesystem); } elseif ($this->args->hasOpt('mysql')) { - $this->logger->debug(sprintf('%s Starting in mysql mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE)); + $this->logger->debug(sprintf('%s Starting in mysql mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE)); exit('Not implemented yet'); } else { @@ -52,10 +61,10 @@ class Sync public function run(): void { if ($this->args->hasOpt('push')) { - $this->logger->debug(sprintf('%s Running push', Emoji::upArrow())); + $this->logger->debug(sprintf('%s Running push', Emoji::upArrow())); $this->syncer->push(); } elseif ($this->args->hasOpt('pull')) { - $this->logger->debug(sprintf('%s Running pull', Emoji::downArrow())); + $this->logger->debug(sprintf('%s Running pull', Emoji::downArrow())); $this->syncer->pull(); } else { $this->logger->critical(sprintf('%s Must be run in either --push or --pull mode!', Emoji::CHARACTER_NERD_FACE));