Initial working version.

.gitignore

@@ -1,3 +1,4 @@
 .php-cs-fixer.cache
 .idea
 /vendor/
+.minio

.php-cs-fixer.php

@@ -1,8 +1,9 @@
 <?php
+
 $finder = PhpCsFixer\Finder::create();
 $finder->in(__DIR__);
 
-return (new PhpCsFixer\Config)
+return (new PhpCsFixer\Config())
     ->setRiskyAllowed(true)
     ->setHideProgress(false)
     ->setRules([
@@ -19,4 +20,4 @@ return (new PhpCsFixer\Config)
         'yoda_style' => false,
     ])
     ->setFinder($finder)
-    ;
+;

Dockerfile

@@ -36,12 +36,20 @@ RUN apk --no-cache --repository https://dl-cdn.alpinelinux.org/alpine/edge/main
         php81-fpm \
         php81-sodium \
         php81-tokenizer \
+        php81-fileinfo \
+        php81-simplexml \
         # Iconv Fix
         php81-pecl-apcu \
+        ncurses \
+        xz \
     && ln -s /usr/bin/php81 /usr/bin/php
 COPY start.sh /usr/local/bin/start.sh
 COPY postgres.runit /etc/service/postgres/run
+COPY sync-pull.runit /etc/service/sync-pull/run
+COPY sync-push.runit /etc/service/sync-push/run
+VOLUME /dumps
 WORKDIR /sync
 COPY . /sync
-RUN chmod +x /sync/sync
+ENV PATH="/sync:${PATH}"
+RUN chmod +x /sync/sync /etc/service/*/run
 CMD ["start.sh"]

composer.json

@@ -12,7 +12,10 @@
         "monolog/monolog": "^2.2",
         "bramus/monolog-colored-line-formatter": "~3.0",
         "adambrett/shell-wrapper": "dev-master",
-        "spatie/emoji": "^2.3"
+        "spatie/emoji": "^2.3",
+        "rych/bytesize": "^1.0",
+        "jimmiw/php-time-ago": "^3.2",
+        "matthewbaggett/wait-for-mysql": "^1.0"
     },
     "require-dev": {
         "friendsofphp/php-cs-fixer": "^3.0"

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "cb65968236e13f317b1a5db79f853c52",
+    "content-hash": "9e33df3b624d4c411ab342f127b432cd",
     "packages": [
         {
             "name": "adambrett/shell-wrapper",
@@ -108,16 +108,16 @@
         },
         {
             "name": "aws/aws-sdk-php",
-            "version": "3.233.6",
+            "version": "3.234.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/aws/aws-sdk-php.git",
-                "reference": "cd9019a38cee8cabfd42dc7692e54106c93c1e6a"
+                "reference": "d2113f1e5ec9f7f19de2472f5063333b39a55280"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/cd9019a38cee8cabfd42dc7692e54106c93c1e6a",
+                "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/d2113f1e5ec9f7f19de2472f5063333b39a55280",
-                "reference": "cd9019a38cee8cabfd42dc7692e54106c93c1e6a",
+                "reference": "d2113f1e5ec9f7f19de2472f5063333b39a55280",
                 "shasum": ""
             },
             "require": {
@@ -194,9 +194,9 @@
             "support": {
                 "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80",
                 "issues": "https://github.com/aws/aws-sdk-php/issues",
-                "source": "https://github.com/aws/aws-sdk-php/tree/3.233.6"
+                "source": "https://github.com/aws/aws-sdk-php/tree/3.234.0"
             },
-            "time": "2022-08-19T18:16:02+00:00"
+            "time": "2022-08-22T18:20:42+00:00"
         },
         {
             "name": "bramus/ansi-php",
@@ -611,6 +611,58 @@
             ],
             "time": "2022-06-20T21:43:11+00:00"
         },
+        {
+            "name": "jimmiw/php-time-ago",
+            "version": "3.2.3",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/jimmiw/php-time-ago.git",
+                "reference": "2e0da84d3bd35344f44582ea78c85c75d937c457"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/jimmiw/php-time-ago/zipball/2e0da84d3bd35344f44582ea78c85c75d937c457",
+                "reference": "2e0da84d3bd35344f44582ea78c85c75d937c457",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=7.0.0"
+            },
+            "require-dev": {
+                "phpmd/phpmd": "@stable",
+                "phpunit/phpunit": "^6",
+                "squizlabs/php_codesniffer": "^3.3"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Westsworld\\": "src/Westsworld/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Jimmi Westerberg",
+                    "homepage": "http://www.westsworld.dk",
+                    "role": "Developer"
+                }
+            ],
+            "description": "Simple module, that displays the date in a \"time ago\" format",
+            "homepage": "https://github.com/jimmiw/php-time-ago",
+            "keywords": [
+                "distance of time",
+                "time ago",
+                "time ago in words"
+            ],
+            "support": {
+                "issues": "https://github.com/jimmiw/php-time-ago/issues",
+                "source": "https://github.com/jimmiw/php-time-ago/tree/3.2.3"
+            },
+            "time": "2022-06-10T10:15:42+00:00"
+        },
         {
             "name": "kint-php/kint",
             "version": "3.3",
@@ -901,6 +953,41 @@
             ],
             "time": "2022-04-17T13:12:02+00:00"
         },
+        {
+            "name": "matthewbaggett/wait-for-mysql",
+            "version": "v1.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/matthewbaggett/wait-for-mysql.git",
+                "reference": "fa8786e752aa77aacf79a8e40df8b84eacde7770"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/matthewbaggett/wait-for-mysql/zipball/fa8786e752aa77aacf79a8e40df8b84eacde7770",
+                "reference": "fa8786e752aa77aacf79a8e40df8b84eacde7770",
+                "shasum": ""
+            },
+            "bin": [
+                "wait-for-mysql",
+                "wait-for-postgresql"
+            ],
+            "type": "library",
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "GPL-3.0-or-later"
+            ],
+            "authors": [
+                {
+                    "name": "Matthew Baggett",
+                    "email": "matthew@baggett.me"
+                }
+            ],
+            "support": {
+                "issues": "https://github.com/matthewbaggett/wait-for-mysql/issues",
+                "source": "https://github.com/matthewbaggett/wait-for-mysql/tree/v1.1"
+            },
+            "time": "2021-09-22T09:25:08+00:00"
+        },
         {
             "name": "monolog/monolog",
             "version": "2.8.0",
@@ -1318,6 +1405,55 @@
             },
             "time": "2019-03-08T08:55:37+00:00"
         },
+        {
+            "name": "rych/bytesize",
+            "version": "v1.0.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/rchouinard/bytesize.git",
+                "reference": "297e16ea047461b91e8d7eb90aa46aaa52917824"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/rchouinard/bytesize/zipball/297e16ea047461b91e8d7eb90aa46aaa52917824",
+                "reference": "297e16ea047461b91e8d7eb90aa46aaa52917824",
+                "shasum": ""
+            },
+            "require": {
+                "ext-bcmath": "*",
+                "php": ">=5.3.4"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "3.7.*"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Rych\\ByteSize\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Ryan Chouinard",
+                    "email": "rchouinard@gmail.com",
+                    "homepage": "http://ryanchouinard.com"
+                }
+            ],
+            "description": "Utility component for nicely formatted file sizes.",
+            "homepage": "https://github.com/rchouinard/bytesize",
+            "keywords": [
+                "filesize"
+            ],
+            "support": {
+                "issues": "https://github.com/rchouinard/bytesize/issues",
+                "source": "https://github.com/rchouinard/bytesize/tree/master"
+            },
+            "time": "2014-04-04T18:06:18+00:00"
+        },
         {
             "name": "spatie/emoji",
             "version": "2.3.1",

docker-compose.yml

@@ -2,11 +2,14 @@ version: "3.7"
 
 services:
   minio:
-    image: quay.io/minio/minio:RELEASE.2022-08-13T21-54-44Z
+    image: minio/minio
-    command: server --console-address ":9001" http://minio{1...4}/data{1...2}
+    command: server --console-address ":9001" /data
+    ports:
+      - "127.0.0.127:9000:9000"
+      - "127.0.0.127:9001:9001"
     expose:
-      - "9000"
+      - 9000
-      - "9001"
+      - 9001
     environment:
      MINIO_ROOT_USER: &s3_key minio
      MINIO_ROOT_PASSWORD: &s3_secret changeme
@@ -15,6 +18,8 @@ services:
       interval: 30s
       timeout: 20s
       retries: 3
+    volumes:
+      - ./.minio/data:/data
 
   postgres-14:
     image: benzine/postgres:14
@@ -29,7 +34,12 @@ services:
       S3_ENDPOINT: http://minio:9000/
       S3_API_KEY: *s3_key
       S3_API_SECRET: *s3_secret
+      S3_USE_PATH_STYLE_ENDPOINT: "yes"
+      S3_BUCKET: "s3db"
+      S3_PREFIX: "test/postgres/"
     ports:
       - "127.0.0.127:5432:5432"
     depends_on:
-      - minio
+      - minio
+    volumes:
+      - ./:/sync

postgres.runit

@@ -3,4 +3,4 @@
 echo "Running docker-entrypoint"
 /usr/local/bin/docker-entrypoint.sh postgres
 
-sleep 60
+sleep 60

sync

@@ -5,4 +5,4 @@ use S3DB\Sync\Sync;
 
 require_once 'vendor/autoload.php';
 
-(new Sync())->run();
+(new Sync())->run();

sync-pull.runit

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+sleep 10
+echo "Running sync-pull"
+vendor/bin/wait-for-postgres
+/sync/sync --pull
+
+sleep infinity

sync-push.runit

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+sleep 600
+/sync/sync --push

syncer/AbstractSyncer.php

@@ -2,12 +2,20 @@
 
 namespace S3DB\Sync;
 
+use League\Flysystem\FileAttributes;
 use Monolog\Logger;
+use Rych\ByteSize\ByteSize;
+use S3DB\Sync\Filesystems\LocalFilesystem;
+use S3DB\Sync\Filesystems\StorageFilesystem;
+use Spatie\Emoji\Emoji;
+use Westsworld\TimeAgo;
 
 abstract class AbstractSyncer
 {
     public function __construct(
-        protected Logger $logger
+        protected Logger $logger,
+        protected StorageFilesystem $storageFilesystem,
+        protected LocalFilesystem $localFilesystem
     ) {
     }
 
@@ -15,7 +23,116 @@ abstract class AbstractSyncer
 
     abstract public function pull();
 
-    public function uploadToS3(): void
+    protected function download(): string
     {
+        $filesInS3 = $this->storageFilesystem->listContents('/')->toArray();
+        usort($filesInS3, function (FileAttributes $a, FileAttributes $b) {
+            return $a->lastModified() < $b->lastModified();
+        });
+
+        /** @var FileAttributes $file */
+        foreach ($filesInS3 as $file) {
+            $this->logger->debug(sprintf(
+                '%s Found %s. It is %s and was created %s',
+                Emoji::magnifyingGlassTiltedLeft(),
+                $file->path(),
+                ByteSize::formatMetric(
+                    $file->fileSize()
+                ),
+                (new TimeAgo())->inWords((new \DateTime())->setTimestamp($file->lastModified()))
+            ));
+        }
+
+        // Choose which we're downloadin'
+        $latest = $filesInS3[0];
+        $this->logger->debug(sprintf(
+            '%s  Selecting %s... Downloading %s...',
+            Emoji::downArrow(),
+            $latest->path(),
+            ByteSize::formatMetric($latest->fileSize())
+        ));
+
+        $localDownloadedFile = basename($latest->path());
+        $this->localFilesystem->writeStream(
+            $localDownloadedFile,
+            $this->storageFilesystem->readStream(
+                $latest->path()
+            )
+        );
+
+        return $localDownloadedFile;
+    }
+    protected function upload(string $remoteStorageFile, string $localCompressedDumpFile): void
+    {
+        $startUpload = microtime(true);
+        $this->storageFilesystem->writeStream(
+            $remoteStorageFile,
+            $this->localFilesystem->readStream($localCompressedDumpFile)
+        );
+        $this->logger->debug(sprintf(
+            'Uploaded %s as %s to S3 in %s seconds',
+            $localCompressedDumpFile,
+            $remoteStorageFile,
+            number_format(microtime(true) - $startUpload, 3)
+        ));
+    }
+
+    protected function cleanup(array $files): void
+    {
+        $cumulativeBytes = 0;
+        foreach ($files as $file) {
+            $cumulativeBytes += $this->localFilesystem->fileSize($file);
+            $this->localFilesystem->delete($file);
+        }
+        $this->logger->debug(sprintf(
+            '%s  Cleanup: Deleted %d files, freed %s',
+            Emoji::wastebasket(),
+            count($files),
+            ByteSize::formatMetric($cumulativeBytes)
+        ));
+    }
+
+    protected function compress(string $file): string
+    {
+        $startCompression = microtime(true);
+        passthru(sprintf('xz -f -T0 -6 /dumps/%s', $file));
+        $compressedFile = "{$file}.xz";
+        $this->logger->debug(sprintf(
+            '%s Dump file was made, and is %s compressed in %s seconds',
+            Emoji::computerDisk(),
+            ByteSize::formatMetric(
+                $this->localFilesystem->fileSize($compressedFile)
+            ),
+            number_format(microtime(true) - $startCompression, 3)
+        ));
+
+        return $compressedFile;
+    }
+
+    protected function decompress(string $compressedFile): string
+    {
+        $startDecompression = microtime(true);
+        if (!substr($compressedFile, -3, 3) == '.xz') {
+            $this->logger->critical(sprintf(
+                '%s Compressed file %s does not end in .xz',
+                Emoji::explodingHead(),
+                $compressedFile
+            ));
+
+            exit;
+        }
+        $uncompressedFile = substr($compressedFile, 0, -3);
+        passthru(sprintf('xz -d -f /dumps/%s', $compressedFile));
+
+        $this->logger->debug(sprintf(
+            '%s Dump file %s was uncompressed from %s to %s in %s seconds',
+            Emoji::computerDisk(),
+            $uncompressedFile,
+            ByteSize::formatMetric($this->storageFilesystem->fileSize($compressedFile)),
+            ByteSize::formatMetric($this->localFilesystem->fileSize($uncompressedFile)),
+            number_format(microtime(true) - $startDecompression, 3)
+        ));
+
+        return $uncompressedFile;
     }
 }

syncer/Filesystems/LocalFilesystem.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace S3DB\Sync\Filesystems;
+
+use League\Flysystem\Filesystem;
+use League\Flysystem\Local\LocalFilesystemAdapter;
+
+class LocalFilesystem extends Filesystem
+{
+    public function __construct()
+    {
+        $environment = array_merge($_ENV, $_SERVER);
+        $localAdapter = new LocalFilesystemAdapter('/dumps/');
+        parent::__construct($localAdapter);
+    }
+}

syncer/Filesystems/StorageFilesystem.php

@@ -0,0 +1,30 @@
+<?php
+
+namespace S3DB\Sync\Filesystems;
+
+use Aws\S3\S3Client;
+use League\Flysystem\AwsS3V3\AwsS3V3Adapter;
+use League\Flysystem\Filesystem;
+
+class StorageFilesystem extends Filesystem
+{
+    public function __construct()
+    {
+        $environment = array_merge($_ENV, $_SERVER);
+        $s3Adapter = new AwsS3V3Adapter(
+            new S3Client([
+                'endpoint' => $environment['S3_ENDPOINT'],
+                'use_path_style_endpoint' => isset($environment['S3_USE_PATH_STYLE_ENDPOINT']),
+                'credentials' => [
+                    'key' => $environment['S3_API_KEY'],
+                    'secret' => $environment['S3_API_SECRET'],
+                ],
+                'region' => $environment['S3_REGION'] ?? 'us-east',
+                'version' => 'latest',
+            ]),
+            $environment['S3_BUCKET'],
+            $environment['S3_PREFIX'] ?? null
+        );
+        parent::__construct($s3Adapter);
+    }
+}

syncer/PostgresAbstractSyncer.php

@@ -2,13 +2,62 @@
 
 namespace S3DB\Sync;
 
+use Rych\ByteSize\ByteSize;
+use Spatie\Emoji\Emoji;
+
 class PostgresAbstractSyncer extends AbstractSyncer
 {
     public function push(): void
     {
+        // Dump file from Postgres
+        $dumpFile = 'dump.sql';
+        $command = sprintf('PG_PASSWORD=$POSTGRESS_PASSWORD pg_dump -U $POSTGRES_USER --clean --inserts > /dumps/%s', $dumpFile);
+        passthru($command);
+
+        // Verify the dump worked
+        if (!$this->localFilesystem->fileExists($dumpFile)) {
+            $this->logger->critical('Database dump failed');
+
+            exit;
+        }
+        $this->logger->debug(sprintf(
+            'Dump file was made, and is %s uncompressed',
+            ByteSize::formatMetric(
+                $this->localFilesystem->fileSize($dumpFile)
+            )
+        ));
+
+        // XZ compress dump
+        $compressedDumpFile = $this->compress($dumpFile);
+
+        // Upload
+        $storageFile = sprintf('s3db-%s.sql.xz', date('Ymd-His'));
+        $this->upload($storageFile, $compressedDumpFile);
+
+        // Cleanup
+        $this->cleanup([$compressedDumpFile]);
     }
 
     public function pull(): void
     {
+        // Download latest dumpfile
+        $localDownloadedFile = $this->download();
+
+        // Decompress
+        $localDecompressedFile = $this->decompress($localDownloadedFile);
+
+        // Push into postgres
+        $startImport = microtime(true);
+        $command = sprintf('PG_PASSWORD=$POSTGRESS_PASSWORD psql -U $POSTGRES_USER --quiet < /dumps/%s', $localDecompressedFile);
+        exec($command);
+        $this->logger->info(sprintf(
+            '%s Imported %s to postgres in %s seconds',
+            Emoji::accordion(),
+            $localDecompressedFile,
+            number_format(microtime(true) - $startImport, 3)
+        ));
+
+        // Cleanup
+        $this->cleanup([$localDecompressedFile]);
     }
 }

syncer/Sync.php

@@ -7,6 +7,8 @@ use Garden\Cli\Args;
 use Garden\Cli\Cli;
 use Monolog\Handler\StreamHandler;
 use Monolog\Logger;
+use S3DB\Sync\Filesystems\LocalFilesystem;
+use S3DB\Sync\Filesystems\StorageFilesystem;
 use Spatie\Emoji\Emoji;
 
 class Sync
@@ -15,6 +17,8 @@ class Sync
     protected Cli $cli;
     protected Args $args;
     protected AbstractSyncer $syncer;
+    protected StorageFilesystem $storageFilesystem;
+    protected LocalFilesystem $localFilesystem;
 
     public function __construct(
     ) {
@@ -35,11 +39,16 @@ class Sync
         $stdout->setFormatter(new ColoredLineFormatter(null, "%level_name%: %message% \n"));
         $this->logger->pushHandler($stdout);
 
-        if ($this->args->hasOpt('postgres')) {
+        $this->storageFilesystem = new StorageFilesystem();
-            $this->logger->debug(sprintf('%s  Starting in postgres mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE));
+        $this->localFilesystem = new LocalFilesystem();
-            $this->syncer = new PostgresAbstractSyncer($this->logger);
+
+        if ($this->args->hasOpt('postgres') || isset($environment['PG_VERSION'])) {
+            // Postgres mode is enabled if --postgres is set, or PG_VERSION envvar is set,
+            // which it is when we're built ontop of the postgres docker container
+            $this->logger->debug(sprintf('%s Starting in postgres mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE));
+            $this->syncer = new PostgresAbstractSyncer($this->logger, $this->storageFilesystem, $this->localFilesystem);
         } elseif ($this->args->hasOpt('mysql')) {
-            $this->logger->debug(sprintf('%s  Starting in mysql mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE));
+            $this->logger->debug(sprintf('%s Starting in mysql mode', Emoji::CHARACTER_HOURGLASS_NOT_DONE));
 
             exit('Not implemented yet');
         } else {
@@ -52,10 +61,10 @@ class Sync
     public function run(): void
     {
         if ($this->args->hasOpt('push')) {
-            $this->logger->debug(sprintf('%s Running push', Emoji::upArrow()));
+            $this->logger->debug(sprintf('%s  Running push', Emoji::upArrow()));
             $this->syncer->push();
         } elseif ($this->args->hasOpt('pull')) {
-            $this->logger->debug(sprintf('%s Running pull', Emoji::downArrow()));
+            $this->logger->debug(sprintf('%s  Running pull', Emoji::downArrow()));
             $this->syncer->pull();
         } else {
             $this->logger->critical(sprintf('%s Must be run in either --push or --pull mode!', Emoji::CHARACTER_NERD_FACE));