diff --git a/Dockerfile.mariadb b/Dockerfile.mariadb
index f42bd93..e1b90d0 100644
--- a/Dockerfile.mariadb
+++ b/Dockerfile.mariadb
@@ -47,6 +47,7 @@ COPY start.sh /usr/local/bin/start.sh
COPY mysql.runit /etc/service/mysql/run
COPY sync-pull.runit /etc/service/sync-pull/run
COPY sync-push.runit /etc/service/sync-push/run
+COPY sync-prune.runit /etc/service/sync-prune/run
VOLUME /dumps
WORKDIR /sync
COPY composer.* /sync/
diff --git a/Dockerfile.postgres b/Dockerfile.postgres
index 3ad1d4d..335d342 100644
--- a/Dockerfile.postgres
+++ b/Dockerfile.postgres
@@ -51,6 +51,7 @@ COPY start.sh /usr/local/bin/start.sh
COPY postgres.runit /etc/service/postgres/run
COPY sync-pull.runit /etc/service/sync-pull/run
COPY sync-push.runit /etc/service/sync-push/run
+COPY sync-prune.runit /etc/service/sync-prune/run
VOLUME /dumps
WORKDIR /sync
COPY composer.* /sync/
diff --git a/TODO.md b/TODO.md
index 35893ab..a6e346a 100644
--- a/TODO.md
+++ b/TODO.md
@@ -1,5 +1,5 @@
All:
- * S3 pruner
+
PostgreSQL:
MariaDB:
diff --git a/composer.json b/composer.json
index c11247d..152d8c6 100644
--- a/composer.json
+++ b/composer.json
@@ -3,19 +3,20 @@
"type": "project",
"require": {
"php": ">8.1",
- "ext-json": "*",
"ext-curl": "*",
- "kint-php/kint": "^3",
- "league/flysystem-aws-s3-v3": "^3.2",
- "league/flysystem": "^3.2",
- "vanilla/garden-cli": "~2.0",
- "monolog/monolog": "^2.2",
- "bramus/monolog-colored-line-formatter": "~3.0",
+ "ext-json": "*",
"adambrett/shell-wrapper": "dev-master",
- "spatie/emoji": "^2.3",
- "rych/bytesize": "^1.0",
+ "bramus/monolog-colored-line-formatter": "~3.0",
"jimmiw/php-time-ago": "^3.2",
- "matthewbaggett/wait-for-mysql": "dev-main"
+ "kint-php/kint": "^3",
+ "league/flysystem": "^3.2",
+ "league/flysystem-aws-s3-v3": "^3.2",
+ "matthewbaggett/wait-for-mysql": "dev-main",
+ "monolog/monolog": "^2.2",
+ "nesbot/carbon": "^2.62",
+ "rych/bytesize": "^1.0",
+ "spatie/emoji": "^2.3",
+ "vanilla/garden-cli": "~2.0"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "^3.0"
@@ -33,6 +34,7 @@
}
],
"config": {
+ "sort-packages": true,
"preferred-install": {
"*": "dist"
}
diff --git a/composer.lock b/composer.lock
index 28ba774..2a294eb 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
- "content-hash": "519bdadff1be790c998c5adf50f9384c",
+ "content-hash": "b25a5a290f0e19df90438e3dd3fb749a",
"packages": [
{
"name": "adambrett/shell-wrapper",
@@ -1161,6 +1161,108 @@
},
"time": "2021-06-14T00:11:39+00:00"
},
+ {
+ "name": "nesbot/carbon",
+ "version": "2.62.0",
+ "source": {
+ "type": "git",
+ "url": "https://github.com/briannesbitt/Carbon.git",
+ "reference": "7507aec3d626797ce2123cf6c6556683be22b5f8"
+ },
+ "dist": {
+ "type": "zip",
+ "url": "https://api.github.com/repos/briannesbitt/Carbon/zipball/7507aec3d626797ce2123cf6c6556683be22b5f8",
+ "reference": "7507aec3d626797ce2123cf6c6556683be22b5f8",
+ "shasum": ""
+ },
+ "require": {
+ "ext-json": "*",
+ "php": "^7.1.8 || ^8.0",
+ "symfony/polyfill-mbstring": "^1.0",
+ "symfony/polyfill-php80": "^1.16",
+ "symfony/translation": "^3.4 || ^4.0 || ^5.0 || ^6.0"
+ },
+ "require-dev": {
+ "doctrine/dbal": "^2.0 || ^3.0",
+ "doctrine/orm": "^2.7",
+ "friendsofphp/php-cs-fixer": "^3.0",
+ "kylekatarnls/multi-tester": "^2.0",
+ "ondrejmirtes/better-reflection": "*",
+ "phpmd/phpmd": "^2.9",
+ "phpstan/extension-installer": "^1.0",
+ "phpstan/phpstan": "^0.12.99 || ^1.7.14",
+ "phpunit/php-file-iterator": "^2.0.5 || ^3.0.6",
+ "phpunit/phpunit": "^7.5.20 || ^8.5.26 || ^9.5.20",
+ "squizlabs/php_codesniffer": "^3.4"
+ },
+ "bin": [
+ "bin/carbon"
+ ],
+ "type": "library",
+ "extra": {
+ "branch-alias": {
+ "dev-3.x": "3.x-dev",
+ "dev-master": "2.x-dev"
+ },
+ "laravel": {
+ "providers": [
+ "Carbon\\Laravel\\ServiceProvider"
+ ]
+ },
+ "phpstan": {
+ "includes": [
+ "extension.neon"
+ ]
+ }
+ },
+ "autoload": {
+ "psr-4": {
+ "Carbon\\": "src/Carbon/"
+ }
+ },
+ "notification-url": "https://packagist.org/downloads/",
+ "license": [
+ "MIT"
+ ],
+ "authors": [
+ {
+ "name": "Brian Nesbitt",
+ "email": "brian@nesbot.com",
+ "homepage": "https://markido.com"
+ },
+ {
+ "name": "kylekatarnls",
+ "homepage": "https://github.com/kylekatarnls"
+ }
+ ],
+ "description": "An API extension for DateTime that supports 281 different languages.",
+ "homepage": "https://carbon.nesbot.com",
+ "keywords": [
+ "date",
+ "datetime",
+ "time"
+ ],
+ "support": {
+ "docs": "https://carbon.nesbot.com/docs",
+ "issues": "https://github.com/briannesbitt/Carbon/issues",
+ "source": "https://github.com/briannesbitt/Carbon"
+ },
+ "funding": [
+ {
+ "url": "https://github.com/sponsors/kylekatarnls",
+ "type": "github"
+ },
+ {
+ "url": "https://opencollective.com/Carbon#sponsor",
+ "type": "opencollective"
+ },
+ {
+ "url": "https://tidelift.com/subscription/pkg/packagist-nesbot-carbon?utm_source=packagist-nesbot-carbon&utm_medium=referral&utm_campaign=readme",
+ "type": "tidelift"
+ }
+ ],
+ "time": "2022-08-28T19:48:05+00:00"
+ },
{
"name": "psr/http-client",
"version": "1.0.1",
@@ -1680,6 +1782,266 @@
],
"time": "2022-05-24T11:49:31+00:00"
},
+ {
+ "name": "symfony/polyfill-php80",
+ "version": "v1.26.0",
+ "source": {
+ "type": "git",
+ "url": "https://github.com/symfony/polyfill-php80.git",
+ "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace"
+ },
+ "dist": {
+ "type": "zip",
+ "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/cfa0ae98841b9e461207c13ab093d76b0fa7bace",
+ "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace",
+ "shasum": ""
+ },
+ "require": {
+ "php": ">=7.1"
+ },
+ "type": "library",
+ "extra": {
+ "branch-alias": {
+ "dev-main": "1.26-dev"
+ },
+ "thanks": {
+ "name": "symfony/polyfill",
+ "url": "https://github.com/symfony/polyfill"
+ }
+ },
+ "autoload": {
+ "files": [
+ "bootstrap.php"
+ ],
+ "psr-4": {
+ "Symfony\\Polyfill\\Php80\\": ""
+ },
+ "classmap": [
+ "Resources/stubs"
+ ]
+ },
+ "notification-url": "https://packagist.org/downloads/",
+ "license": [
+ "MIT"
+ ],
+ "authors": [
+ {
+ "name": "Ion Bazan",
+ "email": "ion.bazan@gmail.com"
+ },
+ {
+ "name": "Nicolas Grekas",
+ "email": "p@tchwork.com"
+ },
+ {
+ "name": "Symfony Community",
+ "homepage": "https://symfony.com/contributors"
+ }
+ ],
+ "description": "Symfony polyfill backporting some PHP 8.0+ features to lower PHP versions",
+ "homepage": "https://symfony.com",
+ "keywords": [
+ "compatibility",
+ "polyfill",
+ "portable",
+ "shim"
+ ],
+ "support": {
+ "source": "https://github.com/symfony/polyfill-php80/tree/v1.26.0"
+ },
+ "funding": [
+ {
+ "url": "https://symfony.com/sponsor",
+ "type": "custom"
+ },
+ {
+ "url": "https://github.com/fabpot",
+ "type": "github"
+ },
+ {
+ "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+ "type": "tidelift"
+ }
+ ],
+ "time": "2022-05-10T07:21:04+00:00"
+ },
+ {
+ "name": "symfony/translation",
+ "version": "v6.1.4",
+ "source": {
+ "type": "git",
+ "url": "https://github.com/symfony/translation.git",
+ "reference": "45d0f5bb8df7255651ca91c122fab604e776af03"
+ },
+ "dist": {
+ "type": "zip",
+ "url": "https://api.github.com/repos/symfony/translation/zipball/45d0f5bb8df7255651ca91c122fab604e776af03",
+ "reference": "45d0f5bb8df7255651ca91c122fab604e776af03",
+ "shasum": ""
+ },
+ "require": {
+ "php": ">=8.1",
+ "symfony/polyfill-mbstring": "~1.0",
+ "symfony/translation-contracts": "^2.3|^3.0"
+ },
+ "conflict": {
+ "symfony/config": "<5.4",
+ "symfony/console": "<5.4",
+ "symfony/dependency-injection": "<5.4",
+ "symfony/http-kernel": "<5.4",
+ "symfony/twig-bundle": "<5.4",
+ "symfony/yaml": "<5.4"
+ },
+ "provide": {
+ "symfony/translation-implementation": "2.3|3.0"
+ },
+ "require-dev": {
+ "psr/log": "^1|^2|^3",
+ "symfony/config": "^5.4|^6.0",
+ "symfony/console": "^5.4|^6.0",
+ "symfony/dependency-injection": "^5.4|^6.0",
+ "symfony/finder": "^5.4|^6.0",
+ "symfony/http-client-contracts": "^1.1|^2.0|^3.0",
+ "symfony/http-kernel": "^5.4|^6.0",
+ "symfony/intl": "^5.4|^6.0",
+ "symfony/polyfill-intl-icu": "^1.21",
+ "symfony/routing": "^5.4|^6.0",
+ "symfony/service-contracts": "^1.1.2|^2|^3",
+ "symfony/yaml": "^5.4|^6.0"
+ },
+ "suggest": {
+ "psr/log-implementation": "To use logging capability in translator",
+ "symfony/config": "",
+ "symfony/yaml": ""
+ },
+ "type": "library",
+ "autoload": {
+ "files": [
+ "Resources/functions.php"
+ ],
+ "psr-4": {
+ "Symfony\\Component\\Translation\\": ""
+ },
+ "exclude-from-classmap": [
+ "/Tests/"
+ ]
+ },
+ "notification-url": "https://packagist.org/downloads/",
+ "license": [
+ "MIT"
+ ],
+ "authors": [
+ {
+ "name": "Fabien Potencier",
+ "email": "fabien@symfony.com"
+ },
+ {
+ "name": "Symfony Community",
+ "homepage": "https://symfony.com/contributors"
+ }
+ ],
+ "description": "Provides tools to internationalize your application",
+ "homepage": "https://symfony.com",
+ "support": {
+ "source": "https://github.com/symfony/translation/tree/v6.1.4"
+ },
+ "funding": [
+ {
+ "url": "https://symfony.com/sponsor",
+ "type": "custom"
+ },
+ {
+ "url": "https://github.com/fabpot",
+ "type": "github"
+ },
+ {
+ "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+ "type": "tidelift"
+ }
+ ],
+ "time": "2022-08-02T16:17:38+00:00"
+ },
+ {
+ "name": "symfony/translation-contracts",
+ "version": "v3.1.1",
+ "source": {
+ "type": "git",
+ "url": "https://github.com/symfony/translation-contracts.git",
+ "reference": "606be0f48e05116baef052f7f3abdb345c8e02cc"
+ },
+ "dist": {
+ "type": "zip",
+ "url": "https://api.github.com/repos/symfony/translation-contracts/zipball/606be0f48e05116baef052f7f3abdb345c8e02cc",
+ "reference": "606be0f48e05116baef052f7f3abdb345c8e02cc",
+ "shasum": ""
+ },
+ "require": {
+ "php": ">=8.1"
+ },
+ "suggest": {
+ "symfony/translation-implementation": ""
+ },
+ "type": "library",
+ "extra": {
+ "branch-alias": {
+ "dev-main": "3.1-dev"
+ },
+ "thanks": {
+ "name": "symfony/contracts",
+ "url": "https://github.com/symfony/contracts"
+ }
+ },
+ "autoload": {
+ "psr-4": {
+ "Symfony\\Contracts\\Translation\\": ""
+ },
+ "exclude-from-classmap": [
+ "/Test/"
+ ]
+ },
+ "notification-url": "https://packagist.org/downloads/",
+ "license": [
+ "MIT"
+ ],
+ "authors": [
+ {
+ "name": "Nicolas Grekas",
+ "email": "p@tchwork.com"
+ },
+ {
+ "name": "Symfony Community",
+ "homepage": "https://symfony.com/contributors"
+ }
+ ],
+ "description": "Generic abstractions related to translation",
+ "homepage": "https://symfony.com",
+ "keywords": [
+ "abstractions",
+ "contracts",
+ "decoupling",
+ "interfaces",
+ "interoperability",
+ "standards"
+ ],
+ "support": {
+ "source": "https://github.com/symfony/translation-contracts/tree/v3.1.1"
+ },
+ "funding": [
+ {
+ "url": "https://symfony.com/sponsor",
+ "type": "custom"
+ },
+ {
+ "url": "https://github.com/fabpot",
+ "type": "github"
+ },
+ {
+ "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+ "type": "tidelift"
+ }
+ ],
+ "time": "2022-06-27T17:24:16+00:00"
+ },
{
"name": "vanilla/garden-cli",
"version": "v2.2",
@@ -3102,89 +3464,6 @@
],
"time": "2022-05-24T11:49:31+00:00"
},
- {
- "name": "symfony/polyfill-php80",
- "version": "v1.26.0",
- "source": {
- "type": "git",
- "url": "https://github.com/symfony/polyfill-php80.git",
- "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace"
- },
- "dist": {
- "type": "zip",
- "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/cfa0ae98841b9e461207c13ab093d76b0fa7bace",
- "reference": "cfa0ae98841b9e461207c13ab093d76b0fa7bace",
- "shasum": ""
- },
- "require": {
- "php": ">=7.1"
- },
- "type": "library",
- "extra": {
- "branch-alias": {
- "dev-main": "1.26-dev"
- },
- "thanks": {
- "name": "symfony/polyfill",
- "url": "https://github.com/symfony/polyfill"
- }
- },
- "autoload": {
- "files": [
- "bootstrap.php"
- ],
- "psr-4": {
- "Symfony\\Polyfill\\Php80\\": ""
- },
- "classmap": [
- "Resources/stubs"
- ]
- },
- "notification-url": "https://packagist.org/downloads/",
- "license": [
- "MIT"
- ],
- "authors": [
- {
- "name": "Ion Bazan",
- "email": "ion.bazan@gmail.com"
- },
- {
- "name": "Nicolas Grekas",
- "email": "p@tchwork.com"
- },
- {
- "name": "Symfony Community",
- "homepage": "https://symfony.com/contributors"
- }
- ],
- "description": "Symfony polyfill backporting some PHP 8.0+ features to lower PHP versions",
- "homepage": "https://symfony.com",
- "keywords": [
- "compatibility",
- "polyfill",
- "portable",
- "shim"
- ],
- "support": {
- "source": "https://github.com/symfony/polyfill-php80/tree/v1.26.0"
- },
- "funding": [
- {
- "url": "https://symfony.com/sponsor",
- "type": "custom"
- },
- {
- "url": "https://github.com/fabpot",
- "type": "github"
- },
- {
- "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
- "type": "tidelift"
- }
- ],
- "time": "2022-05-10T07:21:04+00:00"
- },
{
"name": "symfony/polyfill-php81",
"version": "v1.26.0",
@@ -3568,8 +3847,8 @@
"prefer-lowest": false,
"platform": {
"php": ">8.1",
- "ext-json": "*",
- "ext-curl": "*"
+ "ext-curl": "*",
+ "ext-json": "*"
},
"platform-dev": [],
"plugin-api-version": "2.3.0"
diff --git a/sync-prune.runit b/sync-prune.runit
new file mode 100644
index 0000000..884b26c
--- /dev/null
+++ b/sync-prune.runit
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+/sync/sync --prune
+sleep 84600
+
diff --git a/syncer/AbstractSyncer.php b/syncer/AbstractSyncer.php
index 534192a..f60e5b7 100644
--- a/syncer/AbstractSyncer.php
+++ b/syncer/AbstractSyncer.php
@@ -2,7 +2,9 @@
namespace S3DB\Sync;
+use Carbon\Carbon;
use League\Flysystem\FileAttributes;
+use League\Flysystem\FilesystemReader;
use Monolog\Logger;
use Rych\ByteSize\ByteSize;
use S3DB\Sync\Filesystems\LocalFilesystem;
@@ -181,4 +183,78 @@ abstract class AbstractSyncer
)
));
}
+
+ public function prune($dryRun = true) : void {
+ $timeAgo = new TimeAgo();
+ $buckets = [];
+
+ // Organise each file into buckets
+ $allFiles = $this->storageFilesystem->listContents(".", FilesystemReader::LIST_DEEP)->toArray();
+ foreach($allFiles as $file){
+ $date = (new Carbon())->setTimestamp($file['lastModified']);
+ $buckets[$timeAgo->inWords($date)][$date->format("Y-m-d H:i:s")] = $file;
+ ksort($buckets[$timeAgo->inWords($date)]);
+ }
+
+ // Sift each bucket to get the newest file...
+ $this->logger->debug(sprintf(
+ "%s Sifting %d buckets of %d items...",
+ Emoji::beachWithUmbrella(),
+ count($buckets),
+ count($allFiles)
+ ));
+ $siftedBuckets = [];
+ foreach($buckets as $bucketName => $bucketOptions){
+ $siftedBuckets[$bucketName] = reset($bucketOptions);
+ }
+
+ // Build a list to save...
+ $saveList = [];
+ foreach($siftedBuckets as $bucketName => $selectedFile){
+ /** @var FileAttributes $selectedFile */
+ $saveList[] = $selectedFile->path();
+ $this->logger->debug(sprintf(
+ "%s Saving %s from %s",
+ Emoji::smilingFaceWithHalo(),
+ $selectedFile->path(),
+ $timeAgo->inWords((new Carbon())->setTimestamp($selectedFile->lastModified()))
+ ));
+ }
+ // Build the culling list
+ $cullingList = [];
+ foreach($allFiles as $file){
+ /** @var FileAttributes $file */
+ if(!in_array($file->path(), $saveList)){
+ $cullingList[] = $file;
+ $this->logger->info(sprintf(
+ " %s Culling %s from %s",
+ Emoji::recyclingSymbol(),
+ $file->path(),
+ $timeAgo->inWords((new Carbon())->setTimestamp($file->lastModified()))
+ ));
+ }
+ }
+
+ $freedBytes= 0;
+ foreach($cullingList as $fileToCull){
+ /** @var FileAttributes $fileToCull */
+ $freedBytes += $this->storageFilesystem->fileSize($fileToCull->path());
+ $this->logger->debug(sprintf(
+ "%s Deleting %s saving %s.",
+ Emoji::fire(),
+ $fileToCull->path(),
+ ByteSize::formatMetric($fileToCull->fileSize())
+ ));
+ if(!$dryRun) {
+ $this->storageFilesystem->delete($fileToCull->path());
+ }
+ }
+
+ $this->logger->info(sprintf(
+ " %s Deleted %d files and saved %s disk space",
+ Emoji::trumpet(),
+ count($cullingList),
+ ByteSize::formatMetric($freedBytes)
+ ));
+ }
}
diff --git a/syncer/Sync.php b/syncer/Sync.php
index a9d74dc..21c9c74 100644
--- a/syncer/Sync.php
+++ b/syncer/Sync.php
@@ -30,6 +30,8 @@ class Sync
->opt('mysql', 'mysql mode')
->opt('push', 'push to s3')
->opt('pull', 'pull from s3')
+ ->opt('prune', 'comb and prune the s3 bucket backups to reduce storage mass')
+ ->opt('dry-run', 'do not actually delete things')
;
$this->args = $this->cli->parse($environment['argv'], true);
@@ -60,11 +62,14 @@ class Sync
public function run(): void
{
if ($this->args->hasOpt('push')) {
- $this->logger->debug(sprintf('%s Running push', Emoji::upArrow()));
+ $this->logger->info(sprintf(' %s Running push', Emoji::upArrow()));
$this->syncer->push();
} elseif ($this->args->hasOpt('pull')) {
- $this->logger->debug(sprintf('%s Running pull', Emoji::downArrow()));
+ $this->logger->info(sprintf(' %s Running pull', Emoji::downArrow()));
$this->syncer->pull();
+ } elseif ($this->args->hasOpt('prune')) {
+ $this->logger->info(sprintf(' %s Running pruner', Emoji::recyclingSymbol()));
+ $this->syncer->prune($this->args->hasOpt('dry-run'));
} else {
$this->logger->critical(sprintf('%s Must be run in either --push or --pull mode!', Emoji::CHARACTER_NERD_FACE));