Docker-Swarm-Loadbalancer/bouncer/bouncer

#!/usr/bin/env php
<?php
require_once 'vendor/autoload.php';

use AdamBrett\ShellWrapper\Command\Builder as CommandBuilder;
use AdamBrett\ShellWrapper\Runners\Exec;
use Aws\S3\S3Client;
use Bramus\Monolog\Formatter\ColoredLineFormatter;
use GuzzleHttp\Client as Guzzle;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Exception\ServerException;
use League\Flysystem\AwsS3V3\AwsS3V3Adapter;
use League\Flysystem\FileAttributes;
use League\Flysystem\Filesystem;
use League\Flysystem\Local\LocalFilesystemAdapter;
use Monolog\Handler\StreamHandler;
use Monolog\Level;
use Monolog\Logger;
use Spatie\Emoji\Emoji;
use Symfony\Component\Yaml\Yaml;
use Twig\Environment;
use Twig\Loader\FilesystemLoader;

class BouncerTarget
{
    private string $id;
    private array $domains;
    private string $endpointHostnameOrIp;
    private ?int $port        = null;
    private bool $letsEncrypt = false;
    private string $targetPath;
    private bool $allowNonSSL           = true;
    private bool $useTemporaryCert      = true;
    private bool $useGlobalCert         = false;
    private bool $allowWebsocketSupport = true;
    private bool $allowLargePayloads    = false;
    private ?int $proxyTimeoutSeconds   = null;
    private ?string $username           = null;
    private ?string $password           = null;

    public function __construct(
        private Logger $logger
    ) {
    }

    public function __toArray()
    {
        return [
            'id'                    => $this->getId(),
            'name'                  => $this->getName(),
            'domains'               => $this->getDomains(),
            'letsEncrypt'           => $this->isLetsEncrypt(),
            'targetPath'            => $this->getTargetPath(),
            'useTemporaryCert'      => $this->isUseTemporaryCert(),
            'useGlobalCert'         => $this->isUseGlobalCert(),
            'allowNonSSL'           => $this->isAllowNonSSL(),
            'allowWebsocketSupport' => $this->isAllowWebsocketSupport(),
            'allowLargePayloads'    => $this->isAllowLargePayloads(),
            'proxyTimeoutSeconds'   => $this->getProxyTimeoutSeconds(),
            'hasAuth'               => $this->hasAuth(),
            'authFile'              => $this->getAuthFileName(),
        ];
    }

    public function getUsername(): ?string
    {
        return $this->username;
    }

    /**
     * @param string
     */
    public function setUsername(string $username): BouncerTarget
    {
        $this->username = $username;

        return $this;
    }

    public function getPassword(): ?string
    {
        return $this->password;
    }

    public function setPassword(string $password): BouncerTarget
    {
        $this->password = $password;

        return $this;
    }

    public function setAuth(string $username, string $password): BouncerTarget
    {
        return $this->setUsername($username)->setPassword($password);
    }

    public function hasAuth(): bool
    {
        return $this->username != null && $this->password != null;
    }

    public function getFileName(): string
    {
        return "{$this->getName()}.conf";
    }

    public function getAuthFileName(): string
    {
        return "{$this->getName()}.secret";
    }

    public function getAuthFileData(): string
    {
        $output = shell_exec(sprintf('htpasswd -nibB -C10 %s %s', $this->getUsername(), $this->getPassword()));

        return trim($output) . "\n";
    }

    public function getProxyTimeoutSeconds(): ?int
    {
        return $this->proxyTimeoutSeconds;
    }

    public function setProxyTimeoutSeconds(?int $proxyTimeoutSeconds): BouncerTarget
    {
        $this->proxyTimeoutSeconds = $proxyTimeoutSeconds;

        return $this;
    }

    public function isUseTemporaryCert(): bool
    {
        return $this->useTemporaryCert;
    }

    public function setUseTemporaryCert(bool $useTemporaryCert): BouncerTarget
    {
        $this->useTemporaryCert = $useTemporaryCert;

        return $this;
    }

    public function isUseGlobalCert(): bool
    {
        return $this->useGlobalCert;
    }

    public function setUseGlobalCert(bool $useGlobalCert): BouncerTarget
    {
        $this->useGlobalCert = $useGlobalCert;

        // Global cert overrides temporary certs.
        if ($useGlobalCert) {
            $this->setUseTemporaryCert(false);
        }

        return $this;
    }

    public function isAllowWebsocketSupport(): bool
    {
        return $this->allowWebsocketSupport;
    }

    public function setAllowWebsocketSupport(bool $allowWebsocketSupport): BouncerTarget
    {
        $this->allowWebsocketSupport = $allowWebsocketSupport;

        return $this;
    }

    public function isAllowLargePayloads(): bool
    {
        return $this->allowLargePayloads;
    }

    public function setAllowLargePayloads(bool $allowLargePayloads): BouncerTarget
    {
        $this->allowLargePayloads = $allowLargePayloads;

        return $this;
    }

    public function getId(): string
    {
        return $this->id;
    }

    public function setId(string $id): BouncerTarget
    {
        $this->id = $id;

        return $this;
    }

    /**
     * @return string
     */
    public function getDomains(): array
    {
        return $this->domains;
    }

    /**
     * @param string $domains
     */
    public function setDomains(array $domains): BouncerTarget
    {
        $this->domains = $domains;

        return $this;
    }

    public function isLetsEncrypt(): bool
    {
        return $this->letsEncrypt;
    }

    public function setLetsEncrypt(bool $letsEncrypt): BouncerTarget
    {
        $this->letsEncrypt = $letsEncrypt;

        return $this;
    }

    public function getTargetPath(): string
    {
        return $this->targetPath;
    }

    public function setTargetPath(string $targetPath): BouncerTarget
    {
        $this->targetPath = $targetPath;

        return $this;
    }

    public function getEndpointHostnameOrIp(): string
    {
        return $this->endpointHostnameOrIp;
    }

    public function setEndpointHostnameOrIp(string $endpointHostnameOrIp): BouncerTarget
    {
        $this->endpointHostnameOrIp = $endpointHostnameOrIp;

        return $this;
    }

    public function getPort(): ?int
    {
        return $this->port;
    }

    public function isPortSet(): bool
    {
        return $this->port !== null;
    }

    public function setPort(int $port): BouncerTarget
    {
        $this->port = $port;

        return $this;
    }

    public function getName()
    {
        return reset($this->domains);
    }

    public function isAllowNonSSL(): bool
    {
        return $this->allowNonSSL;
    }

    public function setAllowNonSSL(bool $allowNonSSL): BouncerTarget
    {
        $this->allowNonSSL = $allowNonSSL;

        return $this;
    }

    public function isEndpointValid(): bool
    {
        // Is it just an IP?
        if (filter_var($this->getEndpointHostnameOrIp(), FILTER_VALIDATE_IP)) {
            // $this->logger->debug(sprintf('%s isEndpointValid: %s is a normal IP', Emoji::magnifyingGlassTiltedRight(), $this->getEndpointHostnameOrIp()));

            return true;
        }

        // Is it a Hostname that resolves?
        $resolved = gethostbyname($this->getEndpointHostnameOrIp());
        if (filter_var($resolved, FILTER_VALIDATE_IP)) {
            // $this->logger->debug(sprintf('%s isEndpointValid: %s is a hostname that resolves to a normal IP %s', Emoji::magnifyingGlassTiltedRight(), $this->getEndpointHostnameOrIp(), $resolved));

            return true;
        }

        $this->logger->warning(sprintf('%s isEndpointValid: %s is a hostname that does not resolve', Emoji::magnifyingGlassTiltedRight(), $this->getEndpointHostnameOrIp()));

        return false;
    }

    public function getPresentationDomain(): string
    {
        return sprintf(
            '%s://%s%s',
            $this->isAllowNonSSL() ? 'http' : 'https',
            $this->getUsername() && $this->getPassword() ?
                    sprintf('%s:%s@', $this->getUsername(), $this->getPassword()) :
                    '',
            $this->getName()
        );
    }
}

class Bouncer
{
    private array $environment;
    private Guzzle $client;
    private FilesystemLoader $loader;
    private Environment $twig;
    private Filesystem $configFilesystem;
    private Filesystem $certificateStoreLocal;
    private ?Filesystem $certificateStoreRemote = null;
    private Filesystem $providedCertificateStore;
    private Logger $logger;
    private array $previousContainerState = [];
    private array $previousSwarmState     = [];
    private array $fileHashes;
    private bool $swarmMode                        = false;
    private bool $useGlobalCert                    = false;
    private int $forcedUpdateIntervalSeconds       = 0;
    private ?int $lastUpdateEpoch                  = null;
    private int $maximumNginxConfigCreationNotices = 15;

    public function __construct()
    {
        $this->environment = array_merge($_ENV, $_SERVER);
        ksort($this->environment);

        $this->logger = new Monolog\Logger('bouncer');
        $this->logger->pushHandler(new StreamHandler('/var/log/bouncer.log', Level::Debug));
        $stdout = new StreamHandler('php://stdout', Level::Debug);
        $stdout->setFormatter(new ColoredLineFormatter(
            format: "%level_name%: %message% \n",
            allowInlineLineBreaks: true,
            ignoreEmptyContextAndExtra: true,
        ));
        $this->logger->pushHandler($stdout);

        if (isset($this->environment['DOCKER_HOST'])) {
            $this->logger->info(sprintf('%s Connecting to %s', Emoji::electricPlug(), $this->environment['DOCKER_HOST']));
            $this->client = new Guzzle(['base_uri' => $this->environment['DOCKER_HOST']]);
        } else {
            $this->logger->info(sprintf('%s Connecting to /var/run/docker.sock', Emoji::electricPlug()));
            $this->client = new Guzzle(['base_uri' => 'http://localhost', 'curl' => [CURLOPT_UNIX_SOCKET_PATH => '/var/run/docker.sock']]);
        }

        $this->loader = new FilesystemLoader([
            __DIR__,
        ]);
        $this->twig = new Environment($this->loader);

        // Set up Filesystem for sites-enabled path
        $this->configFilesystem = new Filesystem(new LocalFilesystemAdapter('/etc/nginx/sites-enabled'));

        // Set up Local certificate store
        $this->certificateStoreLocal = new Filesystem(new LocalFilesystemAdapter('/etc/letsencrypt'));

        // Set up Local certificate store for certificates provided to us
        $this->providedCertificateStore = new Filesystem(new LocalFilesystemAdapter('/certs'));

        // Set up Remote certificate store, if configured
        if (isset($this->environment['BOUNCER_S3_BUCKET'])) {
            $this->certificateStoreRemote = new Filesystem(
                new AwsS3V3Adapter(
                    new S3Client([
                        'endpoint'                => $this->environment['BOUNCER_S3_ENDPOINT'],
                        'use_path_style_endpoint' => isset($this->environment['BOUNCER_S3_USE_PATH_STYLE_ENDPOINT']),
                        'credentials'             => [
                            'key'    => $this->environment['BOUNCER_S3_KEY_ID'],
                            'secret' => $this->environment['BOUNCER_S3_KEY_SECRET'],
                        ],
                        'region'  => $this->environment['BOUNCER_S3_REGION'] ?? 'us-east',
                        'version' => 'latest',
                    ]),
                    $this->environment['BOUNCER_S3_BUCKET'],
                    $this->environment['BOUNCER_S3_PREFIX'] ?? ''
                )
            );
        }

        // Allow defined global cert if set
        if (isset($this->environment['GLOBAL_CERT'], $this->environment['GLOBAL_CERT_KEY'])) {
            $this->setUseGlobalCert(true);
            $this->providedCertificateStore->write('global.crt', str_replace('\\n', "\n", trim($this->environment['GLOBAL_CERT'], '"')));
            $this->providedCertificateStore->write('global.key', str_replace('\\n', "\n", trim($this->environment['GLOBAL_CERT_KEY'], '"')));
            $this->logger->info(sprintf("%s GLOBAL_CERT was set, so we're going to use a defined certificate!", Emoji::globeShowingEuropeAfrica()));
        }

        // Determine forced update interval.
        if (isset($this->environment['BOUNCER_FORCED_UPDATE_INTERVAL_SECONDS']) && is_numeric($this->environment['BOUNCER_FORCED_UPDATE_INTERVAL_SECONDS'])) {
            $this->setForcedUpdateIntervalSeconds($this->environment['BOUNCER_FORCED_UPDATE_INTERVAL_SECONDS']);
        }
        if ($this->getForcedUpdateIntervalSeconds() > 0) {
            $this->logger->warning(sprintf('%s  Forced update interval is every %d seconds', Emoji::watch(), $this->getForcedUpdateIntervalSeconds()));
        } else {
            $this->logger->info(sprintf('%s  Forced update interval is disabled', Emoji::watch()));
        }

        // Determine maximum notices for nginx config creation.
        if (isset($this->environment['BOUNCER_MAXIMUM_NGINX_CONFIG_CREATION_NOTICES']) && is_numeric($this->environment['BOUNCER_MAXIMUM_NGINX_CONFIG_CREATION_NOTICES'])) {
            $originalMaximumNginxConfigCreationNotices = $this->getMaximumNginxConfigCreationNotices();
            $this->setMaximumNginxConfigCreationNotices($this->environment['BOUNCER_MAXIMUM_NGINX_CONFIG_CREATION_NOTICES']);
            $this->logger->warning(sprintf('%s  Maximum Nginx config creation notices has been over-ridden: %d => %d', Emoji::upsideDownFace(), $originalMaximumNginxConfigCreationNotices, $this->getMaximumNginxConfigCreationNotices()));
        }
    }

    public function getMaximumNginxConfigCreationNotices(): int
    {
        return $this->maximumNginxConfigCreationNotices;
    }

    public function setMaximumNginxConfigCreationNotices(int $maximumNginxConfigCreationNotices): Bouncer
    {
        $this->maximumNginxConfigCreationNotices = $maximumNginxConfigCreationNotices;

        return $this;
    }

    public function isSwarmMode(): bool
    {
        return $this->swarmMode;
    }

    public function setSwarmMode(bool $swarmMode): Bouncer
    {
        $this->swarmMode = $swarmMode;

        return $this;
    }

    public function isUseGlobalCert(): bool
    {
        return $this->useGlobalCert;
    }

    public function setUseGlobalCert(bool $useGlobalCert): Bouncer
    {
        $this->useGlobalCert = $useGlobalCert;

        return $this;
    }

    public function getForcedUpdateIntervalSeconds(): int
    {
        return $this->forcedUpdateIntervalSeconds;
    }

    public function setForcedUpdateIntervalSeconds(int $forcedUpdateIntervalSeconds): Bouncer
    {
        $this->forcedUpdateIntervalSeconds = $forcedUpdateIntervalSeconds;

        return $this;
    }

    /**
     * @return BouncerTarget[]
     *
     * @throws \GuzzleHttp\Exception\GuzzleException
     */
    public function findContainersContainerMode(): array
    {
        $bouncerTargets = [];

        $containers = json_decode($this->client->request('GET', 'containers/json')->getBody()->getContents(), true);
        foreach ($containers as $container) {
            $envs    = [];
            $inspect = json_decode($this->client->request('GET', "containers/{$container['Id']}/json")->getBody()->getContents(), true);
            if (isset($inspect['Config']['Env'])) {
                foreach ($inspect['Config']['Env'] as $environmentItem) {
                    if (stripos($environmentItem, '=') !== false) {
                        [$envKey, $envVal] = explode('=', $environmentItem, 2);
                        $envs[$envKey]     = $envVal;
                    } else {
                        $envs[$environmentItem] = true;
                    }
                }
            }
            if (isset($envs['BOUNCER_DOMAIN'])) {
                $bouncerTarget = (new BouncerTarget($this->logger))
                    ->setId($inspect['Id'])
                ;
                $bouncerTarget = $this->parseContainerEnvironmentVariables($envs, $bouncerTarget);

                if (isset($inspect['NetworkSettings']['IPAddress']) && !empty($inspect['NetworkSettings']['IPAddress'])) {
                    // As per docker service
                    $bouncerTarget->setEndpointHostnameOrIp($inspect['NetworkSettings']['IPAddress']);
                } else {
                    // As per docker compose
                    $networks = array_values($inspect['NetworkSettings']['Networks']);
                    $bouncerTarget->setEndpointHostnameOrIp($networks[0]['IPAddress']);
                }

                $bouncerTarget->setTargetPath(sprintf('http://%s:%d/', $bouncerTarget->getEndpointHostnameOrIp(), $bouncerTarget->getPort() >= 0 ? $bouncerTarget->getPort() : 80));

                $bouncerTarget->setUseGlobalCert($this->isUseGlobalCert());

                $valid = $bouncerTarget->isEndpointValid();
                // $this->logger->debug(sprintf(
                //    '%s Decided that %s has the endpoint %s and it %s.',
                //    Emoji::magnifyingGlassTiltedLeft(),
                //    $bouncerTarget->getName(),
                //    $bouncerTarget->getEndpointHostnameOrIp(),
                //    $valid ? 'is valid' : 'is not valid'
                // ));
                if ($valid) {
                    $bouncerTargets[] = $bouncerTarget;
                }
            }
        }

        return $bouncerTargets;
    }

    public function findContainersSwarmMode(): array
    {
        $bouncerTargets = [];
        $services       = json_decode($this->client->request('GET', 'services')->getBody()->getContents(), true);

        if (isset($services['message'])) {
            $this->logger->debug(sprintf('Something happened while interrogating services.. This node is not a swarm node, cannot have services: %s', $services['message']));
        } else {
            foreach ($services as $service) {
                $envs = [];
                if (
                    !isset($service['Spec'])
                    || !isset($service['Spec']['TaskTemplate'])
                    || !isset($service['Spec']['TaskTemplate']['ContainerSpec'])
                    || !isset($service['Spec']['TaskTemplate']['ContainerSpec']['Env'])
                ) {
                    continue;
                }
                foreach ($service['Spec']['TaskTemplate']['ContainerSpec']['Env'] as $env) {
                    [$eKey, $eVal] = explode('=', $env, 2);
                    $envs[$eKey]   = $eVal;
                }
                if (isset($envs['BOUNCER_DOMAIN'])) {
                    $bouncerTarget = (new BouncerTarget($this->logger))
                        ->setId($service['ID'])
                    ;
                    $bouncerTarget = $this->parseContainerEnvironmentVariables($envs, $bouncerTarget);

                    if ($bouncerTarget->isPortSet()) {
                        $bouncerTarget->setEndpointHostnameOrIp($service['Spec']['Name']);
                        // $this->logger->info(sprintf('Ports for %s has been explicitly set to %s:%d.', $bouncerTarget->getName(), $bouncerTarget->getEndpointHostnameOrIp(), $bouncerTarget->getPort()));
                    } elseif (isset($service['Endpoint']['Ports'])) {
                        $bouncerTarget->setEndpointHostnameOrIp('172.17.0.1');
                        $bouncerTarget->setPort($service['Endpoint']['Ports'][0]['PublishedPort']);
                    } else {
                        $this->logger->warning(sprintf('Ports block missing for %s.', $bouncerTarget->getName()));

                        continue;
                    }
                    $bouncerTarget->setTargetPath(sprintf('http://%s:%d/', $bouncerTarget->getEndpointHostnameOrIp(), $bouncerTarget->getPort()));

                    $bouncerTarget->setUseGlobalCert($this->isUseGlobalCert());

                    // $this->logger->debug(sprintf('Decided that %s has the target path %s', $bouncerTarget->getName(), $bouncerTarget->getTargetPath()));

                    if ($bouncerTarget->isEndpointValid()) {
                        $bouncerTargets[] = $bouncerTarget;
                    } else {
                        $this->logger->debug(sprintf(
                            '%s Decided that %s has the endpoint %s and it is not valid.',
                            Emoji::magnifyingGlassTiltedLeft(),
                            $bouncerTarget->getName(),
                            $bouncerTarget->getEndpointHostnameOrIp(),
                        ));
                    }
                }
            }
        }

        return $bouncerTargets;
    }

    public function run(): void
    {
        $gitHash = substr($this->environment['GIT_SHA'], 0, 7);
        $this->logger->info(sprintf('%s  Starting Bouncer git=%s...', Emoji::CHARACTER_TIMER_CLOCK, $gitHash));

        try {
            $this->stateHasChanged();
        } catch (ConnectException $connectException) {
            $this->logger->critical(sprintf('%s Could not connect to docker socket! Did you map it?', Emoji::CHARACTER_CRYING_CAT));

            exit;
        }
        while (true) {
            $this->runLoop();
        }
    }

    public function parseContainerEnvironmentVariables(array $envs, BouncerTarget $bouncerTarget): BouncerTarget
    {
        foreach ($envs as $eKey => $eVal) {
            switch ($eKey) {
                case 'BOUNCER_DOMAIN':
                    $domains = explode(',', $eVal);
                    array_walk($domains, function (&$domain, $key): void {
                        $domain = trim($domain);
                    });
                    $bouncerTarget->setDomains($domains);

                    break;

                case 'BOUNCER_AUTH':
                    [$username, $password] = explode(':', $eVal);
                    $bouncerTarget->setAuth($username, $password);

                    break;

                case 'BOUNCER_LETSENCRYPT':
                    $bouncerTarget->setLetsEncrypt(in_array(strtolower($eVal), ['yes', 'true'], true));

                    break;

                case 'BOUNCER_TARGET_PORT':
                    $bouncerTarget->setPort($eVal);

                    break;

                case 'BOUNCER_ALLOW_NON_SSL':
                    $bouncerTarget->setAllowNonSSL(in_array(strtolower($eVal), ['yes', 'true'], true));

                    break;

                case 'BOUNCER_ALLOW_WEBSOCKETS':
                    $bouncerTarget->setAllowWebsocketSupport(in_array(strtolower($eVal), ['yes', 'true'], true));

                    break;

                case 'BOUNCER_ALLOW_LARGE_PAYLOADS':
                    $bouncerTarget->setAllowLargePayloads(in_array(strtolower($eVal), ['yes', 'true'], true));

                    break;

                case 'BOUNCER_PROXY_TIMEOUT_SECONDS':
                    $bouncerTarget->setProxyTimeoutSeconds(is_numeric($eVal) ? $eVal : null);

                    break;
            }
        }

        return $bouncerTarget;
    }

    private function dockerGetContainers(): array
    {
        return json_decode($this->client->request('GET', 'containers/json')->getBody()->getContents(), true);
    }

    private function dockerGetContainer(string $id): array
    {
        return json_decode($this->client->request('GET', "containers/{$id}/json")->getBody()->getContents(), true);
    }

    private function dockerEnvFilter(?array $envs): array
    {
        if ($envs === null) {
            return [];
        }

        return array_filter(array_map(function ($env) {
            if (stripos($env, '=') !== false) {
                [$envKey, $envVal] = explode('=', $env, 2);

                if (strlen($envVal) > 65) {
                    return sprintf('%s=CRC32(%s)', $envKey, crc32($envVal));
                }

                return sprintf('%s=%s', $envKey, $envVal);
            }

            return $env;
        }, $envs));
    }

    /**
     * Returns true when something has changed.
     *
     * @throws \GuzzleHttp\Exception\GuzzleException
     */
    private function stateHasChanged(): bool
    {
        $isTainted = false;
        if ($this->lastUpdateEpoch === null) {
            $isTainted = true;
        } elseif ($this->forcedUpdateIntervalSeconds > 0 && $this->lastUpdateEpoch <= time() - $this->forcedUpdateIntervalSeconds) {
            $this->logger->warning(sprintf('%s  Forced update interval of %d seconds has been reached, forcing update.', Emoji::watch(), $this->forcedUpdateIntervalSeconds));
            $isTainted = true;
        } elseif ($this->previousContainerState === []) {
            $this->logger->warning(sprintf('%s  Initial state has not been set, forcing update.', Emoji::watch()));
            $isTainted = true;
        } elseif ($this->previousSwarmState === []) {
            $this->logger->warning(sprintf('%s  Initial swarm state has not been set, forcing update.', Emoji::watch()));
            $isTainted = true;
        }

        // Standard Containers
        $newContainerState = [];
        $containers        = $this->dockerGetContainers();
        foreach ($containers as $container) {
            $inspect                  = $this->dockerGetContainer($container['Id']);
            $name                     = ltrim($inspect['Name'], '/');
            $newContainerState[$name] = [
                'name'    => $name,
                'created' => $inspect['Created'],
                'image'   => $inspect['Image'],
                'status'  => $inspect['State']['Status'],
                'env'     => $this->dockerEnvFilter($inspect['Config']['Env']),
            ];
            if (is_array($newContainerState[$name]['env'])) {
                sort($newContainerState[$name]['env']);
            }
        }
        ksort($newContainerState);

        // Calculate Container State Hash
        $containerStateDiff = $this->diff($this->previousContainerState, $newContainerState);
        if (!$isTainted && !empty($containerStateDiff)) {
            $this->logger->warning(sprintf('%s  Container state has changed', Emoji::warning()));
            $this->logger->debug(sprintf("Changed state:\n%s", $containerStateDiff));
            $isTainted = true;
        }
        $this->previousContainerState = $newContainerState;

        // Swarm Services
        $newSwarmState = [];
        if ($this->isSwarmMode()) {
            $services = json_decode($this->client->request('GET', 'services')->getBody()->getContents(), true);
            if (isset($services['message'])) {
                $this->logger->warning(sprintf('Something happened while interrogating services.. This node is not a swarm node, cannot have services: %s', $services['message']));
            } else {
                foreach ($services as $service) {
                    $env                  = $service['Spec']['TaskTemplate']['ContainerSpec']['Env'] ?? [];
                    $name                 = $service['Spec']['Name'];
                    $newSwarmState[$name] = [
                        'id'           => $service['ID'],
                        'mode'         => isset($service['Spec']['Mode']['Replicated']) ?
                            sprintf('replicated:%d', $service['Spec']['Mode']['Replicated']['Replicas']) :
                            (isset($service['Spec']['Mode']['Global']) ? 'global' : 'none'),
                        'created'      => $service['CreatedAt'],
                        'image'        => $service['Spec']['TaskTemplate']['ContainerSpec']['Image'],
                        'versionIndex' => $service['Version']['Index'],
                        'updateStatus' => $service['UpdateStatus']['State'] ?? 'unknown',
                        'env'          => $this->dockerEnvFilter($env),
                    ];
                }
            }
        }
        ksort($newSwarmState);

        // Calculate Swarm State Hash, if applicable
        $swarmStateDiff = $this->diff($this->previousSwarmState, $newSwarmState);
        if ($this->isSwarmMode() && !$isTainted && !empty($swarmStateDiff)) {
            $this->logger->warning(sprintf('%s  Swarm state has changed', Emoji::warning()));
            $this->logger->debug(sprintf("Changed state:\n%s", $swarmStateDiff));
            $isTainted = true;
        }
        $this->previousSwarmState = $newSwarmState;

        return $isTainted;
    }

    private function diff($a, $b)
    {
        return (new Diff(
            explode(
                "\n",
                Yaml::dump(input: $a, inline: 5, indent: 2)
            ),
            explode(
                "\n",
                Yaml::dump(input: $b, inline: 5, indent: 2)
            )
        ))->render(new Diff_Renderer_Text_Unified());
    }

    private function runLoop(): void
    {
        if ($this->s3Enabled()) {
            $this->getCertificatesFromS3();
        }

        try {
            $determineSwarmMode = json_decode($this->client->request('GET', 'swarm')->getBody()->getContents(), true);
            $this->setSwarmMode(!isset($determineSwarmMode['message']));
        } catch (ServerException $exception) {
            $this->setSwarmMode(false);
        } catch (ConnectException $exception) {
            $this->logger->critical(sprintf('%s Unable to connect to docker socket!', Emoji::warning()));
            $this->logger->critical($exception->getMessage());

            exit(1);
        }

        $this->logger->info(sprintf('%s Swarm mode is %s.', Emoji::CHARACTER_HONEYBEE, $this->isSwarmMode() ? 'enabled' : 'disabled'));

        $targets = array_values(
            array_merge(
                $this->findContainersContainerMode(),
                $this->isSwarmMode() ? $this->findContainersSwarmMode() : []
            )
        );

        // Use some bs to sort the targets by domain from right to left.
        $sortedTargets = [];
        foreach ($targets as $target) {
            $sortedTargets[strrev($target->getName())] = $target;
        }
        ksort($sortedTargets);
        $targets = array_values($sortedTargets);

        // Wipe configs and rebuild
        $this->wipeNginxConfig();

        $this->logger->info(sprintf('%s Found %d services with BOUNCER_DOMAIN set', Emoji::CHARACTER_MAGNIFYING_GLASS_TILTED_LEFT, count($targets)));
        $this->generateNginxConfigs($targets);
        $this->generateLetsEncryptCerts($targets);
        if ($this->s3Enabled()) {
            $this->writeCertificatesToS3();
        }
        $this->waitUntilContainerChange();
    }

    private function waitUntilContainerChange(): void
    {
        while ($this->stateHasChanged() === false) {
            sleep(5);
        }
        $this->lastUpdateEpoch = time();
    }

    private function s3Enabled(): bool
    {
        return $this->certificateStoreRemote instanceof Filesystem;
    }

    private function getCertificatesFromS3(): void
    {
        $this->logger->info(sprintf('%s Downloading Certificates from S3', Emoji::CHARACTER_DOWN_ARROW));
        foreach ($this->certificateStoreRemote->listContents('/', true) as $file) {
            /** @var FileAttributes $file */
            if ($file->isFile()) {
                $localPath = "archive/{$file->path()}";
                if ($file->fileSize() == 0) {
                    $this->logger->warning(sprintf(' > Downloading %s to %s was skipped, because it was empty', $file->path(), $localPath));

                    continue;
                }
                $this->logger->debug(sprintf(' > Downloading %s to %s (%d bytes)', $file->path(), $localPath, $file->fileSize()));
                $this->certificateStoreLocal->writeStream($localPath, $this->certificateStoreRemote->readStream($file->path()));
                if ($this->certificateStoreLocal->fileSize($localPath) == $this->certificateStoreRemote->fileSize($file->path())) {
                    $this->logger->debug(sprintf('   > Filesize for %s matches %s on remote (%d bytes)', $localPath, $file->path(), $this->certificateStoreLocal->fileSize($localPath)));
                } else {
                    $this->logger->critical(sprintf('   > Filesize for %s DOES NOT MATCH %s on remote (%d != %d bytes)', $localPath, $file->path(), $this->certificateStoreLocal->fileSize($localPath), $this->certificateStoreRemote->fileSize($file->path())));
                }
                $this->fileHashes[$localPath] = sha1($this->certificateStoreLocal->read($localPath));
            }
        }

        // Copy certs into /live because certbot is a pain.
        foreach ($this->certificateStoreLocal->listContents('/archive', true) as $newLocalCert) {
            /** @var FileAttributes $newLocalCert */
            if ($newLocalCert->isFile() && pathinfo($newLocalCert->path(), PATHINFO_EXTENSION) == 'pem') {
                $livePath = str_replace('archive/', 'live/', $newLocalCert->path());
                // Stupid dirty hack bullshit reee
                for ($i = 1; $i <= 9; ++$i) {
                    $livePath = str_replace("{$i}.pem", '.pem', $livePath);
                }
                $this->logger->debug(sprintf(' > Mirroring %s to %s (%d bytes)', $newLocalCert->path(), $livePath, $newLocalCert->fileSize()));
                $this->certificateStoreLocal->writeStream($livePath, $this->certificateStoreLocal->readStream($newLocalCert->path()));
            }
        }
    }

    private function fileChanged(string $localPath)
    {
        if (!isset($this->fileHashes[$localPath])) {
            return true;
        }
        if (sha1($this->certificateStoreLocal->read($localPath)) != $this->fileHashes[$localPath]) {
            return true;
        }

        return false;
    }

    private function writeCertificatesToS3(): void
    {
        $this->logger->info(sprintf('%s  Uploading Certificates to S3', Emoji::CHARACTER_UP_ARROW));
        foreach ($this->certificateStoreLocal->listContents('/archive', true) as $file) {
            /** @var FileAttributes $file */
            if ($file->isFile()) {
                $remotePath = str_replace('archive/', '', $file->path());
                if ($file->fileSize() == 0) {
                    $this->logger->warning(sprintf(" > Skipping uploading {$file->path()}, file is garbage (empty)."));
                } elseif (!$this->certificateStoreRemote->fileExists($remotePath) || $this->fileChanged($file->path())) {
                    $this->logger->debug(sprintf(' > Uploading %s (%d bytes)', $file->path(), $file->fileSize()));
                    $this->certificateStoreRemote->write($remotePath, $this->certificateStoreLocal->read($file->path()));
                } else {
                    $this->logger->debug(sprintf(" > Skipping uploading {$file->path()}, file not changed."));
                }
            }
        }
    }

    /**
     * @var BouncerTarget[]
     */
    private function generateNginxConfigs(array $targets): self
    {
        // get the length of the longest name...
        $longestName = max(array_map(fn (BouncerTarget $target) => strlen($target->getPresentationDomain()), $targets));

        foreach ($targets as $target) {
            $this->generateNginxConfig($target);
            if (count($targets) <= $this->getMaximumNginxConfigCreationNotices()) {
                $this->logger->info(sprintf(
                    '%s  Created Nginx config for %s',
                    Emoji::pencil(),
                    str_pad(
                        $target->getPresentationDomain(),
                        $longestName,
                        ' ',
                        STR_PAD_LEFT
                    )
                ));
            }
        }
        if (count($targets) > $this->getMaximumNginxConfigCreationNotices()) {
            $this->logger->info(sprintf('%s  More than %d Nginx configs generated.. Too many to show them all!', Emoji::pencil(), $this->getMaximumNginxConfigCreationNotices()));
        }
        $this->logger->info(sprintf('%s  Created %d Nginx configs..', Emoji::pencil(), count($targets)));

        return $this;
    }

    private function generateNginxConfig(BouncerTarget $target): self
    {
        $configData = $this->twig->render('NginxTemplate.twig', $target->__toArray());
        $this->configFilesystem->write($target->getFileName(), $configData);
        if ($target->hasAuth()) {
            $this->configFilesystem->write($target->getAuthFileName(), $target->getAuthFileData());
        }

        return $this;
    }

    /**
     * @param BouncerTarget[] $targets
     *
     * @return $this
     */
    private function generateLetsEncryptCerts(array $targets): self
    {
        foreach ($targets as $target) {
            if (!$target->isLetsEncrypt()) {
                continue;
            }

            $testAgeFile = "/archive/{$target->getName()}/fullchain1.pem";
            if ($this->certificateStoreLocal->fileExists($testAgeFile)) {
                $dubious = false;
                if ($this->certificateStoreLocal->fileSize($testAgeFile) == 0) {
                    // File is empty, check its age instead.
                    $timeRemainingSeconds = $this->certificateStoreLocal->lastModified($testAgeFile) - time();
                    $dubious              = true;
                } else {
                    $ssl                  = openssl_x509_parse($this->certificateStoreLocal->read($testAgeFile));
                    $timeRemainingSeconds = $ssl['validTo_time_t'] - time();
                }
                if ($timeRemainingSeconds > 2592000) {
                    $this->logger->info(sprintf(
                        '%s Skipping %s, certificate is %s for %d days',
                        Emoji::CHARACTER_PARTYING_FACE,
                        $target->getName(),
                        $dubious ? 'dubiously good' : 'still good',
                        round($timeRemainingSeconds / 86400)
                    ));

                    $target->setUseTemporaryCert(false);
                    $this->generateNginxConfig($target);

                    continue;
                }
            }

            // Start running shell commands...
            $shell = new Exec();

            // Disable nginx tweaks
            $this->logger->debug('Moving nginx tweak file..');
            $disableNginxTweaksCommand = (new CommandBuilder('mv'))
                ->addSubCommand('/etc/nginx/conf.d/tweak.conf')
                ->addSubCommand('/etc/nginx/conf.d/tweak.disabled')
            ;
            $shell->run($disableNginxTweaksCommand);

            // Generate letsencrypt cert
            $command = new CommandBuilder('/usr/bin/certbot');
            $command->addSubCommand('certonly');
            $command->addArgument('nginx');
            if ($this->environment['BOUNCER_LETSENCRYPT_MODE'] != 'production') {
                $command->addArgument('test-cert');
            }
            $command->addFlag('d', implode(',', $target->getDomains()));
            $command->addFlag('n');
            $command->addFlag('m', $this->environment['BOUNCER_LETSENCRYPT_EMAIL']);
            $command->addArgument('agree-tos');
            $this->logger->info(sprintf('%s Generating letsencrypt for %s - %s', Emoji::CHARACTER_PENCIL, $target->getName(), $command->__toString()));
            $shell->run($command);

            if ($shell->getReturnValue() == 0) {
                $this->logger->info(sprintf('%s Generating successful', Emoji::CHARACTER_PARTY_POPPER));
            } else {
                $this->logger->critical(sprintf('%s Generating failed!', Emoji::CHARACTER_WARNING));
            }

            // Re-enable nginx tweaks
            $this->logger->debug('Moving nginx tweak file back..');
            $disableNginxTweaksCommand = (new CommandBuilder('mv'))
                ->addSubCommand('/etc/nginx/conf.d/tweak.disabled')
                ->addSubCommand('/etc/nginx/conf.d/tweak.conf')
            ;
            $shell->run($disableNginxTweaksCommand);

            $target->setUseTemporaryCert(false);
            $this->generateNginxConfig($target);
        }

        $this->restartNginx();

        return $this;
    }

    private function restartNginx(): void
    {
        $shell   = new Exec();
        $command = new CommandBuilder('/usr/sbin/nginx');
        $command->addFlag('s', 'reload');
        $this->logger->info(sprintf('%s  Restarting nginx', Emoji::CHARACTER_TIMER_CLOCK));
        $shell->run($command);
    }

    private function wipeNginxConfig(): void
    {
        $this->logger->debug('Purging existing config files ...');
        foreach ($this->configFilesystem->listContents('') as $file) {
            /** @var FileAttributes $file */
            if ($file->isFile() && $file->path() != 'default' && $file->path() != 'default-ssl') {
                $this->configFilesystem->delete($file->path());
                // $this->logger->debug(sprintf(' > %s', $file->path()));
            }
        }
        // $this->logger->debug('Purge complete!');
    }
}

(new Bouncer())->run();