Docker-Swarm-Loadbalancer/bouncer/bouncer
2022-05-06 02:28:50 +02:00

592 lines
20 KiB
PHP
Executable file

#!/usr/bin/env php
<?php
require_once 'vendor/autoload.php';
use AdamBrett\ShellWrapper\Command\Builder as CommandBuilder;
use AdamBrett\ShellWrapper\Runners\Exec;
use Aws\S3\S3Client;
use Bramus\Monolog\Formatter\ColoredLineFormatter;
use GuzzleHttp\Client as Guzzle;
use League\Flysystem\AwsS3V3\AwsS3V3Adapter;
use League\Flysystem\FileAttributes;
use League\Flysystem\Filesystem;
use League\Flysystem\Local\LocalFilesystemAdapter;
use Monolog\Handler\StreamHandler;
use Monolog\Logger;
use Spatie\Emoji\Emoji;
use Twig\Environment;
use Twig\Loader\FilesystemLoader;
class BouncerTarget
{
private string $id;
private array $domains;
private string $ip;
private int $port = 80;
private bool $letsEncrypt = false;
private string $targetPath;
private bool $allowNonSSL = false;
private bool $useTemporaryCert = true;
public function __toArray()
{
return [
'id' => $this->getId(),
'name' => $this->getName(),
'domains' => $this->getDomains(),
'letsEncrypt' => $this->isLetsEncrypt(),
'targetPath' => $this->getTargetPath(),
'useTemporaryCert' => $this->isUseTemporaryCert(),
'allowNonSSL' => $this->isAllowNonSSL(),
];
}
public function isUseTemporaryCert(): bool
{
return $this->useTemporaryCert;
}
public function setUseTemporaryCert(bool $useTemporaryCert): BouncerTarget
{
$this->useTemporaryCert = $useTemporaryCert;
return $this;
}
public function getId(): string
{
return $this->id;
}
public function setId(string $id): BouncerTarget
{
$this->id = $id;
return $this;
}
/**
* @return string
*/
public function getDomains(): array
{
return $this->domains;
}
/**
* @param string $domains
*/
public function setDomains(array $domains): BouncerTarget
{
$this->domains = $domains;
return $this;
}
public function isLetsEncrypt(): bool
{
return $this->letsEncrypt;
}
public function setLetsEncrypt(bool $letsEncrypt): BouncerTarget
{
$this->letsEncrypt = $letsEncrypt;
return $this;
}
public function getTargetPath(): string
{
return $this->targetPath;
}
public function setTargetPath(string $targetPath): BouncerTarget
{
$this->targetPath = $targetPath;
return $this;
}
public function getIp(): string
{
return $this->ip;
}
public function setIp(string $ip): BouncerTarget
{
$this->ip = $ip;
return $this;
}
public function getPort(): int
{
return $this->port;
}
public function setPort(int $port): BouncerTarget
{
$this->port = $port;
return $this;
}
public function getName()
{
return reset($this->domains);
}
public function isAllowNonSSL(): bool
{
return $this->allowNonSSL;
}
public function setAllowNonSSL(bool $allowNonSSL): BouncerTarget
{
$this->allowNonSSL = $allowNonSSL;
return $this;
}
}
class Bouncer
{
private array $environment;
private Guzzle $client;
private FilesystemLoader $loader;
private Environment $twig;
private Filesystem $configFilesystem;
private Filesystem $certificateStoreLocal;
private ?Filesystem $certificateStoreRemote = null;
private Logger $logger;
private string $instanceStateHash = '';
private array $fileHashes;
private bool $swarmMode = false;
/**
* @return bool
*/
public function isSwarmMode(): bool
{
return $this->swarmMode;
}
/**
* @param bool $swarmMode
* @return Bouncer
*/
public function setSwarmMode(bool $swarmMode): Bouncer
{
$this->swarmMode = $swarmMode;
return $this;
}
public function __construct()
{
$this->environment = array_merge($_ENV, $_SERVER);
ksort($this->environment);
$this->logger = new Monolog\Logger('bouncer');
$this->logger->pushHandler(new StreamHandler('/var/log/bouncer.log', Logger::DEBUG));
$stdout = new StreamHandler('php://stdout', Logger::DEBUG);
$stdout->setFormatter(new ColoredLineFormatter(null, "%level_name%: %message% \n"));
$this->logger->pushHandler($stdout);
$this->client = new Guzzle(
[
'base_uri' => 'http://localhost',
'curl' => [
CURLOPT_UNIX_SOCKET_PATH => '/var/run/docker.sock',
],
]
);
$this->loader = new FilesystemLoader([
__DIR__,
]);
$this->twig = new Environment($this->loader);
// Set up Filesystem for sites-enabled path
$this->configFilesystem = new Filesystem(new LocalFilesystemAdapter('/etc/nginx/sites-enabled'));
// Set up Local certificate store
$this->certificateStoreLocal = new Filesystem(new LocalFilesystemAdapter('/etc/letsencrypt'));
// Set up Remote certificate store, if configured
if (isset($this->environment['BOUNCER_S3_BUCKET'])) {
$this->certificateStoreRemote = new Filesystem(
new AwsS3V3Adapter(
new S3Client([
'endpoint' => $this->environment['BOUNCER_S3_ENDPOINT'],
'use_path_style_endpoint' => isset($this->environment['BOUNCER_S3_USE_PATH_STYLE_ENDPOINT']),
'credentials' => [
'key' => $this->environment['BOUNCER_S3_KEY_ID'],
'secret' => $this->environment['BOUNCER_S3_KEY_SECRET'],
],
'region' => $this->environment['BOUNCER_S3_REGION'] ?? 'us-east',
'version' => 'latest',
]),
$this->environment['BOUNCER_S3_BUCKET'],
$this->environment['BOUNCER_S3_PREFIX'] ?? ''
)
);
}
}
/**
* @throws \GuzzleHttp\Exception\GuzzleException
*
* @return BouncerTarget[]
*/
public function findContainersContainerMode(): array
{
$bouncerTargets = [];
$containers = json_decode($this->client->request('GET', 'containers/json')->getBody()->getContents(), true);
foreach ($containers as $container) {
$envs = [];
$inspect = json_decode($this->client->request('GET', "containers/{$container['Id']}/json")->getBody()->getContents(), true);
if (isset($inspect['Config']['Env'])) {
foreach ($inspect['Config']['Env'] as $environmentItem) {
if (stripos($environmentItem, '=') !== false) {
[$envKey, $envVal] = explode('=', $environmentItem, 2);
$envs[$envKey] = $envVal;
} else {
$envs[$envKey] = true;
}
}
}
if (isset($envs['BOUNCER_DOMAIN'])) {
$bouncerTarget = (new BouncerTarget())
->setId($inspect['Id'])
;
$bouncerTarget = $this->parseContainerEnvironmentVariables($envs, $bouncerTarget);
if (isset($inspect['NetworkSettings']['IPAddress']) && !empty($inspect['NetworkSettings']['IPAddress'])) {
// As per docker service
$bouncerTarget->setIp($inspect['NetworkSettings']['IPAddress']);
} else {
// As per docker compose
$networks = array_values($inspect['NetworkSettings']['Networks']);
$bouncerTarget->setIp($networks[0]['IPAddress']);
}
//$this->logger->debug(sprintf('Decided that %s has the ip %s', $bouncerTarget->getName(), $bouncerTarget->getIp()));
$bouncerTarget->setTargetPath(sprintf('http://%s:%d/', $bouncerTarget->getIp(), $bouncerTarget->getPort()));
$bouncerTargets[] = $bouncerTarget;
}
}
return $bouncerTargets;
}
public function findContainersSwarmMode(): array
{
$bouncerTargets = [];
$services = json_decode($this->client->request('GET', 'services')->getBody()->getContents(), true);
if(isset($services['message'])){
$this->logger->debug(sprintf('Something happened while interrogating services.. This node is not a swarm node, cannot have services: %s', $services['message']));
}else{
foreach($services as $service){
$envs = [];
if(
!isset($service['Spec'])
|| !isset($service['Spec']['TaskTemplate'])
|| !isset($service['Spec']['TaskTemplate']['ContainerSpec'])
|| !isset($service['Spec']['TaskTemplate']['ContainerSpec']['Env'])
)
{
continue;
}
foreach($service['Spec']['TaskTemplate']['ContainerSpec']['Env'] as $env){
list($eKey, $eVal) = explode("=", $env,2);
$envs[$eKey] = $eVal;
}
if(isset($envs['BOUNCER_DOMAIN'])) {
$bouncerTarget = (new BouncerTarget())
->setId($service['ID']);
$bouncerTarget = $this->parseContainerEnvironmentVariables($envs, $bouncerTarget);
$bouncerTarget->setIp("172.17.0.1");
$bouncerTarget->setPort($service['Endpoint']['Ports'][0]['PublishedPort']);
$bouncerTarget->setTargetPath(sprintf('http://%s:%d/', $bouncerTarget->getIp(), $bouncerTarget->getPort()));
//$this->logger->debug(sprintf('Decided that %s has the target path %s', $bouncerTarget->getName(), $bouncerTarget->getTargetPath()));
$bouncerTargets[] = $bouncerTarget;
}
}
}
return $bouncerTargets;
}
public function run(): void
{
$this->logger->info(sprintf('%s Starting Bouncer...', Emoji::CHARACTER_TIMER_CLOCK));
try {
$this->stateHasChanged();
}catch(\GuzzleHttp\Exception\ConnectException $connectException){
$this->logger->critical(sprintf("%s Could not connect to docker socket! Did you map it?", Emoji::CHARACTER_CRYING_CAT));
exit;
}
while (true) {
$this->runLoop();
}
}
/**
* Returns true when something has changed.
*
* @throws \GuzzleHttp\Exception\GuzzleException
*/
private function stateHasChanged(): bool
{
$newInstanceStates = [];
// Standard Containers
$containers = json_decode($this->client->request('GET', 'containers/json')->getBody()->getContents(), true);
foreach ($containers as $container) {
$inspect = json_decode($this->client->request('GET', "containers/{$container['Id']}/json")->getBody()->getContents(), true);
$newInstanceStates['container-' . $inspect['Id']] = implode('::', [
$inspect['Name'],
$inspect['Created'],
$inspect['Image'],
$inspect['State']['Status'],
sha1(implode('|', $inspect['Config']['Env'])),
]);
}
// Swarm Services
$services = json_decode($this->client->request('GET', 'services')->getBody()->getContents(), true);
if(isset($services['message'])){
$this->logger->debug(sprintf('Something happened while interrogating services.. This node is not a swarm node, cannot have services: %s', $services['message']));
}else{
foreach($services as $service){
$newInstanceStates['service-' . $service['ID']] = implode("::", [
$service['Version']['Index']
]);
}
}
$newStateHash = sha1(implode("\n", $newInstanceStates));
//$this->logger->debug(sprintf("Old state = %s. New State = %s.", substr($this->instanceStateHash,0,7), substr($newStateHash, 0,7)));
if ($this->instanceStateHash != $newStateHash) {
$this->instanceStateHash = $newStateHash;
return true;
}
return false;
}
private function runLoop(): void
{
if ($this->s3Enabled()) {
$this->getCertificatesFromS3();
}
$determineSwarmMode = json_decode($this->client->request('GET', 'swarm')->getBody()->getContents(), true);
$this->setSwarmMode(!isset($determineSwarmMode['message']));
$this->logger->info(sprintf("%s Swarm mode is %s.", Emoji::CHARACTER_HONEYBEE, $this->isSwarmMode() ? 'enabled' : 'disabled'));
$targets = $this->isSwarmMode() ? $this->findContainersSwarmMode() : $this->findContainersContainerMode();
$this->logger->info(sprintf('%s Found %d services with BOUNCER_DOMAIN set', Emoji::CHARACTER_MAGNIFYING_GLASS_TILTED_LEFT, count($targets)));
foreach ($targets as $target) {
$this->generateNginxConfig($target);
}
$this->generateLetsEncryptCerts($targets);
if ($this->s3Enabled()) {
$this->writeCertificatesToS3();
}
$this->waitUntilContainerChange();
}
private function waitUntilContainerChange(): void
{
while ($this->stateHasChanged() === false) {
sleep(5);
}
$this->logger->info(sprintf('%s Host Container state has changed', Emoji::CHARACTER_WARNING));
}
private function s3Enabled(): bool
{
return $this->certificateStoreRemote instanceof Filesystem;
}
private function getCertificatesFromS3(): void
{
$this->logger->info(sprintf('%s Downloading Certificates from S3', Emoji::CHARACTER_DOWN_ARROW));
foreach ($this->certificateStoreRemote->listContents('/', true) as $file) {
/** @var FileAttributes $file */
if ($file->isFile()) {
$localPath = "archive/{$file->path()}";
#$this->logger->debug(sprintf(" > Downloading {$file->path()} "));
$this->certificateStoreLocal->writeStream($localPath, $this->certificateStoreRemote->readStream($file->path()));
$this->fileHashes[$localPath] = sha1($this->certificateStoreLocal->read($localPath));
}
}
// Copy certs into /live because certbot is a pain.
foreach ($this->certificateStoreLocal->listContents('/archive', true) as $newLocalCert) {
/** @var FileAttributes $newLocalCert */
if ($newLocalCert->isFile() && pathinfo($newLocalCert->path(), PATHINFO_EXTENSION) == 'pem') {
$livePath = str_replace('archive/', 'live/', $newLocalCert->path());
// Stupid dirty hack bullshit reee
for($i = 1; $i <= 9; $i++){
$livePath = str_replace("$i.pem", ".pem", $livePath);
}
$this->certificateStoreLocal->writeStream($livePath, $this->certificateStoreLocal->readStream($newLocalCert->path()));
}
}
}
private function fileChanged(string $localPath)
{
if (!isset($this->fileHashes[$localPath])) {
return true;
}
if (sha1($this->certificateStoreLocal->read($localPath)) != $this->fileHashes[$localPath]) {
return true;
}
return false;
}
private function writeCertificatesToS3(): void
{
$this->logger->info(sprintf('%s Uploading Certificates to S3', Emoji::CHARACTER_UP_ARROW));
foreach ($this->certificateStoreLocal->listContents('/archive', true) as $file) {
/** @var FileAttributes $file */
if ($file->isFile()) {
$remotePath = str_replace('archive/', '', $file->path());
if (!$this->certificateStoreRemote->fileExists($remotePath) || $this->fileChanged($file->path())) {
#$this->logger->debug(sprintf(" > Uploading {$file->path()} "));
$this->certificateStoreRemote->writeStream($remotePath, $this->certificateStoreLocal->readStream($file->path()));
} else {
#$this->logger->debug(sprintf(" > Skipping uploading {$file->path()}, file not changed."));
}
}
}
}
private function generateNginxConfig(BouncerTarget $target): self
{
$this->configFilesystem->write(
$target->getName(),
$this->twig->render('NginxTemplate.twig', $target->__toArray())
);
$this->logger->info(sprintf('%s Created Nginx config for http://%s', Emoji::CHARACTER_PENCIL, $target->getName()));
return $this;
}
/**
* @param BouncerTarget[] $targets
*
* @return $this
*/
private function generateLetsEncryptCerts(array $targets): self
{
foreach ($targets as $target) {
if (!$target->isLetsEncrypt()) {
continue;
}
$testAgeFile = "/archive/{$target->getName()}/fullchain1.pem";
if ($this->certificateStoreLocal->fileExists($testAgeFile)) {
$ssl = openssl_x509_parse($this->certificateStoreLocal->read($testAgeFile));
$timeRemainingSeconds = $ssl['validTo_time_t'] - time();
if ($timeRemainingSeconds > 2592000) {
$this->logger->info(sprintf(
'%s Skipping %s, certificate is still good for %d days',
Emoji::CHARACTER_PARTYING_FACE,
$target->getName(),
round($timeRemainingSeconds / 86400)
));
$target->setUseTemporaryCert(false);
$this->generateNginxConfig($target);
continue;
}
}
$shell = new Exec();
$command = new CommandBuilder('/usr/bin/certbot');
$command->addSubCommand('certonly');
$command->addArgument('nginx');
if ($this->environment['BOUNCER_LETSENCRYPT_MODE'] != 'production') {
$command->addArgument('test-cert');
}
$command->addFlag('d', implode(',', $target->getDomains()));
$command->addFlag('n');
$command->addFlag('m', $this->environment['BOUNCER_LETSENCRYPT_EMAIL']);
$command->addArgument('agree-tos');
$this->logger->info(sprintf('%s Generating letsencrypt for %s - %s', Emoji::CHARACTER_PENCIL, $target->getName(), $command->__toString()));
$shell->run($command);
if ($shell->getReturnValue() == 0) {
$this->logger->info(sprintf('%s Generating successful', Emoji::CHARACTER_PARTY_POPPER));
} else {
$this->logger->critical(sprintf('%s Generating failed!', Emoji::CHARACTER_WARNING));
}
$target->setUseTemporaryCert(false);
$this->generateNginxConfig($target);
}
$this->restartNginx();
return $this;
}
private function restartNginx(): void
{
$shell = new Exec();
$command = new CommandBuilder('/usr/sbin/nginx');
$command->addFlag('s', 'reload');
$this->logger->info(sprintf('%s Restarting nginx', Emoji::CHARACTER_TIMER_CLOCK));
$shell->run($command);
}
/**
* @param array $envs
* @param BouncerTarget $bouncerTarget
* @return BouncerTarget
*/
public function parseContainerEnvironmentVariables(array $envs, BouncerTarget $bouncerTarget): BouncerTarget
{
foreach ($envs as $eKey => $eVal) {
switch ($eKey) {
case 'BOUNCER_DOMAIN':
$domains = explode(',', $eVal);
array_walk($domains, function (&$domain, $key): void {
$domain = trim($domain);
});
$bouncerTarget->setDomains($domains);
break;
case 'BOUNCER_LETSENCRYPT':
$bouncerTarget->setLetsEncrypt(in_array(strtolower($eVal), ['yes', 'true'], true));
break;
case 'BOUNCER_TARGET_PORT':
$bouncerTarget->setPort($eVal);
break;
case 'BOUNCER_ALLOW_NON_SSL':
$bouncerTarget->setAllowNonSSL(in_array(strtolower($eVal), ['yes', 'true'], true));
break;
}
}
return $bouncerTarget;
}
}
(new Bouncer())->run();