Benzine/Docker-Swarm-Loadbalancer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions2

Fix loadbalancing bug, add tests

Browse source
This commit is contained in:
Greyscale 2024-05-19 01:53:03 +02:00
parent e330f3f89c
commit e1d8a91c07
12 changed files with 315 additions and 114 deletions

3
.dockerignore Normal file
View file

@ -0,0 +1,3 @@
.github
.trunk
.idea

6
.github/workflows/build.yml vendored
View file

@ -15,6 +15,11 @@ concurrency:
cancel-in-progress: true
jobs:
tests:
name: "Tests"
uses: ./.github/workflows/tests.yml
permissions:
contents: read
build-container:
name: Build
uses: ./.github/workflows/docker.build.yml
@ -52,6 +57,7 @@ jobs:
- validate-container
- check-php
- check-trunk
- tests
uses: ./.github/workflows/docker.release.yml
secrets: inherit
permissions:

22
.github/workflows/docker.clean.yml vendored
View file

@ -12,31 +12,27 @@ on:
types:
- completed
env:
CANDIDATE_IMAGE: ghcr.io/benzine-framework/bouncer
CANDIDATE_TAG: build-${{ github.sha }}
jobs:
cleanup-delete-candidate-image:
name: Delete candidate image
runs-on: ubuntu-latest
steps:
- run: docker login ghcr.io -u ${{ github.repository_owner }} -p ${{ secrets.GITHUB_TOKEN }}
- uses: dataaxiom/ghcr-cleanup-action@main
- uses: dataaxiom/ghcr-cleanup-action@v1.0.3
with:
owner: ${{ github.repository_owner }}
repository: ${{ github.repository }}
name: ${{ env.CANDIDATE_IMAGE }}
tags: ${{ env.CANDIDATE_TAG }}
owner: benzine-framework
repository: docker-swarm-loadbalancer
name: bouncer
tags: build-${{ github.sha }}
token: ${{ secrets.GITHUB_TOKEN }}
cleanup-untagged-images:
name: Delete untagged images
runs-on: ubuntu-latest
steps:
- run: docker login ghcr.io -u ${{ github.repository_owner }} -p ${{ secrets.GITHUB_TOKEN }}
- uses: dataaxiom/ghcr-cleanup-action@main
- uses: dataaxiom/ghcr-cleanup-action@v1.0.3
with:
owner: ${{ github.repository_owner }}
repository: ${{ github.repository }}
name: ${{ env.CANDIDATE_IMAGE }}
owner: benzine-framework
repository: docker-swarm-loadbalancer
name: bouncer
token: ${{ secrets.GITHUB_TOKEN }}

72
.github/workflows/tests.yml vendored Normal file
View file

@ -0,0 +1,72 @@
name: "Tests"
permissions:
contents: read
on:
workflow_call:
workflow_dispatch:
jobs:
test-integration:
name: Integration Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Start Bouncer
run: docker compose up --build -d bouncer test-box
- name: Give it a moment...
run: sleep 5
- name: No-SSL Connect to Web A
run: |
docker compose exec test-box curl -s -D - http://a.example.org > a.nossl.http
grep "HTTP/1.1 200 OK" a.nossl.http;
grep "<h1>Website A</h1>" a.nossl.http;
- name: SSL Connect to Web A
run: |
docker compose exec test-box curl -s -k -D - https://a.example.org 2>&1 > a.ssl.http;
grep "HTTP/1.1 200 OK" a.ssl.http;
grep "<h1>Website A</h1>" a.ssl.http;
- name: No-SSL Connect to Web B
run: |
docker compose exec test-box curl -s -D - http://b.example.org 2>&1 > b.nossl.http
grep "HTTP/1.1 200 OK" b.nossl.http
grep "<h1>Website B</h1>" b.nossl.http
- name: SSL Connect to Web B
run: |
docker compose exec test-box curl -s -k -D - https://b.example.org 2>&1 > b.ssl.http
grep "HTTP/1.1 200 OK" b.ssl.http
grep "<h1>Website B</h1>" b.ssl.http
- name: No-SSL Connect to SSL-redirect
run: |
docker compose exec test-box curl -s -D - http://redirect-to-ssl.example.org 2>&1 > redirect.nossl.http
# Validate its redirected
grep "HTTP/1.1 301 Moved Permanently" redirect.nossl.http
# And going to the right place
grep "Location: https://redirect-to-ssl.example.org" redirect.nossl.http
- name: SSL Connect to SSL-redirect
run: |
docker compose exec test-box curl -s -k -D - https://redirect-to-ssl.example.org 2>&1 > redirect.ssl.http
grep "HTTP/1.1 200 OK" redirect.ssl.http
grep "<h1>Website redirect-to-ssl</h1>" redirect.ssl.http
- name: Connect to Plural multiple times and verify it loadbalances
run: |
rm -f plural_requests
for i in {1..20}; do
docker compose exec test-box curl -s -k https://plural.example.org 2>&1 >> plural_requests
done
requests=$(cat plural_requests | grep "Running on" | sort | uniq | wc -l)
echo "Unique Servers: $requests"
# We should have exactly 3
test $requests -eq 3
- name: Cleanup
if: always()
run: docker compose down -v --remove-orphans

14
.php-cs-fixer.php
View file

@ -1,8 +1,16 @@
<?php
$finder = PhpCsFixer\Finder::create();
declare(strict_types=1);
use PhpCsFixer\Config;
use PhpCsFixer\Finder;
use PhpCsFixer\Runner\Parallel\ParallelConfig;
$finder = Finder::create();
$finder->in(__DIR__);
return (new PhpCsFixer\Config())
return (new Config())
->setParallelConfig(new ParallelConfig(10, 20, 120))
->setRiskyAllowed(true)
->setHideProgress(false)
->setRules([
@ -37,4 +45,4 @@ return (new PhpCsFixer\Config())
'native_function_invocation' => false, // Disabled as adding count($i) -> \count($i) is annoying, but supposedly more performant
])
->setFinder($finder)
;
;

9
Dockerfile
View file

@ -91,11 +91,18 @@ EXPOSE 80
EXPOSE 443
# Set a healthcheck to curl the bouncer and expect a 200
# A moderately long start period is important because while it IS serving a HTTP 200 immediately, it might not have
# completed probing the docker socket and generating the config yet.
HEALTHCHECK --start-period=30s \
CMD curl -s -o /dev/null -w "200" http://localhost:80/ || exit 1
# checkov:skip=CKV_DOCKER_3 This is a test container.
FROM php:nginx as test-app
COPY tests/testsites /app/public
HEALTHCHECK --start-period=30s \
HEALTHCHECK --start-period=3s --interval=3s \
CMD curl -s -o /dev/null -w "200" http://localhost:80/ || exit 1
# checkov:skip=CKV_DOCKER_7 This is a test container.
# checkov:skip=CKV_DOCKER_3 This is a test container.
FROM alpine as test-box
RUN apk add --no-cache curl bash

12
Todo.md Normal file
View file

@ -0,0 +1,12 @@
# Todo List
- Create an actual healthcheck endpoint. It should:
- Display if the initial docker socket scan & generation is complete
- Show Domains served by the proxy behind a per-service option flag
- Domains should be grouped
- Some statistics about the domains such as the number of instances serving it should be presented
- Maybe the amount of traffic being sent to it?
- Switch to using service labels instead of envs
- Write tests to verify the S3/Lets Encrypt functionality still works
- Log traffic to somewhere useful
- Add a way to write logs to external services, exposing the Monolog behavior

80
docker-compose.yml
View file

@ -1,46 +1,74 @@
networks:
default:
services:
bouncer:
image: ghcr.io/benzine-framework/bouncer:latest
build:
context: .
target: bouncer
additional_contexts:
- php:cli=docker-image://ghcr.io/benzine-framework/php:cli-8.2
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ./:/app
# environment:
# - BOUNCER_LETSENCRYPT_MODE=staging
# - BOUNCER_LETSENCRYPT_EMAIL=matthew@baggett.me
# - BOUNCER_S3_ENDPOINT=http://grey.ooo:9000
# - BOUNCER_S3_KEY_ID=geusebio
# - BOUNCER_S3_KEY_SECRET=changeme
# - BOUNCER_S3_BUCKET=bouncer-certificates
# - BOUNCER_S3_USE_PATH_STYLE_ENDPOINT="yes"
ports:
- 127.0.99.100:80:80
- 127.0.99.100:443:443
web-a:
image: test-app-a
- ./src:/app/src
- ./templates:/app/templates
- ./vendor:/app/vendor
networks:
default:
aliases:
- a.example.org
- b.example.org
- plural.example.org
- redirect-to-ssl.example.org
- nope.example.org
depends_on:
web-a:
condition: service_healthy
web-b:
condition: service_healthy
web-redirect-ssl:
condition: service_healthy
web-plural:
condition: service_healthy
web-a: &web
build:
target: test-app-a
context: .
target: test-app
additional_contexts:
- php:nginx=docker-image://ghcr.io/benzine-framework/php:nginx-8.2
volumes:
- ./tests/testsites:/app/public
environment:
- BOUNCER_DOMAIN=a.web.grey.ooo
- BOUNCER_DOMAIN=a.example.org
- BOUNCER_TARGET_PORT=80
- SITE_NAME=A
networks:
- default
web-b:
image: test-app-b
build:
target: test-app-b
additional_contexts:
- php:nginx=docker-image://ghcr.io/benzine-framework/php:nginx-8.2
volumes:
- ./tests/testsites:/app/public
<<: *web
environment:
- BOUNCER_DOMAIN=b.web.grey.ooo
- BOUNCER_DOMAIN=b.example.org
- BOUNCER_TARGET_PORT=80
- SITE_NAME=B
web-plural:
<<: *web
environment:
- BOUNCER_DOMAIN=plural.example.org
- BOUNCER_TARGET_PORT=80
- SITE_NAME=plural
deploy:
replicas: 3
web-redirect-ssl:
<<: *web
environment:
- BOUNCER_DOMAIN=redirect-to-ssl.example.org
- BOUNCER_TARGET_PORT=80
- SITE_NAME=redirect-to-ssl
- BOUNCER_ALLOW_NON_SSL=false
test-box:
build:
context: .
target: test-box
command: ["tail", "-f", "/dev/null"]
networks:
- default

131
src/Bouncer.php
View file

@ -49,6 +49,7 @@ class Bouncer
private ?int $lastUpdateEpoch = null;
private int $maximumNginxConfigCreationNotices = 15;
private Settings $settings;
private bool $testMode;
private const DEFAULT_DOCKER_SOCKET = '/var/run/docker.sock';
private const FILESYSTEM_CONFIG_DIR = '/etc/nginx/sites-enabled';
@ -110,6 +111,8 @@ class Bouncer
)
);
}
$this->setTestMode(isset($this->environment['TEST_MODE']));
}
public function getMaximumNginxConfigCreationNotices(): int
@ -160,6 +163,21 @@ class Bouncer
return $this;
}
public function isTestMode(): bool
{
return $this->testMode;
}
public function setTestMode(bool $testMode): Bouncer
{
$this->testMode = $testMode;
if ($this->testMode) {
$this->logger->critical('Test mode is enabled. It will immediately crash out upon completion of 1 cycle.', ['emoji' => Emoji::warning()]);
}
return $this;
}
/**
* @return Target[]
*
@ -206,32 +224,45 @@ class Bouncer
if (!empty($container['NetworkSettings']['IPAddress'])) {
// As per docker service
$bouncerTarget->setEndpointHostnameOrIp($container['NetworkSettings']['IPAddress']);
$bouncerTarget->setEndpoints([$container['NetworkSettings']['IPAddress']]);
} else {
// As per docker compose
$networks = array_values($container['NetworkSettings']['Networks']);
$bouncerTarget->setEndpointHostnameOrIp($networks[0]['IPAddress']);
$bouncerTarget->setEndpoints([$networks[0]['IPAddress']]);
}
$bouncerTarget->setTargetPath(sprintf('http://%s:%d', $bouncerTarget->getEndpointHostnameOrIp(), $bouncerTarget->getPort() >= 0 ? $bouncerTarget->getPort() : 80));
$bouncerTarget->setUseGlobalCert($this->isUseGlobalCert());
$valid = $bouncerTarget->isEndpointValid();
// $this->logger->debug(sprintf(
// '%s Decided that %s has the endpoint %s and it %s.',
// Emoji::magnifyingGlassTiltedLeft(),
// $bouncerTarget->getName(),
// $bouncerTarget->getEndpointHostnameOrIp(),
// $valid ? 'is valid' : 'is not valid'
// ));
if ($valid) {
// if this bouncerTarget already exists, merge it in instead of adding it.
foreach ($bouncerTargets as $existing) {
if (isset($bouncerTarget) && $existing->getDomains() == $bouncerTarget->getDomains()) {
$this->logger->debug('Found another instance of the same service, merging them together.', ['emoji' => Emoji::cupcake()]);
$existing->setEndpoints(array_merge($existing->getEndpoints(), $bouncerTarget->getEndpoints()));
unset($bouncerTarget);
}
}
if (isset($bouncerTarget)) {
$bouncerTargets[] = $bouncerTarget;
}
}
}
return $bouncerTargets;
$validBouncerTargets = [];
foreach ($bouncerTargets as $bouncerTarget) {
$valid = $bouncerTarget->isEndpointValid();
// $this->logger->debug(sprintf(
// '%s Decided that %s has the endpoint %s and it %s.',
// Emoji::magnifyingGlassTiltedLeft(),
// $bouncerTarget->getName(),
// $bouncerTarget->getEndpointHostnameOrIp(),
// $valid ? 'is valid' : 'is not valid'
// ));
if ($valid) {
$validBouncerTargets[] = $bouncerTarget;
}
}
return $validBouncerTargets;
}
public function findContainersSwarmMode(): array
@ -288,10 +319,10 @@ class Bouncer
continue;
}
if ($bouncerTarget->isPortSet()) {
$bouncerTarget->setEndpointHostnameOrIp($service['Spec']['Name']);
$bouncerTarget->setEndpoints([$service['Spec']['Name']]);
// $this->logger->info('{label}: Ports for {target_name} has been explicitly set to {host}:{port}.', ['emoji' => Emoji::warning().' ', 'target_name' => $bouncerTarget->getName(), 'host' => $bouncerTarget->getEndpointHostnameOrIp(), 'port' => $bouncerTarget->getPort()]);
} elseif (isset($service['Endpoint']['Ports'])) {
$bouncerTarget->setEndpointHostnameOrIp('172.17.0.1');
$bouncerTarget->setEndpoints(['172.17.0.1']);
$bouncerTarget->setPort(intval($service['Endpoint']['Ports'][0]['PublishedPort']));
} else {
$this->logger->warning('{label}: ports block missing for {target_name}. Try setting BOUNCER_TARGET_PORT.', ['emoji' => Emoji::warning() . ' Bouncer.php', 'label' => $bouncerTarget->getLabel(), 'target_name' => $bouncerTarget->getName()]);
@ -303,28 +334,39 @@ class Bouncer
continue;
}
$bouncerTarget->setTargetPath(sprintf('http://%s:%d', $bouncerTarget->getEndpointHostnameOrIp(), $bouncerTarget->getPort()));
$bouncerTarget->setUseGlobalCert($this->isUseGlobalCert());
// @phpstan-ignore-next-line MB: I'm not sure you're right about ->hasCustomNginxConfig only returning false, Stan..
if ($bouncerTarget->isEndpointValid() || $bouncerTarget->hasCustomNginxConfig()) {
$bouncerTargets[] = $bouncerTarget;
} else {
$this->logger->debug(
'Decided that {target_name} has the endpoint {endpoint} and it is not valid.',
[
'emoji' => Emoji::magnifyingGlassTiltedLeft(),
'target_name' => $bouncerTarget->getName(),
'endpoint' => $bouncerTarget->getEndpointHostnameOrIp(),
]
);
// if this bouncerTarget already exists, merge it in instead of adding it.
foreach ($bouncerTargets as $existing) {
if ($existing->getDomains() == $bouncerTarget->getDomains()) {
$this->logger->debug('Found another instance of the same service, merging them together.', ['emoji' => Emoji::cupcake()]);
$existing->setEndpoints(array_merge($existing->getEndpoints(), $bouncerTarget->getEndpoints()));
unset($bouncerTarget);
}
}
}
}
}
return $bouncerTargets;
// Iterate over bouncers and check validity
$validBouncerTargets = [];
foreach ($bouncerTargets as $bouncerTarget) {
// @phpstan-ignore-next-line MB: I'm not sure you're right about ->hasCustomNginxConfig only returning false, Stan..
if ($bouncerTarget->isEndpointValid() || $bouncerTarget->hasCustomNginxConfig()) {
$validBouncerTargets[] = $bouncerTarget;
} else {
$this->logger->debug(
'Decided that {target_name} has the endpoint {endpoint} and it is not valid.',
[
'emoji' => Emoji::magnifyingGlassTiltedLeft(),
'target_name' => $bouncerTarget->getName(),
'endpoint' => $bouncerTarget->getEndpoints()[0],
]
);
}
}
return $validBouncerTargets;
}
public function run(): void
@ -512,10 +554,7 @@ class Bouncer
} elseif ($this->forcedUpdateIntervalSeconds > 0 && $this->lastUpdateEpoch <= time() - $this->forcedUpdateIntervalSeconds) {
$this->logger->warning('Forced update interval of {interval_seconds} seconds has been reached, forcing update.', ['emoji' => Emoji::watch(), 'interval_seconds' => $this->forcedUpdateIntervalSeconds]);
$isTainted = true;
} elseif ($this->previousContainerState === []) {
$this->logger->warning('Initial state has not been set, forcing update.', ['emoji' => Emoji::watch()]);
$isTainted = true;
} elseif ($this->previousSwarmState === []) {
} elseif ($this->previousContainerState === [] && $this->previousSwarmState === []) {
$this->logger->warning('Initial swarm state has not been set, forcing update.', ['emoji' => Emoji::watch()]);
$isTainted = true;
}
@ -663,6 +702,13 @@ class Bouncer
return;
}
if ($this->isTestMode()) {
$this->logger->info('Test mode enabled, not restarting nginx. Infact, I\'ll die now..', ['emoji' => Emoji::warning() . ' Bouncer.php']);
$this->dumpConfigs();
exit(0);
}
// Wait for next change
$this->waitUntilContainerChange();
}
@ -924,7 +970,6 @@ class Bouncer
$target->setUseTemporaryCert(false);
$this->generateNginxConfig($target);
}
$this->restartNginx();
}
@ -937,4 +982,18 @@ class Bouncer
$nginxRestartOutput = $shell->run($command);
$this->logger->debug('Nginx restarted {restart_output}', ['restart_output' => $nginxRestartOutput, 'emoji' => Emoji::partyPopper()]);
}
private function dumpConfigs(): void
{
// Dump the contents of every .conf file in /etc/nginx/sites-enabled
foreach ($this->configFilesystem->listContents('') as $file) {
if ($file['type'] == 'file' && pathinfo($file['path'], PATHINFO_EXTENSION) == 'conf') {
if ($file['path'] == 'default.conf') {
continue;
}
$this->logger->info('Dumping {file}', ['emoji' => Emoji::pencil() . ' Bouncer.php', 'file' => $file['path']]);
echo $this->configFilesystem->read($file['path']);
}
}
}
}

66
src/Target.php
View file

@ -14,10 +14,9 @@ class Target
private string $id;
private ?string $label = null;
private array $domains;
private string $endpointHostnameOrIp;
private array $endpoints = [];
private ?int $port = null;
private bool $letsEncrypt = false;
private string $targetPath;
private bool $allowNonSSL;
private bool $useTemporaryCert = false;
private bool $useGlobalCert = false;
@ -55,8 +54,9 @@ class Target
'name' => $this->getName(),
'label' => $this->getLabel(),
'serverName' => $this->getNginxServerName(),
'backends' => $this->getBackends(),
'backendName' => $this->getBackendName(),
'certType' => $this->getTypeCertInUse()->name,
'targetPath' => $this->getTargetPath(),
'customCertFile' => $this->getCustomCertPath(),
'customCertKeyFile' => $this->getCustomCertKeyPath(),
'useCustomCert' => $this->isUseCustomCert(),
@ -318,6 +318,11 @@ class Target
return implode(' ', $this->getNginxServerNames());
}
public function getBackendName()
{
return sprintf('backend_%s', substr(md5($this->getName()), 0, 8));
}
/**
* @param string[] $domains
*/
@ -341,26 +346,24 @@ class Target
return $this;
}
public function getTargetPath(): string
public function getBackends(): array
{
return $this->targetPath;
$backends = [];
foreach ($this->getEndpoints() as $endpoint) {
$backends[] = sprintf('%s:%d', $endpoint, $this->getPort());
}
return $backends;
}
public function setTargetPath(string $targetPath): self
public function getEndpoints(): array
{
$this->targetPath = $targetPath;
return $this;
return $this->endpoints;
}
public function getEndpointHostnameOrIp(): string
public function setEndpoints(array $endpoints): self
{
return $this->endpointHostnameOrIp;
}
public function setEndpointHostnameOrIp(string $endpointHostnameOrIp): self
{
$this->endpointHostnameOrIp = $endpointHostnameOrIp;
$this->endpoints = $endpoints;
return $this;
}
@ -422,24 +425,25 @@ class Target
public function isEndpointValid(): bool
{
// Is it just an IP?
if (filter_var($this->getEndpointHostnameOrIp(), FILTER_VALIDATE_IP)) {
// $this->logger->debug(sprintf('%s isEndpointValid: %s is a normal IP', Emoji::magnifyingGlassTiltedRight(), $this->getEndpointHostnameOrIp()));
foreach ($this->getEndpoints() as $endpoint) {
// Is it just an IP?
if (filter_var($endpoint, FILTER_VALIDATE_IP)) {
// $this->logger->debug(sprintf('%s isEndpointValid: %s is a normal IP', Emoji::magnifyingGlassTiltedRight(), $this->getEndpointHostnameOrIp()));
return true;
return true;
}
// Is it a Hostname that resolves?
$resolved = gethostbyname($endpoint);
if (filter_var($resolved, FILTER_VALIDATE_IP)) {
// $this->logger->critical(sprintf('%s isEndpointValid: %s is a hostname that resolves to a normal IP %s', Emoji::magnifyingGlassTiltedRight(), $this->getEndpointHostnameOrIp(), $resolved));
return true;
}
$this->logger->critical('isEndpointValid: {endpoint} is a hostname that does not resolve', ['emoji' => Emoji::magnifyingGlassTiltedRight(), 'endpoint' => $endpoint]);
$this->setRequiresForcedScanning(true);
}
// Is it a Hostname that resolves?
$resolved = gethostbyname($this->getEndpointHostnameOrIp());
if (filter_var($resolved, FILTER_VALIDATE_IP)) {
// $this->logger->critical(sprintf('%s isEndpointValid: %s is a hostname that resolves to a normal IP %s', Emoji::magnifyingGlassTiltedRight(), $this->getEndpointHostnameOrIp(), $resolved));
return true;
}
$this->logger->critical('isEndpointValid: {endpoint} is a hostname that does not resolve', ['emoji' => Emoji::magnifyingGlassTiltedRight(), 'endpoint' => $this->getEndpointHostnameOrIp()]);
$this->setRequiresForcedScanning(true);
return false;
}

8
templates/NginxTemplate.twig
View file

@ -1,3 +1,9 @@
upstream {{ backendName }} {
least_conn;
{% for backend in backends %}
server {{ backend }};
{% endfor %}
}
server {
{% if allowNonSSL %}
# Non-SSL Traffic is allowed
@ -39,7 +45,7 @@ server {
{% endif %}
# Server to send the request on to
proxy_pass "{{ targetPath }}";
proxy_pass "http://{{ backendName }}";
# Standard headers setting origin data
proxy_set_header X-Real-IP $remote_addr;

6
tests/testsites/index.php
View file

@ -2,6 +2,6 @@
declare(strict_types=1);
$environment = array_merge($_ENV, $_SERVER);
$site = $environment['SITE_NAME'] ?? 'unknown';
$server = $environment['SERVER_NAME'] ?? gethostname();
printf('<h1>Website %s</h1><p>Running on %s</p>', $site, $server);
$site = $environment['SITE_NAME'] ?? 'unknown';
$server = $environment['HOSTNAME'] ?? gethostname();
printf("<h1>Website %s</h1>\n<p>Running on %s</p>\n", $site, $server);