Swarm Monitor

swarm-monitor/.dockerignore

@@ -0,0 +1 @@
+vendor

swarm-monitor/.gitignore

@@ -0,0 +1,3 @@
+/docker-compose.override.yml
+/vendor
+/.php-cs-fixer.cache

swarm-monitor/.php-cs-fixer.php

@@ -0,0 +1,22 @@
+<?php
+$finder = PhpCsFixer\Finder::create();
+$finder->in(__DIR__);
+
+return (new PhpCsFixer\Config)
+    ->setRiskyAllowed(true)
+    ->setHideProgress(false)
+    ->setRules([
+        '@PSR2' => true,
+        'strict_param' => true,
+        'array_syntax' => ['syntax' => 'short'],
+        '@PhpCsFixer' => true,
+        '@PHP73Migration' => true,
+        'no_php4_constructor' => true,
+        'no_unused_imports' => true,
+        'no_useless_else' => true,
+        'no_superfluous_phpdoc_tags' => true,
+        'void_return' => true,
+        'yoda_style' => false,
+    ])
+    ->setFinder($finder)
+    ;

swarm-monitor/Dockerfile

@@ -0,0 +1,21 @@
+FROM benzine/php:cli-8.1 as swarm-agent
+LABEL maintainer="Matthew Baggett <matthew@baggett.me>" \
+      org.label-schema.vcs-url="https://github.com/benzine-framework/docker" \
+      org.opencontainers.image.source="https://github.com/benzine-framework/docker"
+
+COPY agent.runit /etc/service/agent/run
+RUN chmod +x /etc/service/*/run
+COPY agent /app
+COPY composer.* /app/
+RUN composer install && \
+    chmod +x /app/agent && \
+    mkdir -p /var/log/agent
+
+FROM benzine/php:nginx-8.1 as swarm-stats
+LABEL maintainer="Matthew Baggett <matthew@baggett.me>" \
+      org.label-schema.vcs-url="https://github.com/benzine-framework/docker" \
+      org.opencontainers.image.source="https://github.com/benzine-framework/docker"
+
+COPY public /app/public
+COPY composer.* /app/
+RUN composer install

swarm-monitor/agent

@@ -0,0 +1,379 @@
+#!/usr/bin/env php
+<?php
+require_once 'vendor/autoload.php';
+
+use Bramus\Monolog\Formatter\ColoredLineFormatter;
+use GuzzleHttp\Client as Guzzle;
+use Monolog\Handler\StreamHandler;
+use Monolog\Logger;
+use Spatie\Emoji\Emoji;
+
+class AgentState
+{
+    private string $hostname = 'unknown host';
+    /** @var string[] */
+    private array $labels = [];
+    private int $containerCount = 0;
+    private int $containerCountRunning = 0;
+
+    private int $memAvailableKB = 0;
+    private int $memFreeKB = 0;
+    private int $memTotalKB = 0;
+    private int $swapFreeKB = 0;
+    private int $swapTotalKB = 0;
+
+    public function __toString(): string
+    {
+        return sprintf(
+            'Population: %d running of %d. Memory %.2fGB/%.2fGB. Swap %.2fGB/%.2fGB',
+            $this->getContainerCountRunning(),
+            $this->getContainerCount(),
+            $this->getMemAvailableKB() / 1024 / 1024,
+            $this->getMemTotalKB() / 1024 / 1024,
+            $this->getSwapFreeKB() / 1024 / 1024,
+            $this->getSwapTotalKB() / 1024 / 1024,
+        );
+    }
+
+    public function emit(Redis $redis, int $redisExpiry = 60): void
+    {
+        $instanceDataKey = "swarm:{$this->getHostname()}";
+
+        $redis->hset($instanceDataKey, "population_running", $this->getContainerCountRunning());
+        $redis->hset($instanceDataKey, "population_total", $this->getContainerCount());
+        $redis->hset($instanceDataKey, "memory_available_kb", $this->getMemAvailableKB());
+        $redis->hset($instanceDataKey, "memory_free_kb", $this->getMemFreeKB());
+        $redis->hset($instanceDataKey, "memory_total_kb", $this->getMemTotalKB());
+        $redis->hset($instanceDataKey, "swap_free_kb", $this->getSwapFreeKB());
+        $redis->hset($instanceDataKey, "swap_total_kb", $this->getSwapTotalKB());
+        $redis->hset($instanceDataKey, "updated_at", date("Y-m-d H:i:s"));
+        $redis->hset($instanceDataKey, "labels", implode(", ", $this->getLabels()));
+
+        $redis->expire($instanceDataKey, $redisExpiry);
+    }
+
+    public function getHostname(): string
+    {
+        return strtolower($this->hostname);
+    }
+
+    public function setHostname(string $hostname): AgentState
+    {
+        $this->hostname = $hostname;
+
+        return $this;
+    }
+
+    public function getContainerCount(): int
+    {
+        return $this->containerCount;
+    }
+
+    public function setContainerCount(int $containerCount): AgentState
+    {
+        $this->containerCount = $containerCount;
+
+        return $this;
+    }
+
+    public function getContainerCountRunning(): int
+    {
+        return $this->containerCountRunning;
+    }
+
+    public function setContainerCountRunning(int $containerCountRunning): AgentState
+    {
+        $this->containerCountRunning = $containerCountRunning;
+
+        return $this;
+    }
+
+    public function getMemAvailableKB(): int
+    {
+        return $this->memAvailableKB;
+    }
+
+    public function setMemAvailableKB(int $memAvailableKB): AgentState
+    {
+        $this->memAvailableKB = $memAvailableKB;
+
+        return $this;
+    }
+
+    public function getMemFreeKB(): int
+    {
+        return $this->memFreeKB;
+    }
+
+    public function setMemFreeKB(int $memFreeKB): AgentState
+    {
+        $this->memFreeKB = $memFreeKB;
+
+        return $this;
+    }
+
+    public function getMemTotalKB(): int
+    {
+        return $this->memTotalKB;
+    }
+
+    public function setMemTotalKB(int $memTotalKB): AgentState
+    {
+        $this->memTotalKB = $memTotalKB;
+
+        return $this;
+    }
+
+    public function getSwapFreeKB(): int
+    {
+        return $this->swapFreeKB;
+    }
+
+    public function setSwapFreeKB(int $swapFreeKB): AgentState
+    {
+        $this->swapFreeKB = $swapFreeKB;
+
+        return $this;
+    }
+
+    public function getSwapTotalKB(): int
+    {
+        return $this->swapTotalKB;
+    }
+
+    public function setSwapTotalKB(int $swapTotalKB): AgentState
+    {
+        $this->swapTotalKB = $swapTotalKB;
+
+        return $this;
+    }
+
+    /**
+     * @return string[]
+     */
+    public function getLabels(): array
+    {
+        return $this->labels;
+    }
+
+    /**
+     * @param string[] $labels
+     * @return AgentState
+     */
+    public function setLabels(array $labels): AgentState
+    {
+        $this->labels = $labels;
+        return $this;
+    }
+
+}
+class Agent
+{
+    private array $environment;
+    private Guzzle $client;
+    private Logger $logger;
+    private ?string $instanceStateHash = null;
+    private ?AgentState $currentState = null;
+    private ?AgentState $lastState = null;
+    private Redis $redis;
+    private int $minimumUpdateIntervalSeconds = 60;
+    private int $lastRunTimestamp = 0;
+
+    public function __construct()
+    {
+        $this->environment = array_merge($_ENV, $_SERVER);
+        ksort($this->environment);
+
+        $this->logger = new Monolog\Logger('agent');
+        $this->logger->pushHandler(new StreamHandler('/var/log/swarm-agent.log', Logger::DEBUG));
+        $stdout = new StreamHandler('php://stdout', Logger::DEBUG);
+        $stdout->setFormatter(new ColoredLineFormatter(null, "%level_name%: %message% \n"));
+        $this->logger->pushHandler($stdout);
+
+        if(isset($this->environment['MINIMUM_UPDATE_INTERVAL'])){
+            $this->minimumUpdateIntervalSeconds = $this->environment['MINIMUM_UPDATE_INTERVAL'];
+        }
+
+        $this->redis = new Redis();
+        $this->redis->pconnect(
+                $this->environment['REDIS_HOST'] ?? "redis",
+                $this->environment['REDIS_PORT'] ?? 6379
+        );
+
+        $this->client = new Guzzle(
+            [
+                'base_uri' => 'http://localhost',
+                'curl' => [
+                    CURLOPT_UNIX_SOCKET_PATH => '/var/run/docker.sock',
+                ],
+            ]
+        );
+
+        // Prevent double-starting
+        $this->lastRunTimestamp = time();
+    }
+
+    public function findContainersSwarmMode(): array
+    {
+        $services = json_decode($this->client->request('GET', 'services')->getBody()->getContents(), true);
+    }
+
+    public function run(): void
+    {
+        $this->logger->info(sprintf('%s  Starting Swarm Agent...', Emoji::CHARACTER_TIMER_CLOCK));
+        while (true) {
+            $this->runLoop();
+        }
+    }
+
+    private function calculateStateHash(): string
+    {
+        $newInstanceStates = [];
+
+        $containers = json_decode($this->client->request('GET', 'containers/json')->getBody()->getContents(), true);
+        foreach ($containers as $container) {
+            $inspect = json_decode($this->client->request('GET', "containers/{$container['Id']}/json")->getBody()->getContents(), true);
+            $newInstanceStates['container-'.$inspect['Id']] = implode('::', [
+                $inspect['Name'],
+                $inspect['Created'],
+                $inspect['Image'],
+                $inspect['State']['Status'],
+                sha1(implode('|', $inspect['Config']['Env'])),
+            ]);
+        }
+
+        return sha1(implode("\n", $newInstanceStates));
+    }
+
+    /**
+     * Returns true when something has changed.
+     *
+     * @throws \GuzzleHttp\Exception\GuzzleException
+     */
+    private function stateHasChanged(): bool
+    {
+        $newStateHash = $this->calculateStateHash();
+        // $this->logger->debug(sprintf("Old state = %s. New State = %s.", substr($this->instanceStateHash,0,7), substr($newStateHash, 0,7)));
+        if (!$this->instanceStateHash || $this->instanceStateHash != $newStateHash) {
+            $this->instanceStateHash = $newStateHash;
+
+            return true;
+        }
+
+        return false;
+    }
+
+    private function timeElapsed() : bool
+    {
+        if($this->lastRunTimestamp < time() - $this->minimumUpdateIntervalSeconds){
+            $this->lastRunTimestamp = time();
+            $this->logger->debug(sprintf("Max interval of %d seconds has elapsed", $this->minimumUpdateIntervalSeconds));
+            return true;
+        }
+
+        return false;
+    }
+
+    private function containerInventory(): void
+    {
+        $containers = json_decode($this->client->request('GET', 'containers/json')->getBody()->getContents(), true);
+
+        $runningContainers = 0;
+        foreach ($containers as $container) {
+            if ($container['State'] == 'running') {
+                ++$runningContainers;
+            } else {
+                \Kint::dump($container['State']);
+
+                exit;
+            }
+        }
+
+        $this->currentState
+            ->setContainerCount(count($containers))
+            ->setContainerCountRunning($runningContainers)
+        ;
+    }
+
+    private function runLoop(): void
+    {
+        if ($this->currentState instanceof AgentState) {
+            $this->lastState = $this->currentState;
+        }
+        $this->currentState = (new AgentState());
+
+        $this->probeSysInfo();
+
+        $this->containerInventory();
+        $this->probeMemory();
+        $this->probeDisks();
+        $this->logger->debug($this->currentState->__toString());
+        $this->currentState->emit($this->redis, $this->minimumUpdateIntervalSeconds+30);
+
+        $this->waitUntilContainerChange();
+    }
+
+    private function probeSysInfo() : void
+    {
+        $info = json_decode($this->client->request('GET', 'info')->getBody()->getContents(), true);
+        $this->currentState->setHostname($info['Name']);
+        $this->currentState->setLabels($info['Labels']);
+    }
+
+    private function probeMemory(): void
+    {
+        $memInfo = file_get_contents('/proc/meminfo');
+        foreach (explode("\n", $memInfo) as $line) {
+            if (stripos($line, ':') === false) {
+                continue;
+            }
+            [$key, $value] = explode(':', $line);
+            $key = trim($key);
+            $value = trim($value);
+            $value = str_replace(' kB', '', $value);
+
+            switch ($key) {
+                case 'MemTotal':
+                    $this->currentState->setMemTotalKB($value);
+
+                    break;
+
+                case 'MemFree':
+                    $this->currentState->setMemFreeKB($value);
+
+                    break;
+
+                case 'MemAvailable':
+                    $this->currentState->setMemAvailableKB($value);
+
+                    break;
+
+                case 'SwapTotal':
+                    $this->currentState->setSwapTotalKB($value);
+
+                    break;
+
+                case 'SwapFree':
+                    $this->currentState->setSwapFreeKB($value);
+
+                    break;
+            }
+        }
+    }
+
+    private function probeDisks(): void
+    {
+        // @todo
+        // $diskinfo = file_get_contents("/proc/diskstats")
+    }
+
+    private function waitUntilContainerChange(): void
+    {
+        while ($this->stateHasChanged() === false && $this->timeElapsed() === false) {
+            sleep(5);
+        }
+        $this->logger->info(sprintf('%s  Host Container state has changed', Emoji::CHARACTER_WARNING));
+    }
+}
+
+(new Agent())->run();

swarm-monitor/agent.runit

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+echo "Starting Swarm Agent"
+/app/agent
+sleep 30;

swarm-monitor/composer.json

@@ -0,0 +1,29 @@
+{
+    "name": "benzine/swarm-monitor",
+    "description": "Swarm agent monitoring",
+    "type": "project",
+    "config": {
+        "sort-packages": true
+    },
+    "license": "GPL-3.0-or-later",
+    "require": {
+        "php": "^8.1",
+        "ext-json": "*",
+        "ext-curl": "*",
+        "kint-php/kint": "^3.3",
+        "guzzlehttp/guzzle": "^7.3",
+        "monolog/monolog": "^2.2",
+        "bramus/monolog-colored-line-formatter": "~3.0",
+        "spatie/emoji": "^2.3",
+        "ext-redis": "*"
+    },
+    "authors": [
+        {
+            "name": "Matthew Baggett",
+            "email": "matthew@baggett.me"
+        }
+    ],
+    "require-dev": {
+        "friendsofphp/php-cs-fixer": "^3.0"
+    }
+}

swarm-monitor/docker-compose.yml

@@ -0,0 +1,36 @@
+version: "3.4"
+
+services:
+  redis:
+    image: redis
+    ports:
+      - 127.0.0.30:6379:6379
+  agent:
+    image: matthewbaggett/swarm-agent:agent
+    build:
+      context: .
+      target: swarm-agent
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      #- /proc/meminfo:/proc/meminfo:ro
+      - ./:/app
+    command: ["/app/agent"]
+    environment:
+      - HOSTNAME=exploding-bolts
+      - REDIS_HOST=redis
+      - MINIMUM_UPDATE_INTERVAL=10
+    depends_on:
+     - redis
+  stats:
+    image: matthewbaggett/swarm-agent:stats
+    build:
+      context: .
+      target: swarm-stats
+    volumes:
+      - ./:/app
+    environment:
+      - REDIS_HOST=redis
+    depends_on:
+      - redis
+    ports:
+      - 127.0.0.30:80:80

swarm-monitor/public/index.php

@@ -0,0 +1,41 @@
+<?php
+require_once("../vendor/autoload.php");
+
+$environment = array_merge($_SERVER,$_ENV);
+$redis = new Redis();
+$redis->pconnect(
+    $environment['REDIS_HOST'] ?? "redis",
+$environment['REDIS_PORT'] ?? 6379
+);
+
+$matches = $redis->keys("swarm:*");
+$fleet = [
+    'Status' => "Okay",
+];
+foreach($matches as $match){
+    $machine = explode(":", $match)[1];
+
+    $thisMachine = $redis->hGetAll("swarm:{$machine}");
+
+    foreach(explode(",", $thisMachine['labels']) as $label){
+        $label = empty(trim($label)) ? "Unlabeled" : trim($label);
+        foreach($thisMachine as $key => $value) {
+            switch($key){
+                case "updated_at":
+                case "labels":
+                    break;
+                default:
+                    if(isset($fleet['ByLabel'][$label][$key])){
+                        $fleet['ByLabel'][$label][$key] += $value;
+                    }else{
+                        $fleet['ByLabel'][$label][$key] = $value;
+                    }
+            }
+        }
+    }
+
+    $fleet['Machines'][$machine] = $thisMachine;
+}
+header('Content-type: application/json');
+echo json_encode($fleet);
+