mirror of
https://github.com/lupyuen/lupyuen.github.io.git
synced 2025-01-13 10:18:33 +08:00
1484 lines
No EOL
85 KiB
HTML
1484 lines
No EOL
85 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="generator" content="rustdoc">
|
||
<title>Mastodon Server for Continuous Integration (Apache NuttX RTOS)</title>
|
||
|
||
|
||
<!-- Begin scripts/articles/*-header.html: Article Header for Custom Markdown files processed by rustdoc, like chip8.md -->
|
||
<meta property="og:title"
|
||
content="Mastodon Server for Continuous Integration (Apache NuttX RTOS)"
|
||
data-rh="true">
|
||
<meta property="og:description"
|
||
content="We're out for a 50 km overnight hike. Our Build Farm for Apache NuttX RTOS runs non-stop all day, all night. Continuously compiling over 1,000 NuttX Targets. Can we be 100% sure that NuttX is OK? Without getting spammed by alert emails all night? In this article: We talk about Mastodon as a fun new way to broadcast NuttX Alerts in real time."
|
||
data-rh="true">
|
||
<meta name="description"
|
||
content="We're out for a 50 km overnight hike. Our Build Farm for Apache NuttX RTOS runs non-stop all day, all night. Continuously compiling over 1,000 NuttX Targets. Can we be 100% sure that NuttX is OK? Without getting spammed by alert emails all night? In this article: We talk about Mastodon as a fun new way to broadcast NuttX Alerts in real time.">
|
||
<meta property="og:image"
|
||
content="https://lupyuen.github.io/images/mastodon-register7.png">
|
||
<meta property="og:type"
|
||
content="article" data-rh="true">
|
||
<link rel="canonical"
|
||
href="https://lupyuen.org/articles/mastodon.html" />
|
||
<!-- End scripts/articles/*-header.html -->
|
||
<!-- Begin scripts/rustdoc-header.html: Header for Custom Markdown files processed by rustdoc, like chip8.md -->
|
||
<link rel="alternate" type="application/rss+xml" title="RSS Feed for lupyuen" href="/rss.xml" />
|
||
<link rel="stylesheet" type="text/css" href="../normalize.css">
|
||
<link rel="stylesheet" type="text/css" href="../rustdoc.css" id="mainThemeStyle">
|
||
<link rel="stylesheet" type="text/css" href="../dark.css">
|
||
<link rel="stylesheet" type="text/css" href="../light.css" id="themeStyle">
|
||
<link rel="stylesheet" type="text/css" href="../prism.css">
|
||
<script src="../storage.js"></script><noscript>
|
||
<link rel="stylesheet" href="../noscript.css"></noscript>
|
||
<link rel="shortcut icon" href="../favicon.ico">
|
||
<style type="text/css">
|
||
#crate-search {
|
||
background-image: url("../down-arrow.svg");
|
||
}
|
||
</style>
|
||
<!-- End scripts/rustdoc-header.html -->
|
||
|
||
|
||
</head>
|
||
<body class="rustdoc">
|
||
<!--[if lte IE 8]>
|
||
<div class="warning">
|
||
This old browser is unsupported and will most likely display funky
|
||
things.
|
||
</div>
|
||
<![endif]-->
|
||
|
||
|
||
<!-- Begin scripts/rustdoc-before.html: Pre-HTML for Custom Markdown files processed by rustdoc, like chip8.md -->
|
||
|
||
<!-- Begin Theme Picker -->
|
||
<div class="theme-picker" style="left: 0"><button id="theme-picker" aria-label="Pick another theme!"><img src="../brush.svg"
|
||
width="18" alt="Pick another theme!"></button>
|
||
<div id="theme-choices"></div>
|
||
</div>
|
||
<!-- Theme Picker -->
|
||
|
||
<!-- End scripts/rustdoc-before.html -->
|
||
|
||
|
||
<h1 class="title">Mastodon Server for Continuous Integration (Apache NuttX RTOS)</h1>
|
||
<nav id="rustdoc"><ul>
|
||
<li><a href="#mastodon-for-nuttx-ci" title="Mastodon for NuttX CI">1 Mastodon for NuttX CI</a><ul></ul></li>
|
||
<li><a href="#our-mastodon-server" title="Our Mastodon Server">2 Our Mastodon Server</a><ul></ul></li>
|
||
<li><a href="#bot-user-for-mastodon" title="Bot User for Mastodon">3 Bot User for Mastodon</a><ul></ul></li>
|
||
<li><a href="#email-less-mastodon" title="Email-Less Mastodon">4 Email-Less Mastodon</a><ul></ul></li>
|
||
<li><a href="#post-to-mastodon" title="Post to Mastodon">5 Post to Mastodon</a><ul></ul></li>
|
||
<li><a href="#prometheus-to-mastodon" title="Prometheus to Mastodon">6 Prometheus to Mastodon</a><ul></ul></li>
|
||
<li><a href="#all-toots-considered" title="All Toots Considered">7 All Toots Considered</a><ul></ul></li>
|
||
<li><a href="#whats-next" title="What’s Next">8 What’s Next</a><ul></ul></li>
|
||
<li><a href="#appendix-query-prometheus-for-nuttx-builds" title="Appendix: Query Prometheus for NuttX Builds">9 Appendix: Query Prometheus for NuttX Builds</a><ul></ul></li>
|
||
<li><a href="#appendix-post-nuttx-builds-to-mastodon" title="Appendix: Post NuttX Builds to Mastodon">10 Appendix: Post NuttX Builds to Mastodon</a><ul></ul></li>
|
||
<li><a href="#appendix-install-our-mastodon-server" title="Appendix: Install our Mastodon Server">11 Appendix: Install our Mastodon Server</a><ul></ul></li>
|
||
<li><a href="#appendix-test-our-mastodon-server" title="Appendix: Test our Mastodon Server">12 Appendix: Test our Mastodon Server</a><ul></ul></li>
|
||
<li><a href="#appendix-create-our-mastodon-account" title="Appendix: Create our Mastodon Account">13 Appendix: Create our Mastodon Account</a><ul></ul></li>
|
||
<li><a href="#appendix-create-our-mastodon-app" title="Appendix: Create our Mastodon App">14 Appendix: Create our Mastodon App</a><ul></ul></li>
|
||
<li><a href="#appendix-create-a-mastodon-post" title="Appendix: Create a Mastodon Post">15 Appendix: Create a Mastodon Post</a><ul></ul></li>
|
||
<li><a href="#appendix-backup-our-mastodon-server" title="Appendix: Backup our Mastodon Server">16 Appendix: Backup our Mastodon Server</a><ul></ul></li>
|
||
<li><a href="#appendix-enable-elasticsearch-for-mastodon" title="Appendix: Enable Elasticsearch for Mastodon">17 Appendix: Enable Elasticsearch for Mastodon</a><ul></ul></li>
|
||
<li><a href="#appendix-docker-compose-for-mastodon" title="Appendix: Docker Compose for Mastodon">18 Appendix: Docker Compose for Mastodon</a><ul>
|
||
<li><a href="#database-server" title="Database Server">18.1 Database Server</a><ul></ul></li>
|
||
<li><a href="#web-server" title="Web Server">18.2 Web Server</a><ul></ul></li>
|
||
<li><a href="#redis-server" title="Redis Server">18.3 Redis Server</a><ul></ul></li>
|
||
<li><a href="#sidekiq-server" title="Sidekiq Server">18.4 Sidekiq Server</a><ul></ul></li>
|
||
<li><a href="#streaming-server" title="Streaming Server">18.5 Streaming Server</a><ul></ul></li>
|
||
<li><a href="#elasticsearch-server" title="Elasticsearch Server">18.6 Elasticsearch Server</a><ul></ul></li>
|
||
<li><a href="#volumes-and-networks" title="Volumes and Networks">18.7 Volumes and Networks</a><ul></ul></li>
|
||
<li><a href="#simplest-server-for-mastodon" title="Simplest Server for Mastodon">18.8 Simplest Server for Mastodon</a><ul></ul></li></ul></li></ul></nav><p>📝 <em>29 Dec 2024</em></p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-register7.png" alt="(Experimental) Mastodon Server for Apache NuttX Continuous Integration (macOS Rancher Desktop)" /></p>
|
||
<p>We’re out for an <a href="https://strava.app.link/DDm6627hzPb"><strong>overnight hike</strong></a>, city to airport. Our <a href="https://lupyuen.github.io/articles/ci4"><strong>Build Farm for Apache NuttX RTOS</strong></a> runs non-stop all day, all night. Continuously compiling over <a href="https://lupyuen.github.io/articles/ci#one-thousand-build-targets"><strong>1,000 NuttX Targets</strong></a>: <em>Arm, RISC-V, Xtensa, x64, …</em></p>
|
||
<p>Can we be 100% sure that <strong>NuttX is OK?</strong> Without getting spammed by <strong>alert emails</strong> all night? (Sorry we got zero budget for <em>“paging duty”</em> services)</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-mobile3.png" alt="NuttX Failed Builds appear as Mastodon Alerts" /></p>
|
||
<p>In this article: <strong>Mastodon</strong> (pic above) becomes a fun new way to broadcast NuttX Alerts in real time. We shall…</p>
|
||
<ul>
|
||
<li>
|
||
<p>Install our <strong>Mastodon Server</strong> with Docker Compose (or Rancher Desktop)</p>
|
||
</li>
|
||
<li>
|
||
<p>Create a <strong>Bot User</strong> for pushing Mastodon Alerts</p>
|
||
</li>
|
||
<li>
|
||
<p>Which will work <strong>Without Outgoing Email</strong></p>
|
||
</li>
|
||
<li>
|
||
<p>We fetch the NuttX Builds from <strong>Prometheus Database</strong></p>
|
||
</li>
|
||
<li>
|
||
<p>Post the NuttX Build via <strong>Mastodon API</strong></p>
|
||
</li>
|
||
<li>
|
||
<p>Our Mastodon Server will have <strong>No Local Users</strong></p>
|
||
</li>
|
||
<li>
|
||
<p>But will gladly accept all <strong>Fediverse Users</strong>!</p>
|
||
</li>
|
||
</ul>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-mobile1.png" alt="Following the NuttX Feed on Mastodon" /></p>
|
||
<h1 id="mastodon-for-nuttx-ci"><a class="doc-anchor" href="#mastodon-for-nuttx-ci">§</a>1 Mastodon for NuttX CI</h1>
|
||
<p><em>How to get Mastodon Alerts for NuttX Builds and Continuous Integration? (CI)</em></p>
|
||
<ol>
|
||
<li>
|
||
<p>Register for a <a href="https://joinmastodon.org"><strong>Mastodon Account</strong></a> on any Fediverse Server</p>
|
||
<p>(I got mine at <a href="https://qoto.org"><strong><code>qoto.org</code></strong></a>)</p>
|
||
</li>
|
||
<li>
|
||
<p>On Our Mobile Device: Install a <strong>Mastodon App</strong> and log in</p>
|
||
<p>(Like <a href="https://tusky.app/"><strong>Tusky</strong></a>)</p>
|
||
</li>
|
||
<li>
|
||
<p>Tap the <strong>Search</strong> button. Enter…</p>
|
||
<div class="example-wrap"><pre class="language-text"><code>@nuttx_build@nuttx-feed.org</code></pre></div></li>
|
||
<li>
|
||
<p>Tap the <strong>Accounts</strong> tab. (Pic above)</p>
|
||
<p>Tap the <strong>NuttX Build</strong> account that appears.</p>
|
||
</li>
|
||
<li>
|
||
<p>Tap the <strong>Follow</strong> button. (Pic above)</p>
|
||
<p>And the <strong>Notify</strong> button beside it.</p>
|
||
</li>
|
||
<li>
|
||
<p>That’s all! When a NuttX Build Fails, we’ll see a <strong>Notification in the Mastodon App</strong></p>
|
||
<p>(Which links to NuttX Build History)</p>
|
||
</li>
|
||
</ol>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-mobile3.png" alt="Notification in the Mastodon App links to NuttX Build History" /></p>
|
||
<p><em>How did Mastodon get the Failed Builds?</em></p>
|
||
<p>Thanks to the NuttX Community: We have a (self-hosted) <a href="https://lupyuen.github.io/articles/ci4"><strong>NuttX Build Farm</strong></a> that continuously compiles All NuttX Targets. <em>(1,600 Targets!)</em></p>
|
||
<p>Failed Builds are auto-escalated to our <a href="https://lupyuen.github.io/articles/ci4"><strong>NuttX Dashboard</strong></a>. (Open-source Grafana + Prometheus)</p>
|
||
<p>In a while, we’ll explain how the Failed Builds are channeled from NuttX Dashboard into <strong>Mastodon Posts</strong>.</p>
|
||
<p>First we talk about Mastodon…</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-flow2.jpg" alt="Mastodon Server for Apache NuttX Continuous Integration" /></p>
|
||
<h1 id="our-mastodon-server"><a class="doc-anchor" href="#our-mastodon-server">§</a>2 Our Mastodon Server</h1>
|
||
<p><em>What kind of animal is Mastodon?</em></p>
|
||
<p>Think Twitter… But <strong>Open-Source</strong> and <strong>Self-Hosted</strong>! <em>(Ruby-on-Rails + PostgreSQL + Redis + Elasticsearch)</em> <a href="https://docs.joinmastodon.org/"><strong>Mastodon</strong></a> is mostly used for Global Social Networking on <a href="https://en.wikipedia.org/wiki/Fediverse"><strong>The Fediverse</strong></a>.</p>
|
||
<p>Though today we’re making something unexpected, unconventional with Mastodon: Pushing Notifications of <a href="https://nuttx-feed.org/@nuttx_build"><strong>Failed NuttX Builds</strong></a>.</p>
|
||
<p>(Think: “Social Network for <em>NuttX Maintainers</em>”)</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-register7.png" alt="Mastodon Server for NuttX" /></p>
|
||
<p><em>OK weird flex. How to get started?</em></p>
|
||
<p>We begin by installing our <strong>Mastodon Server with Docker Compose</strong>…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Download the Mastodon Repo
|
||
git clone \
|
||
https://github.com/mastodon/mastodon \
|
||
--branch v4.3.2
|
||
cd mastodon
|
||
echo >.env.production
|
||
|
||
## Patch the Docker Compose Config
|
||
rm docker-compose.yml
|
||
wget https://raw.githubusercontent.com/lupyuen/mastodon/refs/heads/main/docker-compose.yml
|
||
|
||
## Bring Up the Docker Compose (Maybe twice)
|
||
sudo docker compose up
|
||
sudo docker compose up
|
||
|
||
## Omitted: sleep infinity, psql, mastodon:setup, puma, ...</code></pre></div>
|
||
<ul>
|
||
<li>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-install-our-mastodon-server"><strong>“Install our Mastodon Server”</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-test-our-mastodon-server"><strong>“Test our Mastodon Server”</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-enable-elasticsearch-for-mastodon"><strong>“Enable Elasticsearch for Mastodon”</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-docker-compose-for-mastodon"><strong>“Docker Compose for Mastodon”</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p>Based on the excellent <a href="https://docs.joinmastodon.org/admin/prerequisites/"><strong>Mastodon Docs</strong></a></p>
|
||
</li>
|
||
</ul>
|
||
<p>Right now we’re testing on (open-source) <a href="https://rancherdesktop.io/"><strong>macOS Rancher Desktop</strong></a>. Thus we tweaked the steps a bit.</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-containers.png" alt="Mastodon Containers in Rancher Desktop" /></p>
|
||
<h1 id="bot-user-for-mastodon"><a class="doc-anchor" href="#bot-user-for-mastodon">§</a>3 Bot User for Mastodon</h1>
|
||
<p><em>Will we have Users in our Mastodon Server?</em></p>
|
||
<p>Surprisingly, Nope! Our Mastodon Server shall be a tad <strong>Anti-Social</strong>…</p>
|
||
<ul>
|
||
<li>
|
||
<p>We’ll make <strong>One Bot User</strong> <em>(nuttx_build)</em> for posting NuttX Builds</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>No Other Users</strong> on our server, since we’re not really a Social Network</p>
|
||
</li>
|
||
<li>
|
||
<p>But <strong>Users on Other Servers</strong> <em>(like qoto.org)</em> can Follow our Bot User!</p>
|
||
</li>
|
||
<li>
|
||
<p>And receive <strong>Notifications of Failed Builds</strong> through their accounts</p>
|
||
</li>
|
||
<li>
|
||
<p>That’s the power of <a href="https://docs.joinmastodon.org/spec/activitypub/"><strong>Federated ActivityPub</strong></a>!</p>
|
||
</li>
|
||
</ul>
|
||
<p>This is how we create our <strong>Bot User for Mastodon</strong>…</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-register1.png" alt="Create our Mastodon Account" /></p>
|
||
<p>Details in the Appendix…</p>
|
||
<ul>
|
||
<li>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-test-our-mastodon-server"><strong>“Test our Mastodon Server”</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-create-our-mastodon-account"><strong>“Create our Mastodon Account”</strong></a></p>
|
||
</li>
|
||
</ul>
|
||
<p>Things get interesting when we verify our Bot User…</p>
|
||
<h1 id="email-less-mastodon"><a class="doc-anchor" href="#email-less-mastodon">§</a>4 Email-Less Mastodon</h1>
|
||
<p><em>How to verify the Email Address of our Bot User?</em></p>
|
||
<p>Remember our Mastodon Server has <strong>Zero Budget</strong>? This means we won’t have an <strong>Outgoing Email Server</strong>. (SMTP)</p>
|
||
<p>That’s perfectly OK! Mastodon provides <strong>Command-Line Tools</strong> to manage our users…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Connect to Mastodon Web (Docker Container)
|
||
sudo docker exec \
|
||
-it \
|
||
mastodon-web-1 \
|
||
/bin/bash
|
||
|
||
## Approve and Confirm the Email Address
|
||
## https://docs.joinmastodon.org/admin/tootctl/#accounts-approve
|
||
bin/tootctl accounts \
|
||
approve nuttx_build
|
||
bin/tootctl accounts \
|
||
modify nuttx_build \
|
||
--confirm</code></pre></div>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-create-our-mastodon-account">(Explained here)</a></p>
|
||
<h1 id="post-to-mastodon"><a class="doc-anchor" href="#post-to-mastodon">§</a>5 Post to Mastodon</h1>
|
||
<p><em>How will our Bot post a message to Mastodon?</em></p>
|
||
<p><strong>With curl:</strong> Here’s how we post a <strong>Status Update</strong> to Mastodon…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Set the Mastodon Access Token (see below)
|
||
ACCESS_TOKEN=...
|
||
|
||
## Post a message to Mastodon (Status Update)
|
||
curl -X POST \
|
||
-H "Authorization: Bearer $ACCESS_TOKEN" \
|
||
-F "status=Posting a status from curl" \
|
||
https://YOUR_DOMAIN_NAME.org/api/v1/statuses</code></pre></div>
|
||
<p>It appears like so…</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-web4.png" alt="Post a message to Mastodon (Status Update)" /></p>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-create-a-mastodon-post">(Explained here)</a></p>
|
||
<p><em>What’s this Access Token?</em></p>
|
||
<p>To Authenticate our Bot User with Mastodon API, we pass an <strong>Access Token</strong>. This is how we create the Access Token…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Set the Client ID, Secret and Authorization Code (see below)
|
||
CLIENT_ID=...
|
||
CLIENT_SECRET=...
|
||
AUTH_CODE=...
|
||
|
||
## Create an Access Token
|
||
curl -X POST \
|
||
-F "client_id=$CLIENT_ID" \
|
||
-F "client_secret=$CLIENT_SECRET" \
|
||
-F "redirect_uri=urn:ietf:wg:oauth:2.0:oob" \
|
||
-F "grant_type=authorization_code" \
|
||
-F "code=$AUTH_CODE" \
|
||
-F "scope=read write push" \
|
||
https://YOUR_DOMAIN_NAME.org/oauth/token</code></pre></div>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-create-our-mastodon-app">(Explained here)</a></p>
|
||
<p><em>What about the Client ID, Secret and Authorization Code?</em></p>
|
||
<p><strong>Client ID and Secret</strong> will specify the Mastodon App for our Bot User. Here’s how we create our <strong>Mastodon App</strong> for NuttX Dashboard…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Create Our Mastodon App
|
||
curl -X POST \
|
||
-F 'client_name=NuttX Dashboard' \
|
||
-F 'redirect_uris=urn:ietf:wg:oauth:2.0:oob' \
|
||
-F 'scopes=read write push' \
|
||
-F 'website=https://nuttx-dashboard.org' \
|
||
https://YOUR_DOMAIN_NAME.org/api/v1/apps
|
||
|
||
## Returns { "client_id" : "...", "client_secret" : "..." }
|
||
## We save the Client ID and Secret</code></pre></div>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-create-our-mastodon-app">(Explained here)</a></p>
|
||
<p>Which we use to create the <strong>Authorization Code</strong>…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Open a Web Browser. Browse to https://YOUR_DOMAIN_NAME.org
|
||
## Log in as Your New User (nuttx_build)
|
||
## Paste this URL into the Same Web Browser
|
||
https://YOUR_DOMAIN_NAME.org/oauth/authorize
|
||
?client_id=YOUR_CLIENT_ID
|
||
&scope=read+write+push
|
||
&redirect_uri=urn:ietf:wg:oauth:2.0:oob
|
||
&response_type=code
|
||
|
||
## Copy the Authorization Code. It will expire soon!</code></pre></div>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-create-our-mastodon-app">(Explained here)</a></p>
|
||
<h1 id="prometheus-to-mastodon"><a class="doc-anchor" href="#prometheus-to-mastodon">§</a>6 Prometheus to Mastodon</h1>
|
||
<p>Now comes the tricky bit. How to transmogrify <a href="https://nuttx-dashboard.org"><strong>NuttX Dashboard</strong></a>…</p>
|
||
<p><img src="https://lupyuen.github.io/images/ci7-dashboard.png" alt="NuttX Dashboard" /></p>
|
||
<p>Into <a href="https://nuttx-feed.org/@nuttx_build"><strong>Mastodon Posts</strong></a>?</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-register7.png" alt="NuttX Builds in Mastodon" /></p>
|
||
<p>Here comes our Grand Plan…</p>
|
||
<ol>
|
||
<li>
|
||
<p><strong>Outcomes of NuttX Builds</strong> are already recorded…</p>
|
||
</li>
|
||
<li>
|
||
<p>Inside our <strong>Prometheus Time-Series Database</strong> (open-source)</p>
|
||
</li>
|
||
<li>
|
||
<p>Thus we <strong>Query the Failed Builds</strong> from Prometheus Database</p>
|
||
</li>
|
||
<li>
|
||
<p>Reformat them as <strong>Mastodon Posts</strong></p>
|
||
</li>
|
||
<li>
|
||
<p>Post the Failed Builds via <strong>Mastodon API</strong></p>
|
||
</li>
|
||
</ol>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-flow.jpg" alt="Mastodon Server for Apache NuttX Continuous Integration" /></p>
|
||
<hr>
|
||
<p><strong>Prometheus Time-Series Database:</strong> This query will fetch the Failed Builds from Prometheus…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Find all Build Scores < 0.5
|
||
build_score < 0.5</code></pre></div>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-query-prometheus-for-nuttx-builds">(Explained here)</a></p>
|
||
<p>Prometheus returns a huge bunch of fields, we’ll tweak this…</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-prometheus.png" alt="Fetching the Failed NuttX Builds from Prometheus" /></p>
|
||
<hr>
|
||
<p><strong>Query the Failed Builds:</strong> We repeat the above, but in Rust: <a href="https://github.com/lupyuen/nuttx-prometheus-to-mastodon/blob/main/src/main.rs#L32-L70">main.rs</a></p>
|
||
|
||
<div class="example-wrap"><pre class="rust rust-example-rendered"><code><span class="comment">// Fetch the Failed Builds from Prometheus
|
||
</span><span class="kw">let </span>query = <span class="string">r##"
|
||
build_score < 0.5
|
||
"##</span>;
|
||
<span class="kw">let </span>params = [(<span class="string">"query"</span>, query)];
|
||
<span class="kw">let </span>client = reqwest::Client::new();
|
||
<span class="kw">let </span>prometheus = <span class="string">"http://localhost:9090/api/v1/query"</span>;
|
||
<span class="kw">let </span>res = client
|
||
.post(prometheus)
|
||
.form(<span class="kw-2">&</span>params)
|
||
.send()
|
||
.<span class="kw">await</span><span class="question-mark">?</span>;
|
||
<span class="kw">let </span>body = res.text().<span class="kw">await</span><span class="question-mark">?</span>;
|
||
<span class="kw">let </span>data: Value = serde_json::from_str(<span class="kw-2">&</span>body).unwrap();
|
||
<span class="kw">let </span>builds = <span class="kw-2">&</span>data[<span class="string">"data"</span>][<span class="string">"result"</span>];</code></pre></div>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-query-prometheus-for-nuttx-builds">(Explained here)</a></p>
|
||
<hr>
|
||
<p><strong>Reformat as Mastodon Posts:</strong> We turn JSON into Plain Text: <a href="https://github.com/lupyuen/nuttx-prometheus-to-mastodon/blob/main/src/main.rs#L78-L111">main.rs</a></p>
|
||
|
||
<div class="example-wrap"><pre class="rust rust-example-rendered"><code><span class="comment">// For Each Failed Build...
|
||
</span><span class="kw">for </span>build <span class="kw">in </span>builds.as_array().unwrap() {
|
||
...
|
||
<span class="comment">// Compose the Mastodon Post as...
|
||
// rv-virt : CITEST - Build Failed (NuttX)
|
||
// NuttX Dashboard: ...
|
||
// Build History: ...
|
||
// [Error Message]
|
||
</span><span class="kw">let </span><span class="kw-2">mut </span>status = <span class="macro">format!</span>(
|
||
<span class="string">r##"
|
||
{board} : {config_upper} - Build Failed ({user})
|
||
NuttX Dashboard: https://nuttx-dashboard.org
|
||
Build History: https://nuttx-dashboard.org/d/fe2q876wubc3kc/nuttx-build-history?var-board={board}&var-config={config}
|
||
|
||
{msg}
|
||
"##</span>);
|
||
status.truncate(<span class="number">512</span>); <span class="comment">// Mastodon allows only 500 chars
|
||
</span><span class="kw">let </span><span class="kw-2">mut </span>params = Vec::new();
|
||
params.push((<span class="string">"status"</span>, status));</code></pre></div>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-post-nuttx-builds-to-mastodon">(Explained here)</a></p>
|
||
<hr>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-flow3.jpg" alt="Prometheus to Mastodon" /></p>
|
||
<p><strong>Post via Mastodon API:</strong> By creating a Status Update: <a href="https://github.com/lupyuen/nuttx-prometheus-to-mastodon/blob/main/src/main.rs#L126-L148">main.rs</a></p>
|
||
|
||
<div class="example-wrap"><pre class="rust rust-example-rendered"><code> <span class="comment">// Post to Mastodon
|
||
</span><span class="kw">let </span>token = std::env::var(<span class="string">"MASTODON_TOKEN"</span>)
|
||
.expect(<span class="string">"MASTODON_TOKEN env variable is required"</span>);
|
||
<span class="kw">let </span>client = reqwest::Client::new();
|
||
<span class="kw">let </span>mastodon = <span class="string">"https://nuttx-feed.org/api/v1/statuses"</span>;
|
||
<span class="kw">let </span>res = client
|
||
.post(mastodon)
|
||
.header(<span class="string">"Authorization"</span>, <span class="macro">format!</span>(<span class="string">"Bearer {token}"</span>))
|
||
.form(<span class="kw-2">&</span>params)
|
||
.send()
|
||
.<span class="kw">await</span><span class="question-mark">?</span>;
|
||
<span class="kw">if </span>!res.status().is_success() { <span class="kw">continue</span>; }
|
||
<span class="comment">// Omitted: Remember the Mastodon Posts for All Builds
|
||
</span>}</code></pre></div>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-post-nuttx-builds-to-mastodon">(Explained here)</a></p>
|
||
<hr>
|
||
<p><strong>Skip Duplicates:</strong> We remember everything in a JSON File, so we won’t notify the same thing twice: <a href="https://github.com/lupyuen/nuttx-prometheus-to-mastodon/blob/main/src/main.rs#L111-L126">main.rs</a></p>
|
||
|
||
<div class="example-wrap"><pre class="rust rust-example-rendered"><code><span class="comment">// This JSON File remembers the Mastodon Posts for All Builds:
|
||
// {
|
||
// "rv-virt:citest" : {
|
||
// status_id: "12345",
|
||
// users: ["nuttxpr", "NuttX", "lupyuen"]
|
||
// }
|
||
// "rv-virt:citest64" : ...
|
||
// }
|
||
</span><span class="kw">const </span>ALL_BUILDS_FILENAME: <span class="kw-2">&</span>str =
|
||
<span class="string">"/tmp/nuttx-prometheus-to-mastodon.json"</span>; ...
|
||
<span class="kw">let </span><span class="kw-2">mut </span>all_builds = serde_json::from_reader(reader).unwrap();
|
||
...
|
||
<span class="comment">// If the User already exists for the Board and Config:
|
||
// Skip the Mastodon Post
|
||
</span><span class="kw">if let </span><span class="prelude-val">Some</span>(users) = all_builds[<span class="kw-2">&</span>target][<span class="string">"users"</span>].as_array() {
|
||
<span class="kw">if </span>users.contains(<span class="kw-2">&</span><span class="macro">json!</span>(user)) { <span class="kw">continue</span>; }
|
||
}</code></pre></div>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-post-nuttx-builds-to-mastodon">(Explained here)</a></p>
|
||
<p>And we’re done! <a href="https://lupyuen.github.io/articles/mastodon#appendix-post-nuttx-builds-to-mastodon"><strong>The Appendix</strong></a> explains how we thread the Mastodon Posts neatly by <strong>NuttX Target</strong>. (Board + Config)</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-register7.png" alt="NuttX Builds threaded neatly" /></p>
|
||
<h1 id="all-toots-considered"><a class="doc-anchor" href="#all-toots-considered">§</a>7 All Toots Considered</h1>
|
||
<ol>
|
||
<li>
|
||
<p><em>Will we accept Regular Users on our Mastodon Server?</em></p>
|
||
<p>Probably not? We have <strong>Zero Budget for User Moderation</strong>. Instead we’ll ask NuttX Devs to register for an account on any Fediverse Server. The Push Notifications for Failed Builds will work fine with any server.</p>
|
||
</li>
|
||
<li>
|
||
<p><em>But any Fediverse User can reply to our Mastodon Posts?</em></p>
|
||
<p>Yeah this might be helpful! NuttX Devs can discuss a specific Failed Build. Or hyperlink to the <a href="https://github.com/apache/nuttx/issues"><strong>NuttX Issue</strong></a> that was created for the Failed Build. Which might prevent <a href="https://github.com/apache/nuttx/pull/15382"><strong>Conflicting PRs</strong></a>. <a href="https://github.com/apache/nuttx/pull/15388">(And another)</a></p>
|
||
</li>
|
||
<li>
|
||
<p><em>How will we know when a Failed Build recovers?</em></p>
|
||
<p>This gets tricky. Should we pester folks with an <strong>Extra Push Notification</strong> whenever a Failed Build recovers?</p>
|
||
<p>For Complex Notifications: We might integrate <a href="https://prometheus.io/docs/alerting/latest/alertmanager/"><strong>Prometheus Alertmanager</strong></a> with Mastodon.</p>
|
||
</li>
|
||
<li>
|
||
<p><em>Suppose I’m interested only in rv-virt:python. Can I subscribe to the Specific Alert via Mastodon / Fediverse / ActivityPub?</em></p>
|
||
<p>Good question! We’re still trying to figure out.</p>
|
||
</li>
|
||
<li>
|
||
<p><em>Anything else we should monitor with Mastodon?</em></p>
|
||
<p><a href="https://lupyuen.github.io/articles/ci3#move-the-merge-jobs"><strong>Sync-Build-Ingest</strong></a> is a Critical NuttX Job that needs to run non-stop, without fail. We should post a Mastodon Notification if something fails to run.</p>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#prometheus-to-mastodon"><strong>Watching the Watchmen:</strong></a> How to be sure that our Rust App runs forever, always pushing Mastodon Alerts?</p>
|
||
<p><a href="https://lupyuen.github.io/articles/ci3#live-metric-for-full-time-runners"><strong>Cost of GitHub Runners</strong></a> shall be continuously monitored. We should push a Mastodon Alert if it exceeds our budget. (Before ASF comes after us)</p>
|
||
<p><a href="https://lupyuen.github.io/articles/ci3#present-pains"><strong>Over-Running GitHub Jobs</strong></a> shall also be monitored, so our (beloved and respected) NuttX Devs won’t wait forever for our CI Jobs to complete. Mastodon sounds mightly helpful for watching over Everything NuttX! 👍</p>
|
||
</li>
|
||
<li>
|
||
<p><em>How is Mastodon working out so far?</em></p>
|
||
<p>I’m trying to do the least possible work to get meaningful NuttX CI Alerts (since I’m doing this in my spare time). Mastodon works great for me right now!</p>
|
||
<p>I’m not sure if anyone else will use it, so I’ll stick with this setup for now. (I might disconnect from the Fediverse if I hear any complaints)</p>
|
||
</li>
|
||
</ol>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-flow.jpg" alt="Mastodon Server for Apache NuttX Continuous Integration" /></p>
|
||
<h1 id="whats-next"><a class="doc-anchor" href="#whats-next">§</a>8 What’s Next</h1>
|
||
<p>Next Article: We talk about <strong>Git Bisect</strong> and how we auto-magically discover a Breaking Commit in NuttX.</p>
|
||
<ul>
|
||
<li><a href="https://lupyuen.org/articles/bisect.html"><strong>“Git Bisecting a Bug (Apache NuttX RTOS)”</strong></a></li>
|
||
</ul>
|
||
<p>After That: What would NuttX Life be like without GitHub? We try out (self-hosted open-source) <strong>Forgejo Git Forge</strong> with NuttX.</p>
|
||
<p>After After That? Why <strong>Sync-Build-Ingest</strong> is super important for NuttX CI. And how we monitor it with our <strong>Magic Disco Light</strong>.</p>
|
||
<p>Also: Since we can <strong>Rewind NuttX Builds</strong> and automatically <strong>Git Bisect</strong>… Can we create a Bot that will fish the <strong>Failed Builds from NuttX Dashboard</strong>, identify the Breaking PR, and escalate to the right folks via Mastodon?</p>
|
||
<p>Many Thanks to the awesome <strong>NuttX Admins</strong> and <strong>NuttX Devs</strong>! And <a href="https://lupyuen.github.io/articles/sponsor"><strong>My Sponsors</strong></a>, for sticking with me all these years.</p>
|
||
<ul>
|
||
<li>
|
||
<p><a href="https://lupyuen.github.io/articles/sponsor"><strong>Sponsor me a coffee</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="https://news.ycombinator.com/item?id=42534224"><strong>Discuss this article on Hacker News</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="https://github.com/lupyuen/nuttx-sg2000"><strong>My Current Project: “Apache NuttX RTOS for Sophgo SG2000”</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="https://github.com/lupyuen/nuttx-ox64"><strong>My Other Project: “NuttX for Ox64 BL808”</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="https://github.com/lupyuen/nuttx-star64"><strong>Older Project: “NuttX for Star64 JH7110”</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="https://github.com/lupyuen/pinephone-nuttx"><strong>Olderer Project: “NuttX for PinePhone”</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="https://lupyuen.github.io"><strong>Check out my articles</strong></a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="https://lupyuen.github.io/rss.xml"><strong>RSS Feed</strong></a></p>
|
||
</li>
|
||
</ul>
|
||
<p><em>Got a question, comment or suggestion? Create an Issue or submit a Pull Request here…</em></p>
|
||
<p><a href="https://github.com/lupyuen/lupyuen.github.io/blob/master/src/mastodon.md"><strong>lupyuen.org/src/mastodon.md</strong></a></p>
|
||
<p><img src="https://lupyuen.github.io/images/ci4-thinkstation.jpg" alt="Hefty Ubuntu Xeon Workstation for NuttX Build Farm" /></p>
|
||
<span style="font-size:90%">
|
||
<p><a href="https://qoto.org/@lupyuen/113517788288458811"><em>Hefty Ubuntu Xeon Workstation for NuttX Build Farm</em></a></p>
|
||
</span>
|
||
<h1 id="appendix-query-prometheus-for-nuttx-builds"><a class="doc-anchor" href="#appendix-query-prometheus-for-nuttx-builds">§</a>9 Appendix: Query Prometheus for NuttX Builds</h1>
|
||
<p><a href="https://lupyuen.github.io/articles/ci4"><strong>NuttX Build Farm</strong></a> (pic above) runs non-stop all day, all night. Continuously compiling over <a href="https://lupyuen.github.io/articles/ci#one-thousand-build-targets"><strong>1,000 NuttX Targets</strong></a>.</p>
|
||
<p>Outcomes of NuttX Builds are recorded inside our <strong>Prometheus Time-Series Database</strong>…</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-flow3.jpg" alt="Prometheus to Mastodon" /></p>
|
||
<p>To fetch the <strong>Failed NuttX Builds</strong> from Prometheus: We browse to Prometheus at <em>http://localhost:9090</em> and enter this <strong>Prometheus Query</strong>…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Find all Build Scores < 0.5
|
||
## But skip these users...
|
||
build_score{
|
||
user != "rewind", ## Used for Build Rewind only
|
||
user != "nuttxlinux", ## Retired (Blocked by GitHub)
|
||
user != "nuttxmacos" ## Retired (Blocked by GitHub)
|
||
} < 0.5</code></pre></div>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-prometheus.png" alt="Fetching the Failed NuttX Builds from Prometheus" /></p>
|
||
<p><em>Why 0.5?</em></p>
|
||
<p>Build Score is 1.0 for Successful Builds, 0.5 for Warnings, 0.0 for Errors. Thus we search for Build Scores < 0.5.</p>
|
||
<div><table><thead><tr><th style="text-align: center">Score</th><th style="text-align: left">Status</th><th style="text-align: left">Example</th></tr></thead><tbody>
|
||
<tr><td style="text-align: center"><strong><code>0.0</code></strong></td><td style="text-align: left">Error</td><td style="text-align: left"><em>undefined reference to atomic_fetch_add_2</em></td></tr>
|
||
<tr><td style="text-align: center"><strong><code>0.5</code></strong></td><td style="text-align: left">Warning</td><td style="text-align: left"> <em>nuttx has a LOAD segment with RWX permission</em></td></tr>
|
||
<tr><td style="text-align: center"><strong><code>0.8</code></strong></td><td style="text-align: left">Unknown</td><td style="text-align: left"><em>STM32_USE_LEGACY_PINMAP will be deprecated</em></td></tr>
|
||
<tr><td style="text-align: center"><strong><code>1.0</code></strong></td><td style="text-align: left">Success</td><td style="text-align: left"><em>(No Errors and Warnings)</em></td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p><em>What’s returned by Prometheus?</em></p>
|
||
<p>Plenty of fields, describing <a href="https://lupyuen.github.io/articles/ci4#prometheus-metrics"><strong>Every Failed Build</strong></a> in detail (pic above)…</p>
|
||
<span style="font-size:90%">
|
||
<div><table><thead><tr><th style="text-align: left">Field</th><th style="text-align: left">Value</th></tr></thead><tbody>
|
||
<tr><td style="text-align: left"><strong>timestamp</strong></td><td style="text-align: left">Timestamp <em>(2024-12-06T06:14:54)</em></td></tr>
|
||
<tr><td style="text-align: left"><strong>version</strong></td><td style="text-align: left">Always 3</td></tr>
|
||
<tr><td style="text-align: left"><strong>user</strong></td><td style="text-align: left">Which Build PC <em>(nuttxmacos)</em></td></tr>
|
||
<tr><td style="text-align: left"><strong>arch</strong></td><td style="text-align: left">Architecture <em>(risc-v)</em></td></tr>
|
||
<tr><td style="text-align: left"><strong>group</strong></td><td style="text-align: left">Target Group <em>(risc-v-01)</em></td></tr>
|
||
<tr><td style="text-align: left"><strong>board</strong></td><td style="text-align: left">Board <em>(ox64)</em></td></tr>
|
||
<tr><td style="text-align: left"><strong>config</strong></td><td style="text-align: left">Config <em>(nsh)</em></td></tr>
|
||
<tr><td style="text-align: left"><strong>target</strong></td><td style="text-align: left">Board:Config <em>(ox64:nsh)</em></td></tr>
|
||
<tr><td style="text-align: left"><strong>subarch</strong></td><td style="text-align: left">Sub-Architecture <em>(bl808)</em></td></tr>
|
||
<tr><td style="text-align: left"><strong>url</strong></td><td style="text-align: left">Full URL of Build Log</td></tr>
|
||
<tr><td style="text-align: left"><strong>url_display</strong></td><td style="text-align: left">Short URL of Build Log</td></tr>
|
||
<tr><td style="text-align: left"><strong>nuttx_hash</strong></td><td style="text-align: left">Commit Hash of NuttX Repo <em>(7f84a64109f94787d92c2f44465e43fde6f3d28f)</em></td></tr>
|
||
<tr><td style="text-align: left"><strong>apps_hash</strong></td><td style="text-align: left">Commit Hash of NuttX Apps <em>(d6edbd0cec72cb44ceb9d0f5b932cbd7a2b96288)</em></td></tr>
|
||
<tr><td style="text-align: left"><strong>msg</strong></td><td style="text-align: left">Error or Warning Message</td></tr>
|
||
</tbody></table>
|
||
</div></span>
|
||
<p>We can do the same with curl and <strong>HTTP POST</strong>…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>$ curl -X POST \
|
||
-F 'query=
|
||
build_score{
|
||
user != "rewind",
|
||
user != "nuttxlinux",
|
||
user != "nuttxmacos"
|
||
} < 0.5
|
||
' \
|
||
http://localhost:9090/api/v1/query
|
||
|
||
{"status" : "success", "data" : {"resultType" : "vector", "result" : [{"metric"{
|
||
"__name__" : "build_score",
|
||
"timestamp" : "2024-12-06T06:14:54",
|
||
"user" : "nuttxpr",
|
||
"nuttx_hash": "04815338334e63cd82c38ee12244e54829766e88",
|
||
"apps_hash" : "b08c29617bbf1f2c6227f74e23ffdd7706997e0c",
|
||
"arch" : "risc-v",
|
||
"subarch" : "qemu-rv",
|
||
"board" : "rv-virt",
|
||
"config" : "citest",
|
||
"msg" : "virtio/virtio-mmio.c: In function
|
||
'virtio_mmio_config_virtqueue': \n virtio/virtio-mmio.c:346:14:
|
||
error: cast from pointer to integer of different size ...</code></pre></div>
|
||
<p>In the next section: We’ll replicate this with Rust.</p>
|
||
<p><em>How did we get the above Prometheus Query?</em></p>
|
||
<p>We copied and pasted from our <a href="https://lupyuen.github.io/articles/ci4#grafana-dashboard"><strong>NuttX Dashboard in Grafana</strong></a>…</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-grafana.png" alt="Prometheus Query from our NuttX Dashboard in Grafana" /></p>
|
||
<h1 id="appendix-post-nuttx-builds-to-mastodon"><a class="doc-anchor" href="#appendix-post-nuttx-builds-to-mastodon">§</a>10 Appendix: Post NuttX Builds to Mastodon</h1>
|
||
<p>In the previous section: We fetched the <strong>Failed NuttX Builds</strong> from Prometheus. Now we post them to <strong>Mastodon</strong>: <a href="https://github.com/lupyuen/nuttx-prometheus-to-mastodon/blob/main/run.sh">run.sh</a></p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Set the Access Token for Mastodon
|
||
## https://docs.joinmastodon.org/client/authorized/#token
|
||
## export MASTODON_TOKEN=...
|
||
. ../mastodon-token.sh
|
||
|
||
## Do this forever...
|
||
for (( ; ; )); do
|
||
|
||
## Post the Failed Jobs from Prometheus to Mastodon
|
||
cargo run
|
||
|
||
## Wait a while
|
||
date ; sleep 900
|
||
|
||
## Omitted: Copy the Failed Builds to
|
||
## https://lupyuen.org/nuttx-prometheus-to-mastodon.json
|
||
done</code></pre></div>
|
||
<p><a href="https://gist.github.com/lupyuen/37afa9feed4e6eb983845a8c3d500d40">(See the <strong>Complete Log</strong>)</a></p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-flow3.jpg" alt="Prometheus to Mastodon" /></p>
|
||
<p>Inside our Rust App, we fetch the <strong>Failed Builds from Prometheus</strong>: <a href="https://github.com/lupyuen/nuttx-prometheus-to-mastodon/blob/main/src/main.rs#L32-L70">main.rs</a></p>
|
||
|
||
<div class="example-wrap"><pre class="rust rust-example-rendered"><code><span class="comment">// Fetch the Failed Builds from Prometheus
|
||
</span><span class="kw">let </span>query = <span class="string">r##"
|
||
build_score{
|
||
user!="rewind",
|
||
user!="nuttxlinux",
|
||
user!="nuttxmacos"
|
||
} < 0.5
|
||
"##</span>;
|
||
<span class="kw">let </span>params = [(<span class="string">"query"</span>, query)];
|
||
<span class="kw">let </span>client = reqwest::Client::new();
|
||
<span class="kw">let </span>prometheus = <span class="string">"http://localhost:9090/api/v1/query"</span>;
|
||
<span class="kw">let </span>res = client
|
||
.post(prometheus)
|
||
.form(<span class="kw-2">&</span>params)
|
||
.send()
|
||
.<span class="kw">await</span><span class="question-mark">?</span>;
|
||
<span class="kw">let </span>body = res.text().<span class="kw">await</span><span class="question-mark">?</span>;
|
||
<span class="kw">let </span>data: Value = serde_json::from_str(<span class="kw-2">&</span>body).unwrap();
|
||
<span class="kw">let </span>builds = <span class="kw-2">&</span>data[<span class="string">"data"</span>][<span class="string">"result"</span>];</code></pre></div>
|
||
<p><strong>For Every Failed Build:</strong> We compose the <strong>Mastodon Post</strong>: <a href="https://github.com/lupyuen/nuttx-prometheus-to-mastodon/blob/main/src/main.rs#L78-L111">main.rs</a></p>
|
||
|
||
<div class="example-wrap"><pre class="rust rust-example-rendered"><code><span class="comment">// For Each Failed Build...
|
||
</span><span class="kw">for </span>build <span class="kw">in </span>builds.as_array().unwrap() {
|
||
...
|
||
<span class="comment">// Compose the Mastodon Post as...
|
||
// rv-virt : CITEST - Build Failed (NuttX)
|
||
// NuttX Dashboard: ...
|
||
// Build History: ...
|
||
// [Error Message]
|
||
</span><span class="kw">let </span><span class="kw-2">mut </span>status = <span class="macro">format!</span>(
|
||
<span class="string">r##"
|
||
{board} : {config_upper} - Build Failed ({user})
|
||
NuttX Dashboard: https://nuttx-dashboard.org
|
||
Build History: https://nuttx-dashboard.org/d/fe2q876wubc3kc/nuttx-build-history?var-board={board}&var-config={config}
|
||
|
||
{msg}
|
||
"##</span>);
|
||
status.truncate(<span class="number">512</span>); <span class="comment">// Mastodon allows only 500 chars
|
||
</span><span class="kw">let </span><span class="kw-2">mut </span>params = Vec::new();
|
||
params.push((<span class="string">"status"</span>, status));</code></pre></div>
|
||
<p>And we <strong>post to Mastodon</strong>: <a href="https://github.com/lupyuen/nuttx-prometheus-to-mastodon/blob/main/src/main.rs#L126-L148">main.rs</a></p>
|
||
|
||
<div class="example-wrap"><pre class="rust rust-example-rendered"><code> <span class="comment">// Post to Mastodon
|
||
</span><span class="kw">let </span>token = std::env::var(<span class="string">"MASTODON_TOKEN"</span>)
|
||
.expect(<span class="string">"MASTODON_TOKEN env variable is required"</span>);
|
||
<span class="kw">let </span>client = reqwest::Client::new();
|
||
<span class="kw">let </span>mastodon = <span class="string">"https://nuttx-feed.org/api/v1/statuses"</span>;
|
||
<span class="kw">let </span>res = client
|
||
.post(mastodon)
|
||
.header(<span class="string">"Authorization"</span>, <span class="macro">format!</span>(<span class="string">"Bearer {token}"</span>))
|
||
.form(<span class="kw-2">&</span>params)
|
||
.send()
|
||
.<span class="kw">await</span><span class="question-mark">?</span>;
|
||
<span class="kw">if </span>!res.status().is_success() { <span class="kw">continue</span>; }
|
||
<span class="comment">// Omitted: Remember the Mastodon Posts for All Builds
|
||
</span>}</code></pre></div>
|
||
<p><em>Won’t we see repeated Mastodon Posts?</em></p>
|
||
<p>That’s why we <strong>Remember the Mastodon Posts</strong> for All Builds, in a JSON File: <a href="https://github.com/lupyuen/nuttx-prometheus-to-mastodon/blob/main/src/main.rs#L16-L78">main.rs</a></p>
|
||
|
||
<div class="example-wrap"><pre class="rust rust-example-rendered"><code><span class="comment">// Remembers the Mastodon Posts for All Builds:
|
||
// {
|
||
// "rv-virt:citest" : {
|
||
// status_id: "12345",
|
||
// users: ["nuttxpr", "NuttX", "lupyuen"]
|
||
// }
|
||
// "rv-virt:citest64" : ...
|
||
// }
|
||
</span><span class="kw">const </span>ALL_BUILDS_FILENAME: <span class="kw-2">&</span>str =
|
||
<span class="string">"/tmp/nuttx-prometheus-to-mastodon.json"</span>;
|
||
...
|
||
<span class="comment">// Load the Mastodon Posts for All Builds
|
||
</span><span class="kw">let </span><span class="kw-2">mut </span>all_builds = <span class="macro">json!</span>({});
|
||
<span class="kw">if let </span><span class="prelude-val">Ok</span>(file) = File::open(ALL_BUILDS_FILENAME) {
|
||
<span class="kw">let </span>reader = BufReader::new(file);
|
||
all_builds = serde_json::from_reader(reader).unwrap();
|
||
}</code></pre></div>
|
||
<p>If the User already exists for the Board and Config: We <strong>Skip the Mastodon Post</strong>: <a href="https://github.com/lupyuen/nuttx-prometheus-to-mastodon/blob/main/src/main.rs#L111-L126">main.rs</a></p>
|
||
|
||
<div class="example-wrap"><pre class="rust rust-example-rendered"><code><span class="comment">// If the Mastodon Post already exists for Board and Config:
|
||
// Reply to the Mastodon Post
|
||
</span><span class="kw">if let </span><span class="prelude-val">Some</span>(status_id) = all_builds[<span class="kw-2">&</span>target][<span class="string">"status_id"</span>].as_str() {
|
||
params.push((<span class="string">"in_reply_to_id"</span>, status_id.to_string()));
|
||
|
||
<span class="comment">// If the User already exists for the Board and Config:
|
||
// Skip the Mastodon Post
|
||
</span><span class="kw">if let </span><span class="prelude-val">Some</span>(users) = all_builds[<span class="kw-2">&</span>target][<span class="string">"users"</span>].as_array() {
|
||
<span class="kw">if </span>users.contains(<span class="kw-2">&</span><span class="macro">json!</span>(user)) { <span class="kw">continue</span>; }
|
||
}
|
||
}</code></pre></div>
|
||
<p>And if the Mastodon Post already exists for the Board and Config: We <strong>Reply to the Mastodon Post</strong>. (To keep the Failed Builds threaded neatly, pic below)</p>
|
||
<p>This is how we <strong>Remember the Mastodon Post ID</strong> (Status ID): <a href="https://github.com/lupyuen/nuttx-prometheus-to-mastodon/blob/main/src/main.rs#L148-L171">main.rs</a></p>
|
||
|
||
<div class="example-wrap"><pre class="rust rust-example-rendered"><code><span class="comment">// Remember the Mastodon Post ID (Status ID)
|
||
</span><span class="kw">let </span>body = res.text().<span class="kw">await</span><span class="question-mark">?</span>;
|
||
<span class="kw">let </span>status: Value = serde_json::from_str(<span class="kw-2">&</span>body).unwrap();
|
||
<span class="kw">let </span>status_id = status[<span class="string">"id"</span>].as_str().unwrap();
|
||
all_builds[<span class="kw-2">&</span>target][<span class="string">"status_id"</span>] = status_id.into();
|
||
|
||
<span class="comment">// Append the User to All Builds
|
||
</span><span class="kw">if let </span><span class="prelude-val">Some</span>(users) = all_builds[<span class="kw-2">&</span>target][<span class="string">"users"</span>].as_array() {
|
||
<span class="kw">if </span>!users.contains(<span class="kw-2">&</span><span class="macro">json!</span>(user)) {
|
||
<span class="kw">let </span><span class="kw-2">mut </span>users = users.clone();
|
||
users.push(<span class="macro">json!</span>(user));
|
||
all_builds[<span class="kw-2">&</span>target][<span class="string">"users"</span>] = <span class="macro">json!</span>(users);
|
||
}
|
||
} <span class="kw">else </span>{
|
||
all_builds[<span class="kw-2">&</span>target][<span class="string">"users"</span>] = <span class="macro">json!</span>([user]);
|
||
}
|
||
|
||
<span class="comment">// Save the Mastodon Posts for All Builds
|
||
</span><span class="kw">let </span>json = to_string_pretty(<span class="kw-2">&</span>all_builds).unwrap();
|
||
<span class="kw">let </span><span class="kw-2">mut </span>file = File::create(ALL_BUILDS_FILENAME).unwrap();
|
||
file.write_all(json.as_bytes()).unwrap();</code></pre></div>
|
||
<p>Which gets saved into a <strong>JSON File of Failed Builds</strong>, published here every 15 mins: <a href="https://lupyuen.org/nuttx-prometheus-to-mastodon.json"><em>lupyuen.org/nuttx-prometheus-to-mastodon.json</em></a></p>
|
||
<p><a href="https://gist.github.com/lupyuen/37afa9feed4e6eb983845a8c3d500d40">(See the <strong>Complete Log</strong>)</a></p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-register7.png" alt="NuttX Builds threaded neatly" /></p>
|
||
<h1 id="appendix-install-our-mastodon-server"><a class="doc-anchor" href="#appendix-install-our-mastodon-server">§</a>11 Appendix: Install our Mastodon Server</h1>
|
||
<p>Here are the steps to install Mastodon Server with Docker Compose. We tested with <a href="https://rancherdesktop.io/"><strong>Rancher Desktop on macOS</strong></a>, the same steps will probably work on <a href="https://docs.docker.com/engine/install/ubuntu/"><strong>Docker Desktop</strong></a> for Linux / macOS / Windows.</p>
|
||
<p><a href="https://lupyuen.github.io/articles/mastodon#appendix-docker-compose-for-mastodon">(<strong>docker-compose.yml</strong> is explained here)</a></p>
|
||
<ol>
|
||
<li>
|
||
<p>Download the <strong>Mastodon Source Code</strong> and init the Environment Config</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>git clone \
|
||
https://github.com/mastodon/mastodon \
|
||
--branch v4.3.2
|
||
cd mastodon
|
||
echo >.env.production</code></pre></div></li>
|
||
<li>
|
||
<p>Replace <strong>docker-compose.yml</strong> with our slightly-tweaked version</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>rm docker-compose.yml
|
||
wget https://raw.githubusercontent.com/lupyuen/mastodon/refs/heads/main/docker-compose.yml</code></pre></div>
|
||
<p><a href="https://github.com/lupyuen/mastodon/compare/upstream...lupyuen:mastodon:main">(See the <strong>Minor Tweaks</strong>)</a></p>
|
||
</li>
|
||
<li>
|
||
<p>Purge the <strong>Docker Volumes</strong>, if they already exist (see below)</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>docker volume rm postgres-data
|
||
docker volume rm redis-data
|
||
docker volume rm es-data
|
||
docker volume rm lt-data</code></pre></div></li>
|
||
<li>
|
||
<p>Edit <a href="https://github.com/lupyuen/mastodon/blob/main/docker-compose.yml#L58-L67"><strong>docker-compose.yml</strong></a>. Set “<strong>web > command</strong>” to “<strong>sleep infinity</strong>”</p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code>web:
|
||
command: sleep infinity</code></pre></div>
|
||
<p>(Why? Because we’ll start the Web Container to Configure Mastodon)</p>
|
||
</li>
|
||
<li>
|
||
<p>Start the <strong>Docker Containers for Mastodon</strong>: Database, Web, Redis (Memory Cache), Streaming (WebSocket), Sidekiq (Batch Jobs), Elasticsearch (Search Engine)</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## TODO: Is `sudo` needed?
|
||
sudo docker compose up
|
||
|
||
## If It Quits To Command-Line:
|
||
## Run a second time to get it up
|
||
sudo docker compose up
|
||
|
||
## Ignore the Redis, Streaming, Elasticsearch errors
|
||
## redis-1: Memory overcommit must be enabled
|
||
## streaming-1: connect ECONNREFUSED 127.0.0.1:6379
|
||
## es-1: max virtual memory areas vm.max_map_count is too low
|
||
|
||
## Press Ctrl-C to quit the log</code></pre></div>
|
||
<p><a href="https://gist.github.com/lupyuen/fb086d6f5fe84044c6c8dae1093b0328#file-gistfile1-txt-L226-L789">(See the <strong>Complete Log</strong>)</a></p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Init the Postgres Database:</strong> We create the Mastodon User</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## From https://docs.joinmastodon.org/admin/install/#creating-a-user
|
||
sudo docker exec \
|
||
-it \
|
||
mastodon-db-1 \
|
||
/bin/bash
|
||
exec su-exec \
|
||
postgres \
|
||
psql
|
||
CREATE USER mastodon CREATEDB;
|
||
\q</code></pre></div>
|
||
<p><a href="https://gist.github.com/lupyuen/f4f887ccf4ecfda0d5103b834044bd7b#file-gistfile1-txt-L1-L11">(See the <strong>Complete Log</strong>)</a></p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Generate the Mastodon Config:</strong> We connect to Web Container and prep the Mastodon Config</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## From https://docs.joinmastodon.org/admin/install/#generating-a-configuration
|
||
sudo docker exec \
|
||
-it \
|
||
mastodon-web-1 \
|
||
/bin/bash
|
||
RAILS_ENV=production \
|
||
bin/rails \
|
||
mastodon:setup
|
||
exit</code></pre></div>
|
||
<p><a href="https://gist.github.com/lupyuen/f4f887ccf4ecfda0d5103b834044bd7b#file-gistfile1-txt-L11-L95">(See the <strong>Complete Log</strong>)</a></p>
|
||
</li>
|
||
<li>
|
||
<p>Mastodon has <strong>Many Questions</strong>, we answer them</p>
|
||
<p>(Change <em>nuttx-feed.org</em> to Your Domain Name)</p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code>Domain name: nuttx-feed.org
|
||
Enable single user mode? No
|
||
Using Docker to run Mastodon? Yes
|
||
|
||
PostgreSQL host: db
|
||
PostgreSQL port: 5432
|
||
PostgreSQL database: mastodon_production
|
||
PostgreSQL user: mastodon
|
||
Password of user: [ blank ]
|
||
|
||
Redis host: redis
|
||
Redis port: 6379
|
||
Redis password: [ blank ]
|
||
|
||
Store uploaded files on the cloud? No
|
||
Send e-mails from localhost? Yes
|
||
E-mail address: Mastodon <notifications@nuttx-feed.org>
|
||
Send a test e-mail? No
|
||
|
||
Check for important updates? Yes
|
||
Save configuration? Yes
|
||
Save it to .env.production outside Docker:
|
||
# Generated with mastodon:setup on 2024-12-08 23:40:38 UTC
|
||
[ TODO: Please Save Mastodon Config! ]
|
||
|
||
Prepare the database now? Yes
|
||
Create an admin user straight away? Yes
|
||
Username: [ Your Admin Username ]
|
||
E-mail: [ Your Email Address ]
|
||
Login with the password:
|
||
[ TODO: Please Save Admin Password! ]</code></pre></div>
|
||
<p><a href="https://gist.github.com/lupyuen/f4f887ccf4ecfda0d5103b834044bd7b#file-gistfile1-txt-L11-L95">(See the <strong>Complete Log</strong>)</a></p>
|
||
<p>(No Email Server? Read on for our workaround)</p>
|
||
</li>
|
||
<li>
|
||
<p>Copy the Mastodon Config from above to <strong><code>.env.production</code></strong></p>
|
||
<div class="example-wrap"><pre class="language-text"><code># Generated with mastodon:setup on 2024-12-08 23:40:38 UTC
|
||
LOCAL_DOMAIN=nuttx-feed.org
|
||
SINGLE_USER_MODE=false
|
||
SECRET_KEY_BASE=...
|
||
OTP_SECRET=...
|
||
ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=...
|
||
ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=...
|
||
ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=...
|
||
VAPID_PRIVATE_KEY=...
|
||
VAPID_PUBLIC_KEY=...
|
||
DB_HOST=db
|
||
DB_PORT=5432
|
||
DB_NAME=mastodon_production
|
||
DB_USER=mastodon
|
||
DB_PASS=
|
||
REDIS_HOST=redis
|
||
REDIS_PORT=6379
|
||
REDIS_PASSWORD=
|
||
SMTP_SERVER=localhost
|
||
SMTP_PORT=25
|
||
SMTP_AUTH_METHOD=none
|
||
SMTP_OPENSSL_VERIFY_MODE=none
|
||
SMTP_ENABLE_STARTTLS=auto
|
||
SMTP_FROM_ADDRESS=Mastodon <notifications@nuttx-feed.org></code></pre></div>
|
||
<p><a href="https://gist.github.com/lupyuen/f4f887ccf4ecfda0d5103b834044bd7b#file-gistfile1-txt-L46-L75">(See the <strong>Complete Log</strong>)</a></p>
|
||
</li>
|
||
<li>
|
||
<p>Edit <a href="https://github.com/lupyuen/mastodon/blob/main/docker-compose.yml#L58-L67"><strong>docker-compose.yml</strong></a>. Set “<strong>web > command</strong>” to this…</p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code>web:
|
||
command: bundle exec puma -C config/puma.rb</code></pre></div>
|
||
<p>(Why? Because we’re done Configuring Mastodon!)</p>
|
||
</li>
|
||
<li>
|
||
<p>Restart the <strong>Docker Containers</strong> for Mastodon (pic below)</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## TODO: Is `sudo` needed?
|
||
sudo docker compose down
|
||
sudo docker compose up</code></pre></div></li>
|
||
<li>
|
||
<p>And <strong>Mastodon is Up</strong>!</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>redis-1: Ready to accept connections tcp
|
||
db-1: database system is ready to accept connections
|
||
streaming-1: request completed
|
||
web-1: GET /health</code></pre></div>
|
||
<p><a href="https://gist.github.com/lupyuen/420540f9157f2702c14944fc47743742">(See the <strong>Complete Log</strong>)</a></p>
|
||
<p><a href="https://gist.github.com/lupyuen/edbf045433189bebd4ad843608772ce8">(See <strong>Another Log</strong>)</a></p>
|
||
<p>(Sidekiq will have errors, we’ll explain why)</p>
|
||
</li>
|
||
</ol>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-containers.png" alt="Mastodon Containers in Rancher Desktop" /></p>
|
||
<p><em>Why the tweaks to docker-compose.yml?</em></p>
|
||
<p>Somehow Rancher Desktop doesn’t like to <strong>Mount the Local Filesystem</strong>, failing with a permission error…</p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code>## Local Filesystem will fail on macOS Rancher Desktop
|
||
services:
|
||
db:
|
||
volumes:
|
||
- ./postgres14:/var/lib/postgresql/data</code></pre></div>
|
||
<p>Thus we <strong>Mount the Docker Volumes</strong> instead: <a href="https://github.com/lupyuen/mastodon/blob/main/docker-compose.yml#L3-L58">docker-compose.yml</a></p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code>## Docker Volumes will mount OK on macOS Rancher Desktop
|
||
services:
|
||
db:
|
||
volumes:
|
||
- postgres-data:/var/lib/postgresql/data
|
||
|
||
redis:
|
||
volumes:
|
||
- redis-data:/data
|
||
|
||
sidekiq:
|
||
volumes:
|
||
- lt-data:/mastodon/public/system
|
||
|
||
## Declare the Docker Volumes
|
||
volumes:
|
||
postgres-data:
|
||
redis-data:
|
||
es-data:
|
||
lt-data:</code></pre></div>
|
||
<p>Note that Mastodon will appear at <strong>HTTP Port 3001</strong>, because Port 3000 is already taken by Grafana</p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code>web:
|
||
ports:
|
||
- '127.0.0.1:3001:3000'</code></pre></div>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-flow2.jpg" alt="Mastodon Server for Apache NuttX Continuous Integration" /></p>
|
||
<h1 id="appendix-test-our-mastodon-server"><a class="doc-anchor" href="#appendix-test-our-mastodon-server">§</a>12 Appendix: Test our Mastodon Server</h1>
|
||
<p>We’re ready to <strong>Test Mastodon</strong>!</p>
|
||
<ol>
|
||
<li>
|
||
<p>Talk to our <strong>Web Hosting Provider</strong> (or Tunnel Provider).</p>
|
||
<p>Channel all Incoming Requests for <em>https://nuttx-feed.org</em></p>
|
||
<p>To <em>http://YOUR_DOCKER_MACHINE:3001</em></p>
|
||
<p>(<strong>HTTPS Port 443</strong> connects to <strong>HTTP Port 3001</strong> via Reverse Proxy)</p>
|
||
<p>(For CloudFlare Tunnel: Set <strong>Security > Settings > High</strong>)</p>
|
||
<p>(Change <em>nuttx-feed.org</em> to Your Domain Name)</p>
|
||
</li>
|
||
<li>
|
||
<p>Browse to <em>https://nuttx-feed.org</em>. <strong>Mastodon is Up!</strong></p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-web5.png" alt="Mastodon Web UI" /></p>
|
||
</li>
|
||
<li>
|
||
<p>Log in with the <strong>Admin User and Password</strong></p>
|
||
<p>(From previous section)</p>
|
||
</li>
|
||
<li>
|
||
<p>Browse to <strong>Administration > Settings</strong> and fill in…</p>
|
||
<ul>
|
||
<li><strong>Branding</strong></li>
|
||
<li><strong>About</strong></li>
|
||
<li><strong>Registrations > Who Can Sign Up <br> > Approval Required > Require A Reason</strong></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p>Normally we’ll approve New Accounts at <strong>Moderation > Accounts > Approve</strong></p>
|
||
<p>But we don’t have an <strong>Outgoing Mail Server</strong> to validate the email address!</p>
|
||
<p>Let’s work around this…</p>
|
||
</li>
|
||
</ol>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-register1.png" alt="Create our Mastodon Account" /></p>
|
||
<h1 id="appendix-create-our-mastodon-account"><a class="doc-anchor" href="#appendix-create-our-mastodon-account">§</a>13 Appendix: Create our Mastodon Account</h1>
|
||
<p>Remember that we’ll pretend to be a Regular User <em>(nuttx_build)</em> and post Mastodon Updates? This is how we create the Mastodon User…</p>
|
||
<ol>
|
||
<li>
|
||
<p>Browse to <em>https://YOUR_DOMAIN_NAME.org</em>. Click <strong>“Create Account”</strong> and fill in the info (pic above)</p>
|
||
</li>
|
||
<li>
|
||
<p>Normally we’ll approve New Accounts at <strong>Moderation > Accounts > Approve</strong></p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-register3.png" alt="Approving New Accounts at Moderation > Accounts > Approve" /></p>
|
||
<p>But we don’t have an <strong>Outgoing Mail Server</strong> to validate the Email Address!</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-register4.png" alt="We don’t have an Outgoing Mail Server to validate the email address" /></p>
|
||
</li>
|
||
<li>
|
||
<p>Instead we do this…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Approve and Confirm the Email Address
|
||
## From https://docs.joinmastodon.org/admin/tootctl/#accounts-approve
|
||
sudo docker exec \
|
||
-it \
|
||
mastodon-web-1 \
|
||
/bin/bash
|
||
bin/tootctl accounts \
|
||
approve nuttx_build
|
||
bin/tootctl accounts \
|
||
modify nuttx_build \
|
||
--confirm
|
||
exit</code></pre></div>
|
||
<p>(Change <em>nuttx_build</em> to the new username)</p>
|
||
</li>
|
||
<li>
|
||
<p>FYI for a new <strong>Owner Account</strong>, do this…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## From https://docs.joinmastodon.org/admin/setup/#admin-cli
|
||
sudo docker exec \
|
||
-it \
|
||
mastodon-web-1 \
|
||
/bin/bash
|
||
bin/tootctl accounts \
|
||
create YOUR_OWNER_USERNAME \
|
||
--email YOUR_OWNER_EMAIL \
|
||
--confirmed \
|
||
--role Owner
|
||
bin/tootctl accounts \
|
||
approve YOUR_OWNER_NAME
|
||
exit</code></pre></div></li>
|
||
<li>
|
||
<p>That’s why it’s OK to ignore the <strong>Sidekiq Errors</strong> for sending email…</p>
|
||
<div class="example-wrap"><pre class="language-text"><code>sidekiq-1 ...
|
||
Connection refused
|
||
connect(2) for localhost port 25</code></pre></div>
|
||
<p><a href="https://gist.github.com/lupyuen/420540f9157f2702c14944fc47743742#file-gistfile1-txt-L333-L338">(See the <strong>Complete Log</strong>)</a></p>
|
||
</li>
|
||
</ol>
|
||
<h1 id="appendix-create-our-mastodon-app"><a class="doc-anchor" href="#appendix-create-our-mastodon-app">§</a>14 Appendix: Create our Mastodon App</h1>
|
||
<p>Let’s create a <strong>Mastodon App</strong> and an <strong>Access Token</strong> for posting to our Mastodon…</p>
|
||
<ol>
|
||
<li>
|
||
<p>We create a <strong>Mastodon App</strong> for NuttX Dashboard…</p>
|
||
<div class="example-wrap"><pre class="language-text"><code>## Create Our App: https://docs.joinmastodon.org/client/token/#app
|
||
curl -X POST \
|
||
-F 'client_name=NuttX Dashboard' \
|
||
-F 'redirect_uris=urn:ietf:wg:oauth:2.0:oob' \
|
||
-F 'scopes=read write push' \
|
||
-F 'website=https://nuttx-dashboard.org' \
|
||
https://YOUR_DOMAIN_NAME.org/api/v1/apps</code></pre></div></li>
|
||
<li>
|
||
<p>We’ll see the <strong>Client ID</strong> and <strong>Client Secret</strong>. Please save them and keep them secret! (Change <em>nuttx-dashboard</em> to your App Name)</p>
|
||
<div class="example-wrap"><pre class="language-json"><code>{"id":"3",
|
||
"name":"NuttX Dashboard",
|
||
"website":"https://nuttx-dashboard.org",
|
||
"scopes":["read","write","push"],
|
||
"redirect_uris":["urn:ietf:wg:oauth:2.0:oob"],
|
||
"vapid_key":"...",
|
||
"redirect_uri":"urn:ietf:wg:oauth:2.0:oob",
|
||
"client_id":"...",
|
||
"client_secret":"...",
|
||
"client_secret_expires_at":0}</code></pre></div></li>
|
||
<li>
|
||
<p>Open a Web Browser. Browse to <em>https://YOUR_DOMAIN_NAME.org</em></p>
|
||
<p>Log in as Your New User <em>(nuttx_build)</em></p>
|
||
</li>
|
||
<li>
|
||
<p>Paste this URL into the Same Web Browser</p>
|
||
<div class="example-wrap"><pre class="language-text"><code>https://YOUR_DOMAIN_NAME.org/oauth/authorize
|
||
?client_id=YOUR_CLIENT_ID
|
||
&scope=read+write+push
|
||
&redirect_uri=urn:ietf:wg:oauth:2.0:oob
|
||
&response_type=code</code></pre></div>
|
||
<p><a href="https://docs.joinmastodon.org/client/authorized/">(Explained here)</a></p>
|
||
</li>
|
||
<li>
|
||
<p>Click <strong>Authorize</strong>. (Pic below)</p>
|
||
</li>
|
||
<li>
|
||
<p>Copy the <strong>Authorization Code</strong>. (Pic below. It will expire soon!)</p>
|
||
</li>
|
||
<li>
|
||
<p>We transform the Authorization Code into an <strong>Access Token</strong></p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## From https://docs.joinmastodon.org/client/authorized/#token
|
||
export CLIENT_ID=... ## From Above
|
||
export CLIENT_SECRET=... ## From Above
|
||
export AUTH_CODE=... ## From Above
|
||
curl -X POST \
|
||
-F "client_id=$CLIENT_ID" \
|
||
-F "client_secret=$CLIENT_SECRET" \
|
||
-F "redirect_uri=urn:ietf:wg:oauth:2.0:oob" \
|
||
-F "grant_type=authorization_code" \
|
||
-F "code=$AUTH_CODE" \
|
||
-F "scope=read write push" \
|
||
https://YOUR_DOMAIN_NAME.org/oauth/token</code></pre></div></li>
|
||
<li>
|
||
<p>We’ll see the <strong>Access Token</strong>. Please save it and keep secret!</p>
|
||
<div class="example-wrap"><pre class="language-json"><code>{"access_token":"...",
|
||
"token_type":"Bearer",
|
||
"scope":"read write push",
|
||
"created_at":1733966892}</code></pre></div></li>
|
||
<li>
|
||
<p>To test our Access Token…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>export ACCESS_TOKEN=... ## From Above
|
||
curl \
|
||
-H "Authorization: Bearer $ACCESS_TOKEN" \
|
||
https://YOUR_DOMAIN_NAME.org/api/v1/accounts/verify_credentials</code></pre></div></li>
|
||
<li>
|
||
<p>We’ll see…</p>
|
||
<div class="example-wrap"><pre class="language-json"><code>{"username": "nuttx_build",
|
||
"acct": "nuttx_build",
|
||
"display_name": "NuttX Build",
|
||
"locked": false,
|
||
"bot": false,
|
||
"discoverable": null,
|
||
"indexable": false,
|
||
...</code></pre></div>
|
||
<p>Yep looks hunky dory!</p>
|
||
</li>
|
||
</ol>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-register5.png" alt="Getting a Mastodon Authorization Code" /></p>
|
||
<h1 id="appendix-create-a-mastodon-post"><a class="doc-anchor" href="#appendix-create-a-mastodon-post">§</a>15 Appendix: Create a Mastodon Post</h1>
|
||
<p>Our Regular Mastondon User is up! Let’s post something as the user…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Create Status: https://docs.joinmastodon.org/methods/statuses/#create
|
||
export ACCESS_TOKEN=... ## From Above
|
||
curl -X POST \
|
||
-H "Authorization: Bearer $ACCESS_TOKEN" \
|
||
-F "status=Posting a status from curl" \
|
||
https://YOUR_DOMAIN_NAME.org/api/v1/statuses</code></pre></div>
|
||
<p>And our <strong>Mastodon Post</strong> appears!</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-web4.png" alt="Creating a Mastodon Post" /></p>
|
||
<p>Let’s make sure that <strong>Mastodon API</strong> works on our server…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Install `jq` for Browsing JSON
|
||
$ brew install jq ## For macOS
|
||
$ sudo apt install jq ## For Ubuntu
|
||
|
||
## Fetch the Public Timeline for nuttx-feed.org
|
||
## https://docs.joinmastodon.org/client/public/#timelines
|
||
$ curl https://nuttx-feed.org/api/v1/timelines/public \
|
||
| jq
|
||
|
||
{ ... "teensy-4.x : PIKRON-BB - Build Failed" ... }
|
||
|
||
## Fetch the User nuttx_build at nuttx-feed.org
|
||
$ curl \
|
||
-H 'Accept: application/activity+json' \
|
||
https://nuttx-feed.org/@nuttx_build \
|
||
| jq
|
||
|
||
{ "name": "nuttx_build",
|
||
"url" : "https://nuttx-feed.org/@nuttx_build" ... }</code></pre></div>
|
||
<p><a href="https://gist.github.com/lupyuen/c31c426b28f32341301fa28f16a1251e">(See the <strong>Complete Log</strong>)</a></p>
|
||
<p><a href="https://docs.joinmastodon.org/spec/webfinger/"><strong>WebFinger</strong></a> is particularly important, it locates Users within the Fediverse. It should always work at the <a href="https://docs.joinmastodon.org/admin/config/#web_domain"><strong>Root of our Mastodon Server</strong></a>!</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## WebFinger: Fetch the User nuttx_build at nuttx-feed.org
|
||
$ curl \
|
||
https://nuttx-feed.org/.well-known/webfinger\?resource\=acct:nuttx_build@nuttx-feed.org \
|
||
| jq
|
||
|
||
{
|
||
"subject": "acct:nuttx_build@nuttx-feed.org",
|
||
"aliases": [
|
||
"https://nuttx-feed.org/@nuttx_build",
|
||
"https://nuttx-feed.org/users/nuttx_build"
|
||
],
|
||
"links": [
|
||
{
|
||
"rel": "http://webfinger.net/rel/profile-page",
|
||
"type": "text/html",
|
||
"href": "https://nuttx-feed.org/@nuttx_build"
|
||
},
|
||
{
|
||
"rel": "self",
|
||
"type": "application/activity+json",
|
||
"href": "https://nuttx-feed.org/users/nuttx_build"
|
||
},
|
||
{
|
||
"rel": "http://ostatus.org/schema/1.0/subscribe",
|
||
"template": "https://nuttx-feed.org/authorize_interaction?uri={uri}"
|
||
}
|
||
]
|
||
}</code></pre></div>
|
||
<p><a href="https://gist.github.com/lupyuen/209d711d6cd7096a422da55f209d7745">(See the <strong>Complete Log</strong>)</a></p>
|
||
<h1 id="appendix-backup-our-mastodon-server"><a class="doc-anchor" href="#appendix-backup-our-mastodon-server">§</a>16 Appendix: Backup our Mastodon Server</h1>
|
||
<p>Here are the steps to <strong>Backup our Mastodon Server</strong>: PostgreSQL Database, Redis Database and User-Uploaded Files…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## From https://docs.joinmastodon.org/admin/backups/
|
||
## Backup Postgres Database (and check for sensible data)
|
||
sudo docker exec \
|
||
-it \
|
||
mastodon-db-1 \
|
||
/bin/bash -c \
|
||
"exec su-exec postgres pg_dumpall" \
|
||
>mastodon.sql
|
||
head -50 mastodon.sql
|
||
|
||
## Backup Redis (and check for sensible data)
|
||
sudo docker cp \
|
||
mastodon-redis-1:/data/dump.rdb \
|
||
.
|
||
strings dump.rdb \
|
||
| tail -50
|
||
|
||
## Backup User-Uploaded Files
|
||
tar cvf \
|
||
mastodon-public-system.tar \
|
||
mastodon/public/system</code></pre></div>
|
||
<p><em>Is it safe to host Mastodon in Docker?</em></p>
|
||
<p>Docker Engine on Linux is <a href="https://www.opensourceforu.com/2024/12/analysing-linus-torvalds-critique-of-docker/"><strong>not quite as secure</strong></a> compared with a Full VM or QEMU. So be very careful!</p>
|
||
<p><a href="https://docs.rancherdesktop.io/references/architecture">(macOS Rancher Desktop runs Docker with <strong>Lima VM</strong> and <strong>QEMU Arm64</strong>)</a></p>
|
||
<p>Remember to watch our Mastodon Server for <strong>Dubious Web Requests</strong>! Like these pesky WordPress Malware Bots (sigh)</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-log.png" alt="WordPress Malware Bots" /></p>
|
||
<p>These <strong>Firewall Rules</strong> might help…</p>
|
||
<ul>
|
||
<li>
|
||
<p>Block all <strong>URI Paths</strong> matching <strong><code>/wordpress/*</code></strong></p>
|
||
</li>
|
||
<li>
|
||
<p>Or matching <strong><code>/wp-admin/*</code></strong></p>
|
||
</li>
|
||
<li>
|
||
<p>Or matching <strong><code>//*</code></strong></p>
|
||
</li>
|
||
</ul>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-firewall.png" alt="Firewall Rules for Mastodon Server" /></p>
|
||
<h1 id="appendix-enable-elasticsearch-for-mastodon"><a class="doc-anchor" href="#appendix-enable-elasticsearch-for-mastodon">§</a>17 Appendix: Enable Elasticsearch for Mastodon</h1>
|
||
<p>Enabling <strong>Elasticsearch</strong> for macOS Rancher Desktop is a little tricky. That’s why we saved it for last.</p>
|
||
<ol>
|
||
<li>
|
||
<p>In Mastodon Web: Head over to <strong>Administration > Dashboard</strong>. It should say…</p>
|
||
<p><em>“Could not connect to Elasticsearch. Please check that it is running, or disable full-text search”</em></p>
|
||
</li>
|
||
<li>
|
||
<p>To Enable Elasticsearch: Edit <strong><code>.env.production</code></strong> and add these lines…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>ES_ENABLED=true
|
||
ES_HOST=es
|
||
ES_PORT=9200</code></pre></div></li>
|
||
<li>
|
||
<p>Edit <a href="https://github.com/lupyuen/mastodon/blob/main/docker-compose.yml"><strong>docker-compose.yml</strong></a>.</p>
|
||
<p>Uncomment the Section for <strong>“<code>es</code>”</strong></p>
|
||
<p>Map the Docker Volume <strong>es-data</strong> for Elasticsearch</p>
|
||
<p>Web Container should depend on <strong>“<code>es</code>”</strong></p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code> es:
|
||
volumes:
|
||
- es-data:/usr/share/elasticsearch/data
|
||
web:
|
||
depends_on:
|
||
- db
|
||
- redis
|
||
- es</code></pre></div></li>
|
||
<li>
|
||
<p>Restart the Docker Containers</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>sudo docker compose down
|
||
sudo docker compose up</code></pre></div></li>
|
||
<li>
|
||
<p>We’ll see…</p>
|
||
<p><em>“es-1: bootstrap check failure: max virtual memory areas vm.max_map_count 65530 is too low, increase to at least 262144”</em></p>
|
||
</li>
|
||
<li>
|
||
<p>Here comes the tricky part: <strong>max_map_count</strong> is configured here!</p>
|
||
<div class="example-wrap"><pre class="language-text"><code>~/Library/Application\ Support/rancher-desktop/lima/_config/override.yaml</code></pre></div>
|
||
<p><a href="https://docs.rancherdesktop.io/how-to-guides/increasing-open-file-limit/"><strong>Follow the Instructions</strong></a> and set…</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>sysctl -w vm.max_map_count=262144</code></pre></div></li>
|
||
<li>
|
||
<p>Restart Docker Desktop</p>
|
||
</li>
|
||
<li>
|
||
<p>Verify that <strong>max_map_count</strong> has increased</p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Print the Max Virtual Memory Areas
|
||
$ sudo docker exec \
|
||
-it \
|
||
mastodon-es-1 \
|
||
/bin/bash -c \
|
||
"sysctl vm.max_map_count"
|
||
|
||
vm.max_map_count = 262144</code></pre></div></li>
|
||
<li>
|
||
<p>Head back to Mastodon Web. Click <strong>Administration > Dashboard</strong>. We should see…</p>
|
||
<p><em>“Elasticsearch index mappings are outdated”</em></p>
|
||
</li>
|
||
<li>
|
||
<p>Finally we <strong>Reindex Elasticsearch</strong></p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>sudo docker exec \
|
||
-it \
|
||
mastodon-web-1 \
|
||
/bin/bash
|
||
bin/tootctl search \
|
||
deploy --only=instances \
|
||
accounts tags statuses public_statuses
|
||
exit</code></pre></div></li>
|
||
<li>
|
||
<p>At <strong>Administration > Dashboard</strong>: Mastodon complains no more!</p>
|
||
<p><a href="https://gist.github.com/lupyuen/21ad4e38fa00796d132e63d41e4a339f">(See the <strong>Complete Log</strong>)</a></p>
|
||
</li>
|
||
</ol>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-flow.jpg" alt="Mastodon Server for Apache NuttX Continuous Integration" /></p>
|
||
<h1 id="appendix-docker-compose-for-mastodon"><a class="doc-anchor" href="#appendix-docker-compose-for-mastodon">§</a>18 Appendix: Docker Compose for Mastodon</h1>
|
||
<p><em>What’s this Docker Compose? Why use it for Mastodon?</em></p>
|
||
<p>We could install manually <strong>Multiple Docker Containers</strong> for Mastodon: Ruby-on-Rails + PostgreSQL + Redis + Sidekiq + Streaming + Elasticsearch…</p>
|
||
<p>But there’s an easier way: <a href="https://docs.docker.com/compose/"><strong>Docker Compose</strong></a> will create all the Docker Containers with a Single Command: <strong>docker compose up</strong></p>
|
||
<p>In this section we study the <strong>Docker Containers</strong> for Mastodon. And explain the <strong>Minor Tweaks</strong> we made to Mastodon’s Official Docker Compose Config. (Pic above)</p>
|
||
<p><a href="https://github.com/lupyuen/mastodon/compare/upstream...lupyuen:mastodon:main">(See the <strong>Minor Tweaks</strong>)</a></p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-containers.png" alt="Mastodon Containers in Rancher Desktop" /></p>
|
||
<h2 id="database-server"><a class="doc-anchor" href="#database-server">§</a>18.1 Database Server</h2>
|
||
<p><a href="https://www.postgresql.org/"><strong>PostgreSQL</strong></a> is our Database Server for Mastodon: <a href="https://github.com/lupyuen/mastodon/blob/main/docker-compose.yml#L3-L17">docker-compose.yml</a></p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code>services:
|
||
db:
|
||
restart: always
|
||
image: postgres:14-alpine
|
||
shm_size: 256mb
|
||
|
||
## Map the Docker Volume "postgres-data"
|
||
## because macOS Rancher Desktop won't work correctly with a Local Filesystem
|
||
volumes:
|
||
- postgres-data:/var/lib/postgresql/data
|
||
|
||
## Allow auto-login by all connections from localhost
|
||
environment:
|
||
- 'POSTGRES_HOST_AUTH_METHOD=trust'
|
||
|
||
## Database Server is not exposed outside Docker
|
||
networks:
|
||
- internal_network
|
||
healthcheck:
|
||
test: ['CMD', 'pg_isready', '-U', 'postgres']</code></pre></div>
|
||
<p>Note the last line for <em>POSTGRES_HOST_AUTH_METHOD</em>. It says that our Database Server will allow auto-login by <strong>all connections from localhost</strong>. Even without PostgreSQL Password!</p>
|
||
<p>This is probably OK for us, since our Database Server runs in its own Docker Container.</p>
|
||
<p>We map the <strong>Docker Volume</strong> <em>postgres-data</em>, because macOS Rancher Desktop won’t work correctly with a Local Filesystem like <em>./postgres14</em>.</p>
|
||
<h2 id="web-server"><a class="doc-anchor" href="#web-server">§</a>18.2 Web Server</h2>
|
||
<p>Powered by Ruby-on-Rails, <strong>Puma</strong> is our Web Server: <a href="https://github.com/lupyuen/mastodon/blob/main/docker-compose.yml#L58-L81">docker-compose.yml</a></p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code> web:
|
||
## You can uncomment the following line if you want to not use the prebuilt image, for example if you have local code changes
|
||
## build: .
|
||
image: ghcr.io/mastodon/mastodon:v4.3.2
|
||
restart: always
|
||
|
||
## Read the Mastondon Config from Docker Host
|
||
env_file: .env.production
|
||
|
||
## Start the Puma Web Server
|
||
command: bundle exec puma -C config/puma.rb
|
||
## When Configuring Mastodon: Change to...
|
||
## command: sleep infinity
|
||
|
||
## HTTP Port 3000 should always return OK
|
||
healthcheck:
|
||
# prettier-ignore
|
||
test: ['CMD-SHELL',"curl -s --noproxy localhost localhost:3000/health | grep -q 'OK' || exit 1"]
|
||
|
||
## Mastodon will appear outside Docker at HTTP Port 3001
|
||
## because Port 3000 is already taken by Grafana
|
||
ports:
|
||
- '127.0.0.1:3001:3000'
|
||
networks:
|
||
- external_network
|
||
- internal_network
|
||
depends_on:
|
||
- db
|
||
- redis
|
||
- es
|
||
volumes:
|
||
- ./public/system:/mastodon/public/system</code></pre></div>
|
||
<p>Note that Mastodon will appear at <strong>HTTP Port 3001</strong>, because Port 3000 is already taken by Grafana.</p>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-flow2.jpg" alt="Mastodon Server for Apache NuttX Continuous Integration" /></p>
|
||
<h2 id="redis-server"><a class="doc-anchor" href="#redis-server">§</a>18.3 Redis Server</h2>
|
||
<p>Web Server fetching data directly from Database Server will be awfully slow. That’s why we use Redis as an <a href="https://github.com/redis/redis"><strong>In-Memory Caching Database</strong></a>: <a href="https://github.com/lupyuen/mastodon/blob/main/docker-compose.yml#L17-L27">docker-compose.yml</a></p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code> redis:
|
||
restart: always
|
||
image: redis:7-alpine
|
||
|
||
## Map the Docker Volume "redis-data"
|
||
## because macOS Rancher Desktop won't work correctly with a Local Filesystem
|
||
volumes:
|
||
- redis-data:/data
|
||
|
||
## Redis Server is not exposed outside Docker
|
||
networks:
|
||
- internal_network
|
||
healthcheck:
|
||
test: ['CMD', 'redis-cli', 'ping']</code></pre></div><h2 id="sidekiq-server"><a class="doc-anchor" href="#sidekiq-server">§</a>18.4 Sidekiq Server</h2>
|
||
<p>Remember the Emails that Mastodon will send upon User Registration? Mastodon calls <a href="https://github.com/sidekiq/sidekiq"><strong>Sidekiq</strong></a> to run Background Jobs, so they won’t hold up the Web Server: <a href="https://github.com/lupyuen/mastodon/blob/main/docker-compose.yml#L102-L119">docker-compose.yml</a></p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code> sidekiq:
|
||
build: .
|
||
image: ghcr.io/mastodon/mastodon:v4.3.2
|
||
restart: always
|
||
|
||
## Read the Mastondon Config from Docker Host
|
||
env_file: .env.production
|
||
|
||
## Start the Sidekiq Batch Job Server
|
||
command: bundle exec sidekiq
|
||
depends_on:
|
||
- db
|
||
- redis
|
||
volumes:
|
||
- ./public/system:/mastodon/public/system
|
||
|
||
## Sidekiq Server is exposed outside Docker
|
||
## for Outgoing Connections, to deliver emails
|
||
networks:
|
||
- external_network
|
||
- internal_network
|
||
healthcheck:
|
||
test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"]</code></pre></div><h2 id="streaming-server"><a class="doc-anchor" href="#streaming-server">§</a>18.5 Streaming Server</h2>
|
||
<p><em>(Streaming Server is Optional)</em></p>
|
||
<p>Mastodon (and Fediverse) uses <a href="https://docs.joinmastodon.org/spec/activitypub/"><strong>ActivityPub</strong></a> for exchanging lots of info about Users and Posts. Our Web Server supports the <strong>HTTP Rest API</strong>, but there’s a more efficient way: <strong>WebSocket API</strong>.</p>
|
||
<p>WebSocket is <strong>totally optional</strong>, Mastodon works fine without it, probably a little less efficient: <a href="https://github.com/lupyuen/mastodon/blob/main/docker-compose.yml#L81-L102">docker-compose.yml</a></p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code> streaming:
|
||
## You can uncomment the following lines if you want to not use the prebuilt image, for example if you have local code changes
|
||
## build:
|
||
## dockerfile: ./streaming/Dockerfile
|
||
## context: .
|
||
image: ghcr.io/mastodon/mastodon-streaming:v4.3.2
|
||
restart: always
|
||
|
||
## Read the Mastondon Config from Docker Host
|
||
env_file: .env.production
|
||
|
||
## Start the Streaming Server (Node.js!)
|
||
command: node ./streaming/index.js
|
||
depends_on:
|
||
- db
|
||
- redis
|
||
|
||
## WebSocket will listen on HTTP Port 4000
|
||
## for Incoming Connections (totally optional!)
|
||
ports:
|
||
- '127.0.0.1:4000:4000'
|
||
networks:
|
||
- external_network
|
||
- internal_network
|
||
healthcheck:
|
||
# prettier-ignore
|
||
test: ['CMD-SHELL', "curl -s --noproxy localhost localhost:4000/api/v1/streaming/health | grep -q 'OK' || exit 1"]</code></pre></div><h2 id="elasticsearch-server"><a class="doc-anchor" href="#elasticsearch-server">§</a>18.6 Elasticsearch Server</h2>
|
||
<p><em>(Elasticsearch is optional)</em></p>
|
||
<p>Elasticsearch is for <strong>Full-Text Search</strong>. Also totally optional, unless we require Full-Text Search for Users and Posts: <a href="https://github.com/lupyuen/mastodon/blob/main/docker-compose.yml#L27-L58">docker-compose.yml</a></p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code> es:
|
||
restart: always
|
||
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.4
|
||
environment:
|
||
- "ES_JAVA_OPTS=-Xms512m -Xmx512m -Des.enforce.bootstrap.checks=true"
|
||
- "xpack.license.self_generated.type=basic"
|
||
- "xpack.security.enabled=false"
|
||
- "xpack.watcher.enabled=false"
|
||
- "xpack.graph.enabled=false"
|
||
- "xpack.ml.enabled=false"
|
||
- "bootstrap.memory_lock=true"
|
||
- "cluster.name=es-mastodon"
|
||
- "discovery.type=single-node"
|
||
- "thread_pool.write.queue_size=1000"
|
||
|
||
## Elasticsearch is exposed externally at HTTP Port 9200. (Why?)
|
||
ports:
|
||
- '127.0.0.1:9200:9200'
|
||
networks:
|
||
- external_network
|
||
- internal_network
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"]
|
||
|
||
## Map the Docker Volume "es-data"
|
||
## because macOS Rancher Desktop won't work correctly with a Local Filesystem
|
||
volumes:
|
||
- es-data:/usr/share/elasticsearch/data
|
||
ulimits:
|
||
memlock:
|
||
soft: -1
|
||
hard: -1
|
||
nofile:
|
||
soft: 65536
|
||
hard: 65536</code></pre></div><h2 id="volumes-and-networks"><a class="doc-anchor" href="#volumes-and-networks">§</a>18.7 Volumes and Networks</h2>
|
||
<p>Finally we declare the <strong>Volumes and Networks</strong> used by our Docker Containers: <a href="https://github.com/lupyuen/mastodon/blob/main/docker-compose.yml#L136-L146">docker-compose.yml</a></p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code>volumes:
|
||
postgres-data:
|
||
redis-data:
|
||
es-data:
|
||
lt-data:
|
||
|
||
networks:
|
||
external_network:
|
||
internal_network:
|
||
internal: true</code></pre></div><h2 id="simplest-server-for-mastodon"><a class="doc-anchor" href="#simplest-server-for-mastodon">§</a>18.8 Simplest Server for Mastodon</h2>
|
||
<p><em>Phew that looks mighty complicated!</em></p>
|
||
<p>There’s a simpler way: Mastodon provides a Docker Compose Config for <a href="https://github.com/mastodon/mastodon#docker"><strong>Mastodon Development</strong></a>.</p>
|
||
<p>It’s good for Local Experimentation. But <strong>Not Safe for Internet Hosting!</strong></p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>## Based on https://github.com/mastodon/mastodon#docker
|
||
git clone https://github.com/mastodon/mastodon --branch v4.3.2
|
||
cd mastodon
|
||
sudo docker compose -f .devcontainer/compose.yaml up -d
|
||
sudo docker compose -f .devcontainer/compose.yaml exec app bin/setup
|
||
sudo docker compose -f .devcontainer/compose.yaml exec app bin/dev
|
||
|
||
## Browse to Mastodon Web at http://localhost:3000
|
||
## TODO: What's the Default Admin ID and Password?
|
||
|
||
## Create our own Mastodon Owner Account:
|
||
## From https://docs.joinmastodon.org/admin/setup/#admin-cli
|
||
## And https://docs.joinmastodon.org/admin/tootctl/#accounts-approve
|
||
sudo docker exec \
|
||
-it \
|
||
devcontainer-app-1 \
|
||
/bin/bash
|
||
bin/tootctl accounts create \
|
||
YOUR_OWNER_USERNAME \
|
||
--email YOUR_OWNER_EMAIL \
|
||
--confirmed \
|
||
--role Owner
|
||
bin/tootctl accounts \
|
||
approve YOUR_OWNER_USERNAME
|
||
exit
|
||
|
||
## Reindex Elasticsearch
|
||
sudo docker exec \
|
||
-it \
|
||
devcontainer-app-1 \
|
||
/bin/bash
|
||
bin/tootctl search \
|
||
deploy --only=tags
|
||
exit</code></pre></div>
|
||
<p><strong>Optional:</strong> Configure Mastodon Web to listen at <strong>HTTP Port 3001</strong> (since 3000 is used by Grafana). We edit <em>.devcontainer/compose.yaml</em></p>
|
||
<div class="example-wrap"><pre class="language-yaml"><code>services:
|
||
app:
|
||
ports:
|
||
- '127.0.0.1:3001:3000'</code></pre></div>
|
||
<p><strong>Optional:</strong> Configure the Mastodon Domain. We edit <em>.env.development</em></p>
|
||
<div class="example-wrap"><pre class="language-bash"><code>LOCAL_DOMAIN=nuttx-feed.org</code></pre></div>
|
||
<p><img src="https://lupyuen.github.io/images/mastodon-hike.jpg" alt="50 km Overnight Hike: City to Changi Airport to Changi Village … Made possible by Mastodon! 👍" /></p>
|
||
<p><a href="https://www.strava.com/activities/13176081611"><em>50 km Overnight Hike: City to Changi Airport to Changi Village … Made possible by Mastodon! 👍</em></a></p>
|
||
|
||
|
||
<!-- Begin scripts/rustdoc-after.html: Post-HTML for Custom Markdown files processed by rustdoc, like chip8.md -->
|
||
|
||
<!-- Begin Theme Picker and Prism Theme -->
|
||
<script src="../theme.js"></script>
|
||
<script src="../prism.js"></script>
|
||
<!-- Theme Picker and Prism Theme -->
|
||
|
||
<!-- End scripts/rustdoc-after.html -->
|
||
|
||
|
||
</body>
|
||
</html> |