Add email notifications for failed certbot runs

Based on https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO

This can easily be used by other services, with one line added to the
[Unit] section of a service:

OnFailure=status-email-ops@%n.service

Refs #3

site-cookbooks/kosmos-base/files/default/certbot.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Certbot
+Documentation=file:///usr/share/doc/python-certbot-doc/html/index.html
+Documentation=https://letsencrypt.readthedocs.io/en/latest/
+OnFailure=status-email-ops@%n.service
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/certbot -q renew
+PrivateTmp=true

site-cookbooks/kosmos-base/files/default/systemd-email

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+/usr/sbin/sendmail -t <<ERRMAIL
+To: $1
+From: systemd <root@$HOSTNAME>
+Subject: $2
+Content-Transfer-Encoding: 8bit
+Content-Type: text/plain; charset=UTF-8
+
+$(systemctl status --full "$2")
+ERRMAIL

site-cookbooks/kosmos-base/metadata.rb

@@ -4,7 +4,7 @@ maintainer_email 'mail@kosmos.org'
 license          'MIT'
 description      'The Kosmos base cookbook'
 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version          '0.2.0'
+version          '0.3.0'
 chef_version     '>= 14.0' # Uses the new sudo resource
 
 depends 'apt'

site-cookbooks/kosmos-base/recipes/default.rb

@@ -27,6 +27,7 @@
 include_recipe 'apt'
 include_recipe 'timezone_iii'
 include_recipe 'ntp'
+include_recipe 'kosmos-base::systemd_emails'
 
 package 'mailutils'
 package 'mosh'

site-cookbooks/kosmos-base/recipes/letsencrypt.rb

@@ -26,26 +26,12 @@
 
 # Install certbot and set up hooks
 
-# Remove the unless/else when we get rid of dev, running 15.04. No ppa for it
-unless node["lsb"]["codename"] == "vivid"
-  apt_repository "certbot" do
-    uri "ppa:certbot/certbot"
-  end
-
-  package "certbot"
-else
-  remote_file "/usr/bin/certbot" do
-    source "https://dl.eff.org/certbot-auto"
-    mode 0755
-  end
-
-  cron "renew Let's Encrypt certificates" do
-   hour "4"
-   mailto "logs@5apps.com"
-   command "/usr/bin/certbot -q renew"
- end
+apt_repository "certbot" do
+  uri "ppa:certbot/certbot"
 end
 
+package "certbot"
+
 %w(deploy post pre).each do |subdir|
   directory "/etc/letsencrypt/renewal-hooks/#{subdir}" do
     recursive true
@@ -72,3 +58,16 @@ template "/root/gandi_dns_certbot_hook.sh" do
   variables gandi_api_key: gandi_api_data_bag_item["key"]
   mode 0770
 end
+
+include_recipe 'kosmos-base::systemd_emails'
+
+# Overwrite the systemd service to add email notifications on failures
+cookbook_file "/lib/systemd/system/certbot.service" do
+  source "certbot.service"
+  notifies :run, "execute[systemctl daemon-reload]", :delayed
+end
+
+execute "systemctl daemon-reload" do
+  command "systemctl daemon-reload"
+  action :nothing
+end

site-cookbooks/kosmos-base/recipes/systemd_emails.rb

@@ -0,0 +1,45 @@
+#
+# Cookbook Name:: kosmos-base
+# Recipe:: systemd_emails
+#
+# The MIT License (MIT)
+#
+# Copyright:: 2019, Kosmos Developers
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# To add email notifications to a Systemd service, add the following entry to
+# the [Unit] section of its service:
+#
+# OnFailure=status-email-ops@%n.service
+
+cookbook_file "/usr/local/sbin/systemd-email" do
+  source "systemd-email"
+  mode "750"
+end
+
+template "/etc/systemd/system/status-email-ops@.service" do
+  source "status-email-ops@.service"
+  variables email: "ops@kosmos.org"
+end
+
+execute "systemctl daemon-reload" do
+  command "systemctl daemon-reload"
+  action :nothing
+end

site-cookbooks/kosmos-base/templates/default/status-email-ops@.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=status email for %i to ops
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/systemd-email <%= @email %> %i
+User=root
+Group=systemd-journal