WIP: Set up ingress with Let's Encrypt certificates using cert-manager

This is using haproxy-ingress to support forwarding SSH on port 22

Since we're using cert-manager with ingress to get Let's Encrypt certs,
we're not using the Let's Encrypt functionality that's part of Gitea. To
run this we need to change the config file, have Gitea run on port 3000
as HTTP and disable all the Let's Encrypt config keys. Currently the
gitea-ingress.yaml uses the letsencrypt-staging ClusterIssuer

This has been tested on a local Kubernetes cluster using Docker for Mac

kubernetes/gitea-ingress.yaml

@@ -0,0 +1,276 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ingress-controller
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: ingress-controller
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - endpoints
+      - nodes
+      - pods
+      - secrets
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "extensions"
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+  - apiGroups:
+      - "extensions"
+    resources:
+      - ingresses/status
+    verbs:
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: ingress-controller
+  namespace: default
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - pods
+      - secrets
+      - namespaces
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - get
+      - create
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: ingress-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-controller
+subjects:
+  - kind: ServiceAccount
+    name: ingress-controller
+    namespace: default
+  - apiGroup: rbac.authorization.k8s.io
+    kind: User
+    name: ingress-controller
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: ingress-controller
+  namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-controller
+subjects:
+  - kind: ServiceAccount
+    name: ingress-controller
+    namespace: default
+  - apiGroup: rbac.authorization.k8s.io
+    kind: User
+    name: ingress-controller
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    run: ingress-default-backend
+  name: ingress-default-backend
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      run: ingress-default-backend
+  template:
+    metadata:
+      labels:
+        run: ingress-default-backend
+    spec:
+      containers:
+      - name: ingress-default-backend
+        image: gcr.io/google_containers/defaultbackend:1.0
+        ports:
+        - containerPort: 8080
+        resources:
+          limits:
+            cpu: 10m
+            memory: 20Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ingress-default-backend
+  namespace: default
+spec:
+  ports:
+  - port: 8080
+  selector:
+    run: ingress-default-backend
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: haproxy-ingress
+  namespace: default
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: haproxy-ingress-tcp
+  namespace: default
+data:
+  "22": "default/gitea-server:22"
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    run: haproxy-ingress
+  name: haproxy-ingress
+  namespace: default
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      run: haproxy-ingress
+  template:
+    metadata:
+      labels:
+        run: haproxy-ingress
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        role: ingress-controller
+      serviceAccountName: ingress-controller
+      containers:
+      - name: haproxy-ingress
+        image: quay.io/jcmoraisjr/haproxy-ingress
+        args:
+        - --default-backend-service=$(POD_NAMESPACE)/ingress-default-backend
+        - --configmap=$(POD_NAMESPACE)/haproxy-ingress
+        - --tcp-services-configmap=$(POD_NAMESPACE)/haproxy-ingress-tcp
+        - --sort-backends
+        ports:
+        - name: http
+          containerPort: 80
+        - name: https
+          containerPort: 443
+        - name: stat
+          containerPort: 1936
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10253
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          value: default
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitea-server-nodeport
+  namespace: default
+  labels:
+    app: gitea
+    name: gitea-server
+  annotations:
+    # add an annotation indicating the issuer to use.
+    # TODO: Switch to production when we're ready
+    certmanager.k8s.io/cluster-issuer: letsencrypt-staging
+spec:
+  ports:
+    - name: http
+      port: 3000
+      targetPort: 3000
+    - name: ssh
+      port: 22
+      targetPort: 22
+      protocol: TCP
+  type: NodePort
+  selector:
+    name: gitea-server
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: gitea-ingress
+  namespace: default
+  labels:
+    name: gitea-server
+    app: gitea
+  annotations:
+    kubernetes.io/ingress.class: "haproxy"
+spec:
+  tls:
+  - hosts:
+    - gitea.kosmos.org
+    secretName: gitea-kosmos-org-cert
+  rules:
+  - host: gitea.kosmos.org
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: gitea-server-nodeport
+          servicePort: 3000

kubernetes/gitea-server.yaml

@@ -32,8 +32,21 @@ spec:
         image: gitea/gitea:1.8.1
         ports:
         - containerPort: 3000
-        - containerPort: 3001
         - containerPort: 22
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 3000
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 3000
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
         volumeMounts:
         - mountPath: /data
           name: gitea-server-data
@@ -93,9 +106,6 @@ spec:
       targetPort: 22
     - name: "http"
       port: 80
-      targetPort: 3001
-    - name: "https"
-      port: 443
       targetPort: 3000
   selector:
     name: gitea-server

kubernetes/letsencrypt-production.yaml

@@ -0,0 +1,19 @@
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-production
+spec:
+  acme:
+    # You must replace this email address with your own.
+    # Let's Encrypt will use this to contact you about expiring
+    # certificates, and issues related to your account.
+    email: ops@kosmos.org
+    server: https://acme-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      # Secret resource used to store the account's private key.
+      name: letsencrypt-production-account-key
+    # Add a single challenge solver, HTTP01 using nginx
+    solvers:
+    - http01:
+        ingress:
+          class: nginx

kubernetes/letsencrypt-staging.yaml

@@ -0,0 +1,19 @@
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-staging
+spec:
+  acme:
+    # You must replace this email address with your own.
+    # Let's Encrypt will use this to contact you about expiring
+    # certificates, and issues related to your account.
+    email: ops@kosmos.org
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      # Secret resource used to store the account's private key.
+      name: letsencrypt-staging-account-key
+    # Add a single challenge solver, HTTP01 using nginx
+    solvers:
+    - http01:
+        ingress:
+          class: nginx