Compare commits

...

3 Commits

Author SHA1 Message Date
Râu Cao
23fe8d5723
Add basic Garage doc 2022-11-25 11:49:35 +01:00
Râu Cao
caf8342085
Add production environment, replication for garage
Also deploy a third node in a different data center
2022-11-24 20:01:18 +01:00
Râu Cao
b8bf462ae0
Install/configure Garage
Add a garage cookbook that installs the garage binary distribution and
creates the necessary configuration and system service.

Also deploy two new VMs to act as storage nodes.

refs #428
2022-11-24 13:59:22 +01:00
27 changed files with 680 additions and 0 deletions

4
clients/garage-2.json Normal file
View File

@ -0,0 +1,4 @@
{
"name": "garage-2",
"public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwuZbclnx/1Oas1+q5vUz\nsvCpTwKBrb3dah2YoZfZg0K15+MZshSyCZxo5T+SGp2OwhV65UptMJZbeyhVtzEp\ncN62G7exf65rNesXOL82PNQC6iInxNvyOgzdTOo7tdQ2ln/3QRpZOtUOB9PEkK17\nNmHfVIWKEc9YajRff5zE1LzSWulTNJ3D4GAIhsli//Rv45MhjyYoQKf1AXtqI72A\n2FE2YWXOjjSHJIPRfcUrmBOmEt/gkWySxGAs8Dg112vOC1ftk0KiQFWKVydMicIj\nyySQH1/neQFSq2HLNajDc9S2l7cjhPEjov7taS9LkXfPtnfN8ajEEP0S2MgZnf4N\ngwIDAQAB\n-----END PUBLIC KEY-----\n"
}

4
clients/garage-3.json Normal file
View File

@ -0,0 +1,4 @@
{
"name": "garage-3",
"public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtRSB8/ObjvQq6WuOVS/f\nypdX/2fLsUlt5tQ8GNuSY9rSM8gdvcXUvnPlxthZO4yvcPX85wmtBZX8fRJFdkJg\nYRCJbuVKO9sLTq8OUWXYpfU1q10FUhl034zxOMslpxVB6toirnk025vyq9jbuKP+\nYO+c40KZr67mgm0hveJfylayfiKP1HGm4HrV0maFivCgC8D+MPDDv75CsqRe5WSc\nh2CoauDJwVlhKZ92yq87ugGBhJJRUGOQZcfEvkUGj/HNAS6tuHl8YmVmhO8hBdee\nNto6RF54E1zB80R9oT/qitw23miEyUcHHVxhTR4tTWflZgd8l4wDOhX3Nf20xknu\nFQIDAQAB\n-----END PUBLIC KEY-----\n"
}

4
clients/garage-4.json Normal file
View File

@ -0,0 +1,4 @@
{
"name": "garage-4",
"public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA8it7QtT6zDiJJqlyHKfQ\nLqwu6bLblD15WWxlUSiOdhz3njWDv1BIDCAdkCR3HAXgxvk8sMj9QkvWS7u1+bc4\nxvHrY4Tgfg+Tk1h3gGa7ukll8s1WLIbGjj89vrK8PFr4iuDqRytYRMmcdMsNzPkS\nKcsOjFYWGV7KM/OwoQGVIOUPB+WtkrFAvNkXtIU6Wd5orzFMjt/9DPF2aO7QegL8\nG1mQmXcPGl9NSDUXptn/kzFKm/p4n7pjy6OypFT192ak7OA/s+CvQlaVE2tb/M3c\ne4J6A+PInV5AGKY6BxI3QRQLZIlqE0FXawFKr1iRU4JP4tVnICXZqy+SDXQU1zar\nTQIDAQAB\n-----END PUBLIC KEY-----\n"
}

View File

@ -0,0 +1,17 @@
{
"id": "garage",
"rpc_secret": {
"encrypted_data": "E3XtqLPuJXnRq6AIatVJe1+hoG236iRxz9s//qyYYgaBcvYRnBWwFSH/+cT9\n3bzZ+WE6lOqAPxYbj2riAPkdhdLbrR9tPipJNZyTncX5ByL510Q=\n",
"iv": "qBW1jUvUvHYPhjkn\n",
"auth_tag": "ao36nanF1pnAzmaoHwhJNQ==\n",
"version": 3,
"cipher": "aes-256-gcm"
},
"admin_token": {
"encrypted_data": "O0Cndl8n8/I1igGeMej46fSi9nje9CYGkLB/PfUhIxcZOkmRpvOnKSSn4B6l\nzC59xZmsEWT51hF4UmR1k2ATvWeLHdk24dWM/LK1Is16RmmlAeU=\n",
"iv": "kGTropuG44BUOJ7W\n",
"auth_tag": "/i9fVJ2iLcYSRZ5APe03qQ==\n",
"version": 3,
"cipher": "aes-256-gcm"
}
}

20
doc/garage.md Normal file
View File

@ -0,0 +1,20 @@
# Garage
Our S3-compatible object storage service is based on [Garage][1].
Garage is running as a cluster, with one VM each on 3 different physical hosts
in 2 different zones (data centers).
Replication mode is set to "2", meaning PUTs are always immediately synced to
nodes in 2 different zones (write/read consistency guaranteed by default).
When all nodes in one zone become unavailable, the cluster is switching to
read-only mode automatically, since the write quorum cannot be reached in that
case. If it is necessary (in an emergency) to write to nodes in only one zone,
you can change the replication mode to "2-dangerous", which lowers the write
quorum to 1.
Please refer to the [replication documentation][2] for more detailed information.
[1]: https://garagehq.deuxfleurs.fr
[2]: https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication-mode

View File

@ -0,0 +1,10 @@
{
"name": "production",
"override_attributes": {
"garage": {
"replication_mode": "2",
"s3_api_root_domain": ".s3.garage.kosmos.org",
"s3_web_root_domain": ".web.garage.kosmos.org"
}
}
}

64
nodes/garage-2.json Normal file
View File

@ -0,0 +1,64 @@
{
"name": "garage-2",
"chef_environment": "production",
"normal": {
"knife_zero": {
"host": "10.1.1.40"
}
},
"automatic": {
"fqdn": "garage-2",
"os": "linux",
"os_version": "5.4.0-132-generic",
"hostname": "garage-2",
"ipaddress": "192.168.122.241",
"roles": [
"base",
"kvm_guest",
"garage_node"
],
"recipes": [
"kosmos-base",
"kosmos-base::default",
"kosmos_kvm::guest",
"kosmos_garage",
"kosmos_garage::default",
"kosmos_garage::firewall",
"apt::default",
"timezone_iii::default",
"timezone_iii::debian",
"ntp::default",
"ntp::apparmor",
"kosmos-base::systemd_emails",
"apt::unattended-upgrades",
"kosmos-base::firewall",
"kosmos-postfix::default",
"postfix::default",
"postfix::_common",
"postfix::_attributes",
"postfix::sasl_auth",
"hostname::default",
"firewall::default",
"chef-sugar::default"
],
"platform": "ubuntu",
"platform_version": "20.04",
"cloud": null,
"chef_packages": {
"chef": {
"version": "17.10.3",
"chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib",
"chef_effortless": null
},
"ohai": {
"version": "17.9.0",
"ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
}
}
},
"run_list": [
"role[base]",
"role[kvm_guest]",
"role[garage_node]"
]
}

64
nodes/garage-3.json Normal file
View File

@ -0,0 +1,64 @@
{
"name": "garage-3",
"chef_environment": "production",
"normal": {
"knife_zero": {
"host": "10.1.1.39"
}
},
"automatic": {
"fqdn": "garage-3",
"os": "linux",
"os_version": "5.4.0-132-generic",
"hostname": "garage-3",
"ipaddress": "192.168.122.191",
"roles": [
"base",
"kvm_guest",
"garage_node"
],
"recipes": [
"kosmos-base",
"kosmos-base::default",
"kosmos_kvm::guest",
"kosmos_garage",
"kosmos_garage::default",
"kosmos_garage::firewall",
"apt::default",
"timezone_iii::default",
"timezone_iii::debian",
"ntp::default",
"ntp::apparmor",
"kosmos-base::systemd_emails",
"apt::unattended-upgrades",
"kosmos-base::firewall",
"kosmos-postfix::default",
"postfix::default",
"postfix::_common",
"postfix::_attributes",
"postfix::sasl_auth",
"hostname::default",
"firewall::default",
"chef-sugar::default"
],
"platform": "ubuntu",
"platform_version": "20.04",
"cloud": null,
"chef_packages": {
"chef": {
"version": "17.10.3",
"chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib",
"chef_effortless": null
},
"ohai": {
"version": "17.9.0",
"ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
}
}
},
"run_list": [
"role[base]",
"role[kvm_guest]",
"role[garage_node]"
]
}

64
nodes/garage-4.json Normal file
View File

@ -0,0 +1,64 @@
{
"name": "garage-4",
"chef_environment": "production",
"normal": {
"knife_zero": {
"host": "10.1.1.104"
}
},
"automatic": {
"fqdn": "garage-4",
"os": "linux",
"os_version": "5.4.0-132-generic",
"hostname": "garage-4",
"ipaddress": "192.168.122.123",
"roles": [
"base",
"kvm_guest",
"garage_node"
],
"recipes": [
"kosmos-base",
"kosmos-base::default",
"kosmos_kvm::guest",
"kosmos_garage",
"kosmos_garage::default",
"kosmos_garage::firewall",
"apt::default",
"timezone_iii::default",
"timezone_iii::debian",
"ntp::default",
"ntp::apparmor",
"kosmos-base::systemd_emails",
"apt::unattended-upgrades",
"kosmos-base::firewall",
"kosmos-postfix::default",
"postfix::default",
"postfix::_common",
"postfix::_attributes",
"postfix::sasl_auth",
"hostname::default",
"firewall::default",
"chef-sugar::default"
],
"platform": "ubuntu",
"platform_version": "20.04",
"cloud": null,
"chef_packages": {
"chef": {
"version": "17.10.3",
"chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib",
"chef_effortless": null
},
"ohai": {
"version": "17.9.0",
"ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
}
}
},
"run_list": [
"role[base]",
"role[kvm_guest]",
"role[garage_node]"
]
}

6
roles/garage_node.rb Normal file
View File

@ -0,0 +1,6 @@
name "garage_node"
run_list %w(
kosmos_garage::default
kosmos_garage::firewall
)

View File

@ -0,0 +1,32 @@
# Delivery for Local Phases Execution
#
# This file allows you to execute test phases locally on a workstation or
# in a CI pipeline. The delivery-cli will read this file and execute the
# command(s) that are configured for each phase. You can customize them
# by just modifying the phase key on this file.
#
# By default these phases are configured for Cookbook Workflow only
#
[local_phases]
unit = "echo skipping unit phase."
lint = "chef exec cookstyle"
# foodcritic has been deprecated in favor of cookstyle so we skip the syntax
# phase now.
syntax = "echo skipping syntax phase. Use lint phase instead."
provision = "chef exec kitchen create"
deploy = "chef exec kitchen converge"
smoke = "chef exec kitchen verify"
# The functional phase is optional, you can define it by uncommenting
# the line below and running the command: `delivery local functional`
# functional = ""
cleanup = "chef exec kitchen destroy"
# Remote project.toml file
#
# Instead of the local phases above, you may specify a remote URI location for
# the `project.toml` file. This is useful for teams that wish to centrally
# manage the behavior of the `delivery local` command across many different
# projects.
#
# remote_file = "https://url/project.toml"

25
site-cookbooks/kosmos_garage/.gitignore vendored Normal file
View File

@ -0,0 +1,25 @@
.vagrant
*~
*#
.#*
\#*#
.*.sw[a-z]
*.un~
# Bundler
Gemfile.lock
gems.locked
bin/*
.bundle/*
# test kitchen
.kitchen/
kitchen.local.yml
# Chef Infra
Berksfile.lock
.zero-knife.rb
Policyfile.lock.json
.idea/

View File

@ -0,0 +1,3 @@
source 'https://supermarket.chef.io'
metadata

View File

@ -0,0 +1,5 @@
# kosmos_garage CHANGELOG
## 0.1.0
Initial release.

View File

@ -0,0 +1,3 @@
Copyright 2021 The Authors
All rights reserved, do not redistribute.

View File

@ -0,0 +1,14 @@
# kosmos_garage
Configures/deploys Garage
## Integration tests
With a Docker daemon running on your system, change to
`site-cookbooks/kosmos_garage/`, and use the following commands to create,
converge, and verify a local node:
chef exec kitchen create
chef exec kitchen converge
chef exec kitchen verify
chef exec kitchen desroy

View File

@ -0,0 +1,10 @@
node.default['garage']['version'] = '0.8.0'
node.default['garage']['checksum']['amd64'] = '66dd2ea1f677281a43e10eb619523b1b269f8fde9047ce8caa70958f3b13ca74'
node.default['garage']['s3_api_port'] = 3900
node.default['garage']['rpc_port'] = 3901
node.default['garage']['s3_web_port'] = 3902
node.default['garage']['admin_port'] = 3903
node.default['garage']['k2v_api_port'] = 3904
node.default['garage']['s3_api_root_domain'] = '.s3.garage.localhost'
node.default['garage']['s3_web_root_domain'] = '.web.garage.localhost'
node.default['garage']['replication_mode'] = 'none'

View File

@ -0,0 +1,115 @@
# Put files/directories that should be ignored in this file when uploading
# to a Chef Infra Server or Supermarket.
# Lines that start with '# ' are comments.
# OS generated files #
######################
.DS_Store
ehthumbs.db
Icon?
nohup.out
Thumbs.db
.envrc
# EDITORS #
###########
.#*
.project
.settings
*_flymake
*_flymake.*
*.bak
*.sw[a-z]
*.tmproj
*~
\#*
REVISION
TAGS*
tmtags
.vscode
.editorconfig
## COMPILED ##
##############
*.class
*.com
*.dll
*.exe
*.o
*.pyc
*.so
*/rdoc/
a.out
mkmf.log
# Testing #
###########
.circleci/*
.codeclimate.yml
.delivery/*
.foodcritic
.kitchen*
.mdlrc
.overcommit.yml
.rspec
.rubocop.yml
.travis.yml
.watchr
.yamllint
azure-pipelines.yml
Dangerfile
examples/*
features/*
Guardfile
kitchen.yml*
mlc_config.json
Procfile
Rakefile
spec/*
test/*
# SCM #
#######
.git
.gitattributes
.gitconfig
.github/*
.gitignore
.gitkeep
.gitmodules
.svn
*/.bzr/*
*/.git
*/.hg/*
*/.svn/*
# Berkshelf #
#############
Berksfile
Berksfile.lock
cookbooks/*
tmp
# Bundler #
###########
vendor/*
Gemfile
Gemfile.lock
# Policyfile #
##############
Policyfile.rb
Policyfile.lock.json
# Documentation #
#############
CODE_OF_CONDUCT*
CONTRIBUTING*
documentation/*
TESTING*
UPGRADING*
# Vagrant #
###########
.vagrant
Vagrantfile

View File

@ -0,0 +1,3 @@
{
"name": "testing"
}

View File

@ -0,0 +1,36 @@
---
driver:
name: dokken
pull_platform_image: false
pull_chef_image: false
memory_limit: 2147483648 # 2GB
transport:
name: dokken
provisioner:
name: dokken
clean_dokken_sandbox: false
client_rb:
environment: testing
verifier:
name: inspec
platforms:
- name: ubuntu-20.04
driver:
image: dokken/ubuntu-20.04
privileged: true
pid_one_command: /usr/lib/systemd/systemd
suites:
- name: garage
data_bags_path: "test/integration/default/data_bags"
encrypted_data_bag_secret_key_path: "test/integration/default/encrypted_data_bag_secret"
run_list:
- recipe[kosmos_garage::default]
verifier:
inspec_tests:
- test/integration/default
attributes:

View File

@ -0,0 +1,11 @@
name 'kosmos_garage'
maintainer 'Kosmos Developers'
maintainer_email 'ops@kosmos.org'
license 'MIT'
description 'Installs/configures Garage'
version '0.1.0'
chef_version '>= 15.0'
issues_url 'https://gitea.kosmos.org/kosmos/chef/issues'
source_url 'https://gitea.kosmos.org/kosmos/chef'
depends 'firewall'

View File

@ -0,0 +1,65 @@
#
# Cookbook:: kosmos_garage
# Recipe:: default
#
remote_file 'garage' do
source "https://garagehq.deuxfleurs.fr/_releases/v#{node['garage']['version']}/x86_64-unknown-linux-musl/garage"
checksum node['garage']['checksum']['amd64']
path '/usr/local/bin/garage'
mode '0755'
ssl_verify_mode :verify_none if node.chef_environment == 'testing'
notifies :restart, 'service[garage]', :delayed
end
credentials = Chef::EncryptedDataBagItem.load('credentials', 'garage')
template '/etc/garage.toml' do
source 'garage.toml.erb'
mode '0744'
variables metadata_dir: node['garage']['metadata_dir'] || '/var/lib/garage/meta',
data_dir: node['garage']['data_dir'] || '/var/lib/garage/data',
db_engine: node['garage']['db_engine'] || 'lmdb',
rpc_port: node['garage']['rpc_port'],
rpc_public_addr: "#{node.dig('knife_zero', 'host') || '127.0.0.1'}:#{node['garage']['rpc_port']}",
rpc_secret: credentials['rpc_secret'],
s3_region: node['garage']['s3_region'] || 'garage',
s3_api_port: node['garage']['s3_api_port'],
s3_api_root_domain: node['garage']['s3_api_root_domain'] || '.s3.garage.localhost',
s3_web_port: node['garage']['s3_web_port'],
s3_web_root_domain: node['garage']['s3_web_root_domain'] || '.web.garage.localhost',
k2v_api_port: node['garage']['k2v_api_port'],
admin_port: node['garage']['admin_port'],
admin_token: credentials['admin_token'],
replication_mode: node['garage']['replication_mode']
notifies :restart, 'service[garage]', :delayed
end
systemd_unit 'garage.service' do
content({
Unit: {
Description: 'Garage Data Store',
Documentation: ['https://garagehq.deuxfleurs.fr/documentation/quick-start/'],
After: 'network-online.target',
Wants: 'network-online.target'
},
Service: {
Environment: 'RUST_LOG=garage=info RUST_BACKTRACE=1',
ExecStart: '/usr/local/bin/garage server',
StateDirectory: 'garage',
DynamicUser: true,
ProtectHome: true,
NoNewPrivileges: true
},
Install: {
WantedBy: 'multi-user.target'
}
})
verify false
triggers_reload true
action [:create]
end
service 'garage' do
action [:enable, :start]
end

View File

@ -0,0 +1,36 @@
include_recipe 'firewall'
firewall_rule 'garage_s3_api' do
command :allow
protocol :tcp
source "10.1.1.0/24"
port node['garage']['s3_api_port']
end
firewall_rule 'garage_rpc' do
command :allow
protocol :tcp
source "10.1.1.0/24"
port node['garage']['rpc_port']
end
firewall_rule 'garage_s3_web' do
command :allow
protocol :tcp
source "10.1.1.0/24"
port node['garage']['s3_web_port']
end
firewall_rule 'garage_admin' do
command :allow
protocol :tcp
source "10.1.1.0/24"
port node['garage']['admin_port']
end
firewall_rule 'garage_k2v_api' do
command :allow
protocol :tcp
source "10.1.1.0/24"
port node['garage']['k2v_api_port']
end

View File

@ -0,0 +1,26 @@
metadata_dir = "<%= @metadata_dir %>"
data_dir = "<%= @data_dir %>"
db_engine = "<%= @db_engine %>"
replication_mode = "<%= @replication_mode %>"
rpc_bind_addr = "[::]:<%= @rpc_port %>"
rpc_public_addr = "<%= @rpc_public_addr %>"
rpc_secret = "<%= @rpc_secret %>"
[s3_api]
s3_region = "<%= @s3_region %>"
api_bind_addr = "[::]:<%= @s3_api_port %>"
root_domain = "<%= @s3_api_root_domain %>"
[s3_web]
bind_addr = "[::]:<%= @s3_web_port %>"
root_domain = "<%= @s3_web_root_domain %>"
index = "index.html"
[k2v_api]
api_bind_addr = "[::]:<%= @k2v_api_port %>"
[admin]
api_bind_addr = "0.0.0.0:<%= @admin_port %>"
admin_token = "<%= @admin_token %>"

View File

@ -0,0 +1,17 @@
{
"id": "garage",
"admin_token": {
"encrypted_data": "BYRysR7CokS4943A3QL4/bN4dDdBs7TWgzbuTntB7cBIJqpnUbu2sd9PNjxo\nSjGTlwdnUUlbmCJzPfQ8oKCINrs+yilH3XIyzb4x//3h9rzE+qI=\n",
"iv": "rwOuaLi2kwg2Uw9g\n",
"auth_tag": "68j6nGYan1DiQQKmmpPW9A==\n",
"version": 3,
"cipher": "aes-256-gcm"
},
"rpc_secret": {
"encrypted_data": "x9qfeSGGBkGsErJ1vQuQKTcNksPh3xnyM1V09CvecNewVHkmWeP03WE3gjJH\nzUWooHrDn2Gaci8Pi9VYHg6+Gsw/w/l6BhTWByd6k/251pNTRps=\n",
"iv": "/QiOCspNokU3QCDB\n",
"auth_tag": "MWkUnKEAEDduPLG0kWd8Bg==\n",
"version": 3,
"cipher": "aes-256-gcm"
}
}

View File

@ -0,0 +1,21 @@
# InSpec test for recipe kosmos_garage::garagej
# The Chef InSpec reference, with examples and extensive documentation, can be
# found at https://docs.chef.io/inspec/resources/
describe file('/usr/local/bin/garage') do
it { should exist }
its('mode') { should cmp '00755' }
end
describe service('garage') do
it { should be_enabled }
it { should be_installed }
it { should be_running }
end
[3900, 3901, 3902, 3903, 3904].each do |port_number|
describe port(port_number) do
it { should be_listening }
end
end

View File

@ -0,0 +1 @@
I7L6DW2FOYGfZIVrXWN6aLYTswhHnDR7ZmdlGp5KeAlpEIjaB7IgPO3EAV/QxgzMaHJzBwDzThpp8PTAUXUH+9M5I+G06Y/F21cCkpe5mNOjiP9SukdggGbiZdMRxS//Ym511hqrWSpowF2yYwX6V/rprlWAc0EKzZWNDceC2g/sKdX725/TXz4mhouimr8GFnlCqPoBPpM8OFPQy2TMoK66IXjA6zW190FhpySsAWE+Evnjj1FtVjTrQBW8A00NVU5SH8APn21RLZGY/uk/acX9sKsa5XcUSU1MlKU5DZejLzu1Vf0DKoDAQpI9/JkQajOxhCC77buXWP83GL0Ru9w9eWq2rYQuDawRKZViHC5aFatPZFu0GvRq58WcbordxDtH+TZYoYEIIf32kCe0sNxbO8iAgMHD7HV8n21zHVps7nTD4HRbXvURA+fwOkQSxcxET3pS5AJDbVIJ6FHbn3MZj9MCv2miwAVCYcZ5Wt+dhmQxjYgZ8PUUlVvB+eCauY2m6J4+nEjg+yx15yvdnBOmEGmj+HQG+rdD+6mAbtK2Adwq2zoqkQbUKJNmyZAvHEkCLHmWswdV8kN2Eh35wlyG/9FBQ7WAlsvpwT8cjHvxwsh644EsBJPlmoBXT0I5Q5S6Gx6PDP/x/swO2VNgISWOLimPvUD1bI4ObIiVTWo=