diff --git a/clients/garage-2.json b/clients/garage-2.json new file mode 100644 index 0000000..8b43f12 --- /dev/null +++ b/clients/garage-2.json @@ -0,0 +1,4 @@ +{ + "name": "garage-2", + "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwuZbclnx/1Oas1+q5vUz\nsvCpTwKBrb3dah2YoZfZg0K15+MZshSyCZxo5T+SGp2OwhV65UptMJZbeyhVtzEp\ncN62G7exf65rNesXOL82PNQC6iInxNvyOgzdTOo7tdQ2ln/3QRpZOtUOB9PEkK17\nNmHfVIWKEc9YajRff5zE1LzSWulTNJ3D4GAIhsli//Rv45MhjyYoQKf1AXtqI72A\n2FE2YWXOjjSHJIPRfcUrmBOmEt/gkWySxGAs8Dg112vOC1ftk0KiQFWKVydMicIj\nyySQH1/neQFSq2HLNajDc9S2l7cjhPEjov7taS9LkXfPtnfN8ajEEP0S2MgZnf4N\ngwIDAQAB\n-----END PUBLIC KEY-----\n" +} \ No newline at end of file diff --git a/clients/garage-3.json b/clients/garage-3.json new file mode 100644 index 0000000..49ea275 --- /dev/null +++ b/clients/garage-3.json @@ -0,0 +1,4 @@ +{ + "name": "garage-3", + "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtRSB8/ObjvQq6WuOVS/f\nypdX/2fLsUlt5tQ8GNuSY9rSM8gdvcXUvnPlxthZO4yvcPX85wmtBZX8fRJFdkJg\nYRCJbuVKO9sLTq8OUWXYpfU1q10FUhl034zxOMslpxVB6toirnk025vyq9jbuKP+\nYO+c40KZr67mgm0hveJfylayfiKP1HGm4HrV0maFivCgC8D+MPDDv75CsqRe5WSc\nh2CoauDJwVlhKZ92yq87ugGBhJJRUGOQZcfEvkUGj/HNAS6tuHl8YmVmhO8hBdee\nNto6RF54E1zB80R9oT/qitw23miEyUcHHVxhTR4tTWflZgd8l4wDOhX3Nf20xknu\nFQIDAQAB\n-----END PUBLIC KEY-----\n" +} \ No newline at end of file diff --git a/clients/garage-4.json b/clients/garage-4.json new file mode 100644 index 0000000..52b8248 --- /dev/null +++ b/clients/garage-4.json @@ -0,0 +1,4 @@ +{ + "name": "garage-4", + "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA8it7QtT6zDiJJqlyHKfQ\nLqwu6bLblD15WWxlUSiOdhz3njWDv1BIDCAdkCR3HAXgxvk8sMj9QkvWS7u1+bc4\nxvHrY4Tgfg+Tk1h3gGa7ukll8s1WLIbGjj89vrK8PFr4iuDqRytYRMmcdMsNzPkS\nKcsOjFYWGV7KM/OwoQGVIOUPB+WtkrFAvNkXtIU6Wd5orzFMjt/9DPF2aO7QegL8\nG1mQmXcPGl9NSDUXptn/kzFKm/p4n7pjy6OypFT192ak7OA/s+CvQlaVE2tb/M3c\ne4J6A+PInV5AGKY6BxI3QRQLZIlqE0FXawFKr1iRU4JP4tVnICXZqy+SDXQU1zar\nTQIDAQAB\n-----END PUBLIC KEY-----\n" +} \ No newline at end of file diff --git a/data_bags/credentials/garage.json b/data_bags/credentials/garage.json new file mode 100644 index 0000000..7097d4b --- /dev/null +++ b/data_bags/credentials/garage.json @@ -0,0 +1,17 @@ +{ + "id": "garage", + "rpc_secret": { + "encrypted_data": "E3XtqLPuJXnRq6AIatVJe1+hoG236iRxz9s//qyYYgaBcvYRnBWwFSH/+cT9\n3bzZ+WE6lOqAPxYbj2riAPkdhdLbrR9tPipJNZyTncX5ByL510Q=\n", + "iv": "qBW1jUvUvHYPhjkn\n", + "auth_tag": "ao36nanF1pnAzmaoHwhJNQ==\n", + "version": 3, + "cipher": "aes-256-gcm" + }, + "admin_token": { + "encrypted_data": "O0Cndl8n8/I1igGeMej46fSi9nje9CYGkLB/PfUhIxcZOkmRpvOnKSSn4B6l\nzC59xZmsEWT51hF4UmR1k2ATvWeLHdk24dWM/LK1Is16RmmlAeU=\n", + "iv": "kGTropuG44BUOJ7W\n", + "auth_tag": "/i9fVJ2iLcYSRZ5APe03qQ==\n", + "version": 3, + "cipher": "aes-256-gcm" + } +} \ No newline at end of file diff --git a/data_bags/credentials/gitea.json b/data_bags/credentials/gitea.json index f8431b2..f976777 100644 --- a/data_bags/credentials/gitea.json +++ b/data_bags/credentials/gitea.json @@ -1,30 +1,51 @@ { "id": "gitea", "jwt_secret": { - "encrypted_data": "jTNhXpJ1mhUXjfRZ3OAR8lrGgxyyob44kN0TyNec5zO2Wb46hJgYMWwtKlZ9\nohNexOKV+wXCjZNeVw0kNgI=\n", - "iv": "NYkJTeTzLilMLptE\n", - "auth_tag": "a/PuBmOmhyCx0ooepz7n1w==\n", + "encrypted_data": "suy7Vwlg7tyJFBSjlnNRv7qR4jp1o9F0TbwxGcwWqbCpQW2NHl9QS1SCXJml\n4UbKklppjp+7Axvvs7YiOX8=\n", + "iv": "ojZAtLDxV6569XHN\n", + "auth_tag": "j15eLXjGMIIsXh5dHET/lw==\n", "version": 3, "cipher": "aes-256-gcm" }, "internal_token": { - "encrypted_data": "HbyEfyrupc06vGHhSqKUUT8NAIrlvbK4LbMdqxmJMgeltvDItqGgFa0ZdD51\n0djRqQMrRZ4MEdqVTFSBL+8QVdriKeUcLcummp52Sp9tYZKSQKympJFx3fsS\n49rBJhDKRlc3+jUpejJu4jHY4xR2MMNvWWqkkufTvZHhzg==\n", - "iv": "DUSCP7Q3dgjyYXwl\n", - "auth_tag": "HkPLLvY8uVNK871OsMshcg==\n", + "encrypted_data": "y7VG9w8Gz/jxgz86p/OtpVvJBYjD6yGOPhCM3SEPlbQF/gqI8VuTkJlUQLFB\nrsPiCcjjynuTPJPLvdkVUu1XjOfp5dtbPDc0hqp8KhvBx4DhnH7Mspp/kWfb\n9DWzJ6zeGBB/nrNay0jTV1MoqzKc3Nl0GSkzBLMbr15vVw==\n", + "iv": "wcx+w1Ij5Dee/81s\n", + "auth_tag": "C7QMXezMU+jcYZAjlm86rg==\n", "version": 3, "cipher": "aes-256-gcm" }, "secret_key": { - "encrypted_data": "bvxdPokzagjZkdGG37hbWBi6ywu+1UuOrlJJ4p5zOG03b4PN4N40ztO4fWr5\ncMHfO7FER779fRc+tA2H7L1SKqSvlJThgk7X8R7AGGQmrQy7Jvc=\n", - "iv": "0uTGeUjnbvnW2WGp\n", - "auth_tag": "Dzfb3Jiim5eYWfwpN3HO5Q==\n", + "encrypted_data": "4DGRaIbqqa5oCzFwNUjRPcP+uauWidjWwmBZY0BNyI3c/XmQBEb8wGV9Leoc\n3avqM5jhS/Ov43SBMpCrR71x4eAPJ3vlSeQ3GnpkgFyWfolmbEg=\n", + "iv": "SOTJFH8JkBNtPKyF\n", + "auth_tag": "fYSfkMMvGnPdiBOP7NnP8Q==\n", "version": 3, "cipher": "aes-256-gcm" }, "postgresql_password": { - "encrypted_data": "yv2gQYUxMTa7eeC0GJqE+fujOvM9GIwj/OL/L1wvn7uNTjJE97Xt1gYXRw==\n", - "iv": "F6yrDSav9EShCf2N\n", - "auth_tag": "08b4vT71g41qu6A6jZ6opw==\n", + "encrypted_data": "tA/mMteX2aO7dozNe/YWB8S9sVDdUgzKDnAdgnsXF5qTVT0slHe3KRg7og==\n", + "iv": "3/rdo8uCdhrFOWOf\n", + "auth_tag": "uNl4R3T5ylEBgAM8P6fdYA==\n", + "version": 3, + "cipher": "aes-256-gcm" + }, + "s3_key_id": { + "encrypted_data": "Pjaw1MM+GNZN68XDbM+PGJUwSSXwu1+ASgm4S0VZ3MvylVG3uBPdqdDUZ9g8\n", + "iv": "mPL4HvodGKMD+30N\n", + "auth_tag": "nrej5vDLEzAI9HkKJxa/mQ==\n", + "version": 3, + "cipher": "aes-256-gcm" + }, + "s3_secret_key": { + "encrypted_data": "yBWAUGyyoetZ8EDD+kVffGDQbFPVXxpiWCdWL5xn3ohlclrrcWBQP/cGj2Ts\nlSZ2l4ZIuHX6ZdAHe5O2C1h5nYVtWx+u5kVa9n6EoUbz/6iseHU=\n", + "iv": "jmIdQZVMCLLKs1pi\n", + "auth_tag": "0Jvgjuvhv11/QNV43zm1LQ==\n", + "version": 3, + "cipher": "aes-256-gcm" + }, + "s3_bucket": { + "encrypted_data": "MyR5WhJMGfu+StFPVt3wSzVSNsHnEiLfzKXm2xJeb/cEQVw=\n", + "iv": "CHmMCjdVzw+qKHIV\n", + "auth_tag": "tiQegK0hQfCjcgRxg1G8Rg==\n", "version": 3, "cipher": "aes-256-gcm" } diff --git a/doc/garage.md b/doc/garage.md new file mode 100644 index 0000000..f9eb3ae --- /dev/null +++ b/doc/garage.md @@ -0,0 +1,20 @@ +# Garage + +Our S3-compatible object storage service is based on [Garage][1]. + +Garage is running as a cluster, with one VM each on 3 different physical hosts +in 2 different zones (data centers). + +Replication mode is set to "2", meaning PUTs are always immediately synced to +nodes in 2 different zones (write/read consistency guaranteed by default). + +When all nodes in one zone become unavailable, the cluster is switching to +read-only mode automatically, since the write quorum cannot be reached in that +case. If it is necessary (in an emergency) to write to nodes in only one zone, +you can change the replication mode to "2-dangerous", which lowers the write +quorum to 1. + +Please refer to the [replication documentation][2] for more detailed information. + +[1]: https://garagehq.deuxfleurs.fr +[2]: https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication-mode diff --git a/environments/production.json b/environments/production.json new file mode 100644 index 0000000..ac5f7e6 --- /dev/null +++ b/environments/production.json @@ -0,0 +1,21 @@ +{ + "name": "production", + "override_attributes": { + "garage": { + "replication_mode": "2", + "s3_api_root_domain": ".s3.garage.kosmos.org", + "s3_web_root_domain": ".web.garage.kosmos.org" + }, + "gitea": { + "postgresql_host": "pg.kosmos.local:5432", + "config": { + "storage": { + "type": "minio", + "endpoint": "localhost:3900", + "location": "garage", + "use_ssl": "false" + } + } + } + } +} \ No newline at end of file diff --git a/nodes/garage-2.json b/nodes/garage-2.json new file mode 100644 index 0000000..5d80fc4 --- /dev/null +++ b/nodes/garage-2.json @@ -0,0 +1,64 @@ +{ + "name": "garage-2", + "chef_environment": "production", + "normal": { + "knife_zero": { + "host": "10.1.1.40" + } + }, + "automatic": { + "fqdn": "garage-2", + "os": "linux", + "os_version": "5.4.0-132-generic", + "hostname": "garage-2", + "ipaddress": "192.168.122.241", + "roles": [ + "base", + "kvm_guest", + "garage_node" + ], + "recipes": [ + "kosmos-base", + "kosmos-base::default", + "kosmos_kvm::guest", + "kosmos_garage", + "kosmos_garage::default", + "kosmos_garage::firewall", + "apt::default", + "timezone_iii::default", + "timezone_iii::debian", + "ntp::default", + "ntp::apparmor", + "kosmos-base::systemd_emails", + "apt::unattended-upgrades", + "kosmos-base::firewall", + "kosmos-postfix::default", + "postfix::default", + "postfix::_common", + "postfix::_attributes", + "postfix::sasl_auth", + "hostname::default", + "firewall::default", + "chef-sugar::default" + ], + "platform": "ubuntu", + "platform_version": "20.04", + "cloud": null, + "chef_packages": { + "chef": { + "version": "17.10.3", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib", + "chef_effortless": null + }, + "ohai": { + "version": "17.9.0", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai" + } + } + }, + "run_list": [ + "role[base]", + "role[kvm_guest]", + "role[garage_node]" + ] +} \ No newline at end of file diff --git a/nodes/garage-3.json b/nodes/garage-3.json new file mode 100644 index 0000000..3205be1 --- /dev/null +++ b/nodes/garage-3.json @@ -0,0 +1,64 @@ +{ + "name": "garage-3", + "chef_environment": "production", + "normal": { + "knife_zero": { + "host": "10.1.1.39" + } + }, + "automatic": { + "fqdn": "garage-3", + "os": "linux", + "os_version": "5.4.0-132-generic", + "hostname": "garage-3", + "ipaddress": "192.168.122.191", + "roles": [ + "base", + "kvm_guest", + "garage_node" + ], + "recipes": [ + "kosmos-base", + "kosmos-base::default", + "kosmos_kvm::guest", + "kosmos_garage", + "kosmos_garage::default", + "kosmos_garage::firewall", + "apt::default", + "timezone_iii::default", + "timezone_iii::debian", + "ntp::default", + "ntp::apparmor", + "kosmos-base::systemd_emails", + "apt::unattended-upgrades", + "kosmos-base::firewall", + "kosmos-postfix::default", + "postfix::default", + "postfix::_common", + "postfix::_attributes", + "postfix::sasl_auth", + "hostname::default", + "firewall::default", + "chef-sugar::default" + ], + "platform": "ubuntu", + "platform_version": "20.04", + "cloud": null, + "chef_packages": { + "chef": { + "version": "17.10.3", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib", + "chef_effortless": null + }, + "ohai": { + "version": "17.9.0", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai" + } + } + }, + "run_list": [ + "role[base]", + "role[kvm_guest]", + "role[garage_node]" + ] +} \ No newline at end of file diff --git a/nodes/garage-4.json b/nodes/garage-4.json new file mode 100644 index 0000000..1d44336 --- /dev/null +++ b/nodes/garage-4.json @@ -0,0 +1,64 @@ +{ + "name": "garage-4", + "chef_environment": "production", + "normal": { + "knife_zero": { + "host": "10.1.1.104" + } + }, + "automatic": { + "fqdn": "garage-4", + "os": "linux", + "os_version": "5.4.0-132-generic", + "hostname": "garage-4", + "ipaddress": "192.168.122.123", + "roles": [ + "base", + "kvm_guest", + "garage_node" + ], + "recipes": [ + "kosmos-base", + "kosmos-base::default", + "kosmos_kvm::guest", + "kosmos_garage", + "kosmos_garage::default", + "kosmos_garage::firewall", + "apt::default", + "timezone_iii::default", + "timezone_iii::debian", + "ntp::default", + "ntp::apparmor", + "kosmos-base::systemd_emails", + "apt::unattended-upgrades", + "kosmos-base::firewall", + "kosmos-postfix::default", + "postfix::default", + "postfix::_common", + "postfix::_attributes", + "postfix::sasl_auth", + "hostname::default", + "firewall::default", + "chef-sugar::default" + ], + "platform": "ubuntu", + "platform_version": "20.04", + "cloud": null, + "chef_packages": { + "chef": { + "version": "17.10.3", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib", + "chef_effortless": null + }, + "ohai": { + "version": "17.9.0", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai" + } + } + }, + "run_list": [ + "role[base]", + "role[kvm_guest]", + "role[garage_node]" + ] +} \ No newline at end of file diff --git a/nodes/gitea-2.json b/nodes/gitea-2.json index 4aef119..d326adc 100644 --- a/nodes/gitea-2.json +++ b/nodes/gitea-2.json @@ -1,5 +1,6 @@ { "name": "gitea-2", + "chef_environment": "production", "normal": { "knife_zero": { "host": "10.1.1.21" @@ -13,6 +14,7 @@ "ipaddress": "192.168.122.189", "roles": [ "kvm_guest", + "garage_gateway", "gitea", "postgresql_client" ], @@ -20,6 +22,8 @@ "kosmos-base", "kosmos-base::default", "kosmos_kvm::guest", + "kosmos_garage", + "kosmos_garage::default", "kosmos_postgresql::hostsfile", "kosmos_gitea", "kosmos_gitea::default", @@ -58,8 +62,9 @@ } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", + "role[garage_gateway]", "role[gitea]" ] -} \ No newline at end of file +} diff --git a/roles/garage_gateway.rb b/roles/garage_gateway.rb new file mode 100644 index 0000000..fb65920 --- /dev/null +++ b/roles/garage_gateway.rb @@ -0,0 +1,6 @@ +name "garage_gateway" + +run_list %w( + kosmos_garage::default + kosmos_garage::firewall_rpc +) diff --git a/roles/garage_node.rb b/roles/garage_node.rb new file mode 100644 index 0000000..e9b06fb --- /dev/null +++ b/roles/garage_node.rb @@ -0,0 +1,7 @@ +name "garage_node" + +run_list %w( + kosmos_garage::default + kosmos_garage::firewall_rpc + kosmos_garage::firewall_apis +) diff --git a/site-cookbooks/kosmos_garage/.delivery/project.toml b/site-cookbooks/kosmos_garage/.delivery/project.toml new file mode 100644 index 0000000..3496f78 --- /dev/null +++ b/site-cookbooks/kosmos_garage/.delivery/project.toml @@ -0,0 +1,32 @@ +# Delivery for Local Phases Execution +# +# This file allows you to execute test phases locally on a workstation or +# in a CI pipeline. The delivery-cli will read this file and execute the +# command(s) that are configured for each phase. You can customize them +# by just modifying the phase key on this file. +# +# By default these phases are configured for Cookbook Workflow only +# + +[local_phases] +unit = "echo skipping unit phase." +lint = "chef exec cookstyle" +# foodcritic has been deprecated in favor of cookstyle so we skip the syntax +# phase now. +syntax = "echo skipping syntax phase. Use lint phase instead." +provision = "chef exec kitchen create" +deploy = "chef exec kitchen converge" +smoke = "chef exec kitchen verify" +# The functional phase is optional, you can define it by uncommenting +# the line below and running the command: `delivery local functional` +# functional = "" +cleanup = "chef exec kitchen destroy" + +# Remote project.toml file +# +# Instead of the local phases above, you may specify a remote URI location for +# the `project.toml` file. This is useful for teams that wish to centrally +# manage the behavior of the `delivery local` command across many different +# projects. +# +# remote_file = "https://url/project.toml" \ No newline at end of file diff --git a/site-cookbooks/kosmos_garage/.gitignore b/site-cookbooks/kosmos_garage/.gitignore new file mode 100644 index 0000000..f1e57b8 --- /dev/null +++ b/site-cookbooks/kosmos_garage/.gitignore @@ -0,0 +1,25 @@ +.vagrant +*~ +*# +.#* +\#*# +.*.sw[a-z] +*.un~ + +# Bundler +Gemfile.lock +gems.locked +bin/* +.bundle/* + +# test kitchen +.kitchen/ +kitchen.local.yml + +# Chef Infra +Berksfile.lock +.zero-knife.rb +Policyfile.lock.json + +.idea/ + diff --git a/site-cookbooks/kosmos_garage/Berksfile b/site-cookbooks/kosmos_garage/Berksfile new file mode 100644 index 0000000..34fea21 --- /dev/null +++ b/site-cookbooks/kosmos_garage/Berksfile @@ -0,0 +1,3 @@ +source 'https://supermarket.chef.io' + +metadata diff --git a/site-cookbooks/kosmos_garage/CHANGELOG.md b/site-cookbooks/kosmos_garage/CHANGELOG.md new file mode 100644 index 0000000..0576387 --- /dev/null +++ b/site-cookbooks/kosmos_garage/CHANGELOG.md @@ -0,0 +1,5 @@ +# kosmos_garage CHANGELOG + +## 0.1.0 + +Initial release. diff --git a/site-cookbooks/kosmos_garage/LICENSE b/site-cookbooks/kosmos_garage/LICENSE new file mode 100644 index 0000000..080dee9 --- /dev/null +++ b/site-cookbooks/kosmos_garage/LICENSE @@ -0,0 +1,3 @@ +Copyright 2021 The Authors + +All rights reserved, do not redistribute. diff --git a/site-cookbooks/kosmos_garage/README.md b/site-cookbooks/kosmos_garage/README.md new file mode 100644 index 0000000..aaf85cb --- /dev/null +++ b/site-cookbooks/kosmos_garage/README.md @@ -0,0 +1,14 @@ +# kosmos_garage + +Configures/deploys Garage + +## Integration tests + +With a Docker daemon running on your system, change to +`site-cookbooks/kosmos_garage/`, and use the following commands to create, +converge, and verify a local node: + + chef exec kitchen create + chef exec kitchen converge + chef exec kitchen verify + chef exec kitchen desroy diff --git a/site-cookbooks/kosmos_garage/attributes/default.rb b/site-cookbooks/kosmos_garage/attributes/default.rb new file mode 100644 index 0000000..068ede8 --- /dev/null +++ b/site-cookbooks/kosmos_garage/attributes/default.rb @@ -0,0 +1,10 @@ +node.default['garage']['version'] = '0.8.0' +node.default['garage']['checksum']['amd64'] = '66dd2ea1f677281a43e10eb619523b1b269f8fde9047ce8caa70958f3b13ca74' +node.default['garage']['s3_api_port'] = 3900 +node.default['garage']['rpc_port'] = 3901 +node.default['garage']['s3_web_port'] = 3902 +node.default['garage']['admin_port'] = 3903 +node.default['garage']['k2v_api_port'] = 3904 +node.default['garage']['s3_api_root_domain'] = '.s3.garage.localhost' +node.default['garage']['s3_web_root_domain'] = '.web.garage.localhost' +node.default['garage']['replication_mode'] = 'none' diff --git a/site-cookbooks/kosmos_garage/chefignore b/site-cookbooks/kosmos_garage/chefignore new file mode 100644 index 0000000..cc170ea --- /dev/null +++ b/site-cookbooks/kosmos_garage/chefignore @@ -0,0 +1,115 @@ +# Put files/directories that should be ignored in this file when uploading +# to a Chef Infra Server or Supermarket. +# Lines that start with '# ' are comments. + +# OS generated files # +###################### +.DS_Store +ehthumbs.db +Icon? +nohup.out +Thumbs.db +.envrc + +# EDITORS # +########### +.#* +.project +.settings +*_flymake +*_flymake.* +*.bak +*.sw[a-z] +*.tmproj +*~ +\#* +REVISION +TAGS* +tmtags +.vscode +.editorconfig + +## COMPILED ## +############## +*.class +*.com +*.dll +*.exe +*.o +*.pyc +*.so +*/rdoc/ +a.out +mkmf.log + +# Testing # +########### +.circleci/* +.codeclimate.yml +.delivery/* +.foodcritic +.kitchen* +.mdlrc +.overcommit.yml +.rspec +.rubocop.yml +.travis.yml +.watchr +.yamllint +azure-pipelines.yml +Dangerfile +examples/* +features/* +Guardfile +kitchen.yml* +mlc_config.json +Procfile +Rakefile +spec/* +test/* + +# SCM # +####### +.git +.gitattributes +.gitconfig +.github/* +.gitignore +.gitkeep +.gitmodules +.svn +*/.bzr/* +*/.git +*/.hg/* +*/.svn/* + +# Berkshelf # +############# +Berksfile +Berksfile.lock +cookbooks/* +tmp + +# Bundler # +########### +vendor/* +Gemfile +Gemfile.lock + +# Policyfile # +############## +Policyfile.rb +Policyfile.lock.json + +# Documentation # +############# +CODE_OF_CONDUCT* +CONTRIBUTING* +documentation/* +TESTING* +UPGRADING* + +# Vagrant # +########### +.vagrant +Vagrantfile diff --git a/site-cookbooks/kosmos_garage/environments/testing.json b/site-cookbooks/kosmos_garage/environments/testing.json new file mode 100644 index 0000000..0d3d5b3 --- /dev/null +++ b/site-cookbooks/kosmos_garage/environments/testing.json @@ -0,0 +1,3 @@ +{ + "name": "testing" +} \ No newline at end of file diff --git a/site-cookbooks/kosmos_garage/kitchen.yml b/site-cookbooks/kosmos_garage/kitchen.yml new file mode 100644 index 0000000..699bf10 --- /dev/null +++ b/site-cookbooks/kosmos_garage/kitchen.yml @@ -0,0 +1,36 @@ +--- +driver: + name: dokken + pull_platform_image: false + pull_chef_image: false + memory_limit: 2147483648 # 2GB + +transport: + name: dokken + +provisioner: + name: dokken + clean_dokken_sandbox: false + client_rb: + environment: testing + +verifier: + name: inspec + +platforms: + - name: ubuntu-20.04 + driver: + image: dokken/ubuntu-20.04 + privileged: true + pid_one_command: /usr/lib/systemd/systemd + +suites: + - name: garage + data_bags_path: "test/integration/default/data_bags" + encrypted_data_bag_secret_key_path: "test/integration/default/encrypted_data_bag_secret" + run_list: + - recipe[kosmos_garage::default] + verifier: + inspec_tests: + - test/integration/default + attributes: diff --git a/site-cookbooks/kosmos_garage/metadata.rb b/site-cookbooks/kosmos_garage/metadata.rb new file mode 100644 index 0000000..f1fde81 --- /dev/null +++ b/site-cookbooks/kosmos_garage/metadata.rb @@ -0,0 +1,11 @@ +name 'kosmos_garage' +maintainer 'Kosmos Developers' +maintainer_email 'ops@kosmos.org' +license 'MIT' +description 'Installs/configures Garage' +version '0.1.0' +chef_version '>= 15.0' +issues_url 'https://gitea.kosmos.org/kosmos/chef/issues' +source_url 'https://gitea.kosmos.org/kosmos/chef' + +depends 'firewall' diff --git a/site-cookbooks/kosmos_garage/recipes/default.rb b/site-cookbooks/kosmos_garage/recipes/default.rb new file mode 100644 index 0000000..214bca8 --- /dev/null +++ b/site-cookbooks/kosmos_garage/recipes/default.rb @@ -0,0 +1,65 @@ +# +# Cookbook:: kosmos_garage +# Recipe:: default +# + +remote_file 'garage' do + source "https://garagehq.deuxfleurs.fr/_releases/v#{node['garage']['version']}/x86_64-unknown-linux-musl/garage" + checksum node['garage']['checksum']['amd64'] + path '/usr/local/bin/garage' + mode '0755' + ssl_verify_mode :verify_none if node.chef_environment == 'testing' + notifies :restart, 'service[garage]', :delayed +end + +credentials = Chef::EncryptedDataBagItem.load('credentials', 'garage') + +template '/etc/garage.toml' do + source 'garage.toml.erb' + mode '0744' + variables metadata_dir: node['garage']['metadata_dir'] || '/var/lib/garage/meta', + data_dir: node['garage']['data_dir'] || '/var/lib/garage/data', + db_engine: node['garage']['db_engine'] || 'lmdb', + rpc_port: node['garage']['rpc_port'], + rpc_public_addr: "#{node.dig('knife_zero', 'host') || '127.0.0.1'}:#{node['garage']['rpc_port']}", + rpc_secret: credentials['rpc_secret'], + s3_region: node['garage']['s3_region'] || 'garage', + s3_api_port: node['garage']['s3_api_port'], + s3_api_root_domain: node['garage']['s3_api_root_domain'] || '.s3.garage.localhost', + s3_web_port: node['garage']['s3_web_port'], + s3_web_root_domain: node['garage']['s3_web_root_domain'] || '.web.garage.localhost', + k2v_api_port: node['garage']['k2v_api_port'], + admin_port: node['garage']['admin_port'], + admin_token: credentials['admin_token'], + replication_mode: node['garage']['replication_mode'] + notifies :restart, 'service[garage]', :delayed +end + +systemd_unit 'garage.service' do + content({ + Unit: { + Description: 'Garage Data Store', + Documentation: ['https://garagehq.deuxfleurs.fr/documentation/quick-start/'], + After: 'network-online.target', + Wants: 'network-online.target' + }, + Service: { + Environment: 'RUST_LOG=garage=info RUST_BACKTRACE=1', + ExecStart: '/usr/local/bin/garage server', + StateDirectory: 'garage', + DynamicUser: true, + ProtectHome: true, + NoNewPrivileges: true + }, + Install: { + WantedBy: 'multi-user.target' + } + }) + verify false + triggers_reload true + action [:create] +end + +service 'garage' do + action [:enable, :start] +end diff --git a/site-cookbooks/kosmos_garage/recipes/firewall_apis.rb b/site-cookbooks/kosmos_garage/recipes/firewall_apis.rb new file mode 100644 index 0000000..3b169c3 --- /dev/null +++ b/site-cookbooks/kosmos_garage/recipes/firewall_apis.rb @@ -0,0 +1,34 @@ +include_recipe 'firewall' + +firewall_rule 'garage_s3_api' do + command :allow + protocol :tcp + source "10.1.1.0/24" + port node['garage']['s3_api_port'] +end + +firewall_rule 'garage_s3_web' do + command :allow + protocol :tcp + source "10.1.1.0/24" + port node['garage']['s3_web_port'] +end + +firewall_rule 'garage_admin' do + command :allow + protocol :tcp + source "10.1.1.0/24" + port node['garage']['admin_port'] +end + +# K2V is currently disabled by default in release +# builds, but may be interesting for RS usage: +# +# https://garagehq.deuxfleurs.fr/documentation/reference-manual/k2v/ +# +# firewall_rule 'garage_k2v_api' do +# command :allow +# protocol :tcp +# source "10.1.1.0/24" +# port node['garage']['k2v_api_port'] +# end diff --git a/site-cookbooks/kosmos_garage/recipes/firewall_rpc.rb b/site-cookbooks/kosmos_garage/recipes/firewall_rpc.rb new file mode 100644 index 0000000..4f45898 --- /dev/null +++ b/site-cookbooks/kosmos_garage/recipes/firewall_rpc.rb @@ -0,0 +1,8 @@ +include_recipe 'firewall' + +firewall_rule 'garage_rpc' do + command :allow + protocol :tcp + source "10.1.1.0/24" + port node['garage']['rpc_port'] +end diff --git a/site-cookbooks/kosmos_garage/templates/garage.toml.erb b/site-cookbooks/kosmos_garage/templates/garage.toml.erb new file mode 100644 index 0000000..0bf6125 --- /dev/null +++ b/site-cookbooks/kosmos_garage/templates/garage.toml.erb @@ -0,0 +1,26 @@ +metadata_dir = "<%= @metadata_dir %>" +data_dir = "<%= @data_dir %>" +db_engine = "<%= @db_engine %>" + +replication_mode = "<%= @replication_mode %>" + +rpc_bind_addr = "[::]:<%= @rpc_port %>" +rpc_public_addr = "<%= @rpc_public_addr %>" +rpc_secret = "<%= @rpc_secret %>" + +[s3_api] +s3_region = "<%= @s3_region %>" +api_bind_addr = "[::]:<%= @s3_api_port %>" +root_domain = "<%= @s3_api_root_domain %>" + +[s3_web] +bind_addr = "[::]:<%= @s3_web_port %>" +root_domain = "<%= @s3_web_root_domain %>" +index = "index.html" + +[k2v_api] +api_bind_addr = "[::]:<%= @k2v_api_port %>" + +[admin] +api_bind_addr = "0.0.0.0:<%= @admin_port %>" +admin_token = "<%= @admin_token %>" diff --git a/site-cookbooks/kosmos_garage/test/integration/default/data_bags/credentials/garage.json b/site-cookbooks/kosmos_garage/test/integration/default/data_bags/credentials/garage.json new file mode 100644 index 0000000..ad71d47 --- /dev/null +++ b/site-cookbooks/kosmos_garage/test/integration/default/data_bags/credentials/garage.json @@ -0,0 +1,17 @@ +{ + "id": "garage", + "admin_token": { + "encrypted_data": "BYRysR7CokS4943A3QL4/bN4dDdBs7TWgzbuTntB7cBIJqpnUbu2sd9PNjxo\nSjGTlwdnUUlbmCJzPfQ8oKCINrs+yilH3XIyzb4x//3h9rzE+qI=\n", + "iv": "rwOuaLi2kwg2Uw9g\n", + "auth_tag": "68j6nGYan1DiQQKmmpPW9A==\n", + "version": 3, + "cipher": "aes-256-gcm" + }, + "rpc_secret": { + "encrypted_data": "x9qfeSGGBkGsErJ1vQuQKTcNksPh3xnyM1V09CvecNewVHkmWeP03WE3gjJH\nzUWooHrDn2Gaci8Pi9VYHg6+Gsw/w/l6BhTWByd6k/251pNTRps=\n", + "iv": "/QiOCspNokU3QCDB\n", + "auth_tag": "MWkUnKEAEDduPLG0kWd8Bg==\n", + "version": 3, + "cipher": "aes-256-gcm" + } +} \ No newline at end of file diff --git a/site-cookbooks/kosmos_garage/test/integration/default/default_test.rb b/site-cookbooks/kosmos_garage/test/integration/default/default_test.rb new file mode 100644 index 0000000..aea8367 --- /dev/null +++ b/site-cookbooks/kosmos_garage/test/integration/default/default_test.rb @@ -0,0 +1,21 @@ +# InSpec test for recipe kosmos_garage::garagej + +# The Chef InSpec reference, with examples and extensive documentation, can be +# found at https://docs.chef.io/inspec/resources/ + +describe file('/usr/local/bin/garage') do + it { should exist } + its('mode') { should cmp '00755' } +end + +describe service('garage') do + it { should be_enabled } + it { should be_installed } + it { should be_running } +end + +[3900, 3901, 3902, 3903, 3904].each do |port_number| + describe port(port_number) do + it { should be_listening } + end +end diff --git a/site-cookbooks/kosmos_garage/test/integration/default/encrypted_data_bag_secret b/site-cookbooks/kosmos_garage/test/integration/default/encrypted_data_bag_secret new file mode 100644 index 0000000..4bee0e9 --- /dev/null +++ b/site-cookbooks/kosmos_garage/test/integration/default/encrypted_data_bag_secret @@ -0,0 +1 @@ +I7L6DW2FOYGfZIVrXWN6aLYTswhHnDR7ZmdlGp5KeAlpEIjaB7IgPO3EAV/QxgzMaHJzBwDzThpp8PTAUXUH+9M5I+G06Y/F21cCkpe5mNOjiP9SukdggGbiZdMRxS//Ym511hqrWSpowF2yYwX6V/rprlWAc0EKzZWNDceC2g/sKdX725/TXz4mhouimr8GFnlCqPoBPpM8OFPQy2TMoK66IXjA6zW190FhpySsAWE+Evnjj1FtVjTrQBW8A00NVU5SH8APn21RLZGY/uk/acX9sKsa5XcUSU1MlKU5DZejLzu1Vf0DKoDAQpI9/JkQajOxhCC77buXWP83GL0Ru9w9eWq2rYQuDawRKZViHC5aFatPZFu0GvRq58WcbordxDtH+TZYoYEIIf32kCe0sNxbO8iAgMHD7HV8n21zHVps7nTD4HRbXvURA+fwOkQSxcxET3pS5AJDbVIJ6FHbn3MZj9MCv2miwAVCYcZ5Wt+dhmQxjYgZ8PUUlVvB+eCauY2m6J4+nEjg+yx15yvdnBOmEGmj+HQG+rdD+6mAbtK2Adwq2zoqkQbUKJNmyZAvHEkCLHmWswdV8kN2Eh35wlyG/9FBQ7WAlsvpwT8cjHvxwsh644EsBJPlmoBXT0I5Q5S6Gx6PDP/x/swO2VNgISWOLimPvUD1bI4ObIiVTWo= \ No newline at end of file diff --git a/site-cookbooks/kosmos_gitea/attributes/default.rb b/site-cookbooks/kosmos_gitea/attributes/default.rb index 3ab874b..e25126d 100644 --- a/site-cookbooks/kosmos_gitea/attributes/default.rb +++ b/site-cookbooks/kosmos_gitea/attributes/default.rb @@ -1,12 +1,13 @@ -gitea_version = "1.17.2" -node.default["kosmos_gitea"]["version"] = gitea_version -node.default["kosmos_gitea"]["binary_url"] = "https://dl.gitea.io/gitea/#{gitea_version}/gitea-#{gitea_version}-linux-amd64" -node.default["kosmos_gitea"]["binary_checksum"] = "d0e903671ae04007c5956beb65985825795c1d9b24c9f354b48008fd44db1b57" -node.default["kosmos_gitea"]["nginx"]["domain"] = "gitea.kosmos.org" -node.default["kosmos_gitea"]["working_directory"] = "/var/lib/gitea" -node.default["kosmos_gitea"]["port"] = 3000 +gitea_version = "1.17.3" +node.default["gitea"]["version"] = gitea_version +node.default["gitea"]["binary_url"] = "https://dl.gitea.io/gitea/#{gitea_version}/gitea-#{gitea_version}-linux-amd64" +node.default["gitea"]["binary_checksum"] = "38c4e1228cd051b785c556bcadc378280d76c285b70e8761cd3f5051aed61b5e" +node.default["gitea"]["working_directory"] = "/var/lib/gitea" +node.default["gitea"]["port"] = 3000 +node.default["gitea"]["postgresql_host"] = "localhost:5432" +node.default["gitea"]["nginx"]["domain"] = "gitea.kosmos.org" -node.default["kosmos_gitea"]["config"] = { +node.default["gitea"]["config"] = { "webhook": { "allowed_host_list" => "external,127.0.1.1" } diff --git a/site-cookbooks/kosmos_gitea/recipes/backup.rb b/site-cookbooks/kosmos_gitea/recipes/backup.rb index f363577..e929ba3 100644 --- a/site-cookbooks/kosmos_gitea/recipes/backup.rb +++ b/site-cookbooks/kosmos_gitea/recipes/backup.rb @@ -7,6 +7,6 @@ unless node.chef_environment == "development" # backup the data dir and the config files - node.override["backup"]["archives"]["gitea"] = [node["kosmos_gitea"]["working_directory"]] + node.override["backup"]["archives"]["gitea"] = [node["gitea"]["working_directory"]] include_recipe "backup" end diff --git a/site-cookbooks/kosmos_gitea/recipes/default.rb b/site-cookbooks/kosmos_gitea/recipes/default.rb index 8035deb..8327ae1 100644 --- a/site-cookbooks/kosmos_gitea/recipes/default.rb +++ b/site-cookbooks/kosmos_gitea/recipes/default.rb @@ -5,7 +5,7 @@ include_recipe "kosmos-dirsrv::hostsfile" -working_directory = node["kosmos_gitea"]["working_directory"] +working_directory = node["gitea"]["working_directory"] git_home_directory = "/home/git" repository_root_directory = "#{git_home_directory}/gitea-repositories" config_directory = "/etc/gitea" @@ -62,15 +62,37 @@ directory config_directory do mode "0750" end -nginx_proxy_ip_addresses = [] -search(:node, "role:nginx_proxy").each do |node| - nginx_proxy_ip_addresses << node["knife_zero"]["host"] +if node.chef_environment == "production" + allowed_webhook_hosts = [] + search(:node, "role:nginx_proxy OR role:hubot").each do |node| + allowed_webhook_hosts << node["knife_zero"]["host"] + end + + node.normal["gitea"]["config"] = { + "webhook": { + "allowed_host_list" => "external,#{allowed_webhook_hosts.join(",")}" + } + } end -node.default["kosmos_gitea"]["config"] = { - "webhook": { - "allowed_host_list" => "external,#{nginx_proxy_ip_addresses.join(",")}" - } +config_variables = { + working_directory: working_directory, + git_home_directory: git_home_directory, + repository_root_directory: repository_root_directory, + config_directory: config_directory, + gitea_binary_path: gitea_binary_path, + jwt_secret: jwt_secret, + internal_token: internal_token, + secret_key: secret_key, + postgresql_host: node["gitea"]["postgresql_host"], + postgresql_password: gitea_data_bag_item["postgresql_password"], + smtp_host: smtp_credentials["relayhost"], + smtp_user: smtp_credentials["user_name"], + smtp_password: smtp_credentials["password"], + config: node["gitea"]["config"], + s3_key_id: gitea_data_bag_item["s3_key_id"], + s3_secret_key: gitea_data_bag_item["s3_secret_key"], + s3_bucket: gitea_data_bag_item["s3_bucket"] } template "#{config_directory}/app.ini" do @@ -79,26 +101,13 @@ template "#{config_directory}/app.ini" do group "git" mode "0600" sensitive true - variables working_directory: working_directory, - git_home_directory: git_home_directory, - repository_root_directory: repository_root_directory, - config_directory: config_directory, - gitea_binary_path: gitea_binary_path, - jwt_secret: jwt_secret, - internal_token: internal_token, - secret_key: secret_key, - postgresql_host: "pg.kosmos.local:5432", - postgresql_password: gitea_data_bag_item["postgresql_password"], - smtp_host: smtp_credentials["relayhost"], - smtp_user: smtp_credentials["user_name"], - smtp_password: smtp_credentials["password"], - config: node["kosmos_gitea"]["config"] + variables config_variables notifies :restart, "service[gitea]", :delayed end remote_file gitea_binary_path do - source node['kosmos_gitea']['binary_url'] - checksum node['kosmos_gitea']['binary_checksum'] + source node['gitea']['binary_url'] + checksum node['gitea']['binary_checksum'] mode "0755" notifies :restart, "service[gitea]", :delayed end @@ -121,7 +130,7 @@ service "gitea" do end firewall_rule 'gitea' do - port [node["kosmos_gitea"]["port"]] + port [node["gitea"]["port"]] source "10.1.1.0/24" # TODO only allow nginx proxy IPs protocol :tcp command :allow diff --git a/site-cookbooks/kosmos_gitea/recipes/nginx.rb b/site-cookbooks/kosmos_gitea/recipes/nginx.rb index 20bd979..e145676 100644 --- a/site-cookbooks/kosmos_gitea/recipes/nginx.rb +++ b/site-cookbooks/kosmos_gitea/recipes/nginx.rb @@ -5,7 +5,7 @@ include_recipe "kosmos-nginx" -domain = node["kosmos_gitea"]["nginx"]["domain"] +domain = node["gitea"]["nginx"]["domain"] # upstream_ip_addresses = [] # search(:node, "role:gitea").each do |n| @@ -28,7 +28,7 @@ template "#{node['nginx']['dir']}/sites-available/#{domain}" do ssl_cert: "/etc/letsencrypt/live/#{domain}/fullchain.pem", ssl_key: "/etc/letsencrypt/live/#{domain}/privkey.pem", upstream_host: upstream_ip_address, - upstream_port: node["kosmos_gitea"]["port"] + upstream_port: node["gitea"]["port"] notifies :reload, 'service[nginx]', :delayed end diff --git a/site-cookbooks/kosmos_gitea/templates/default/app.ini.erb b/site-cookbooks/kosmos_gitea/templates/default/app.ini.erb index 788c45e..a5863a3 100644 --- a/site-cookbooks/kosmos_gitea/templates/default/app.ini.erb +++ b/site-cookbooks/kosmos_gitea/templates/default/app.ini.erb @@ -92,3 +92,16 @@ SCHEDULE = @every 15m [webhook] <% if c["allowed_host_list"] %>ALLOWED_HOST_LIST = <%= c["allowed_host_list"] %><% end %> <% end %> + +<% if c = @config["storage"] %> +[storage] +<% if c["type"] == "minio" %> +STORAGE_TYPE=minio +MINIO_ENDPOINT=<%= c["endpoint"] %> +MINIO_ACCESS_KEY_ID=<%= @s3_key_id %> +MINIO_SECRET_ACCESS_KEY=<%= @s3_secret_key %> +MINIO_BUCKET=<%= @s3_bucket %> +MINIO_LOCATION=<%= c["location"] %> +MINIO_USE_SSL=<%= c["use_ssl"] %> +<% end %> +<% end %>