Update the elasticsearch cookbook

cookbooks/elasticsearch/resources/configure.rb

@@ -0,0 +1,230 @@
+unified_mode true
+# this is what helps the various resources find each other
+property :instance_name, String
+
+# If you override one of these, you should probably override them all
+property :path_home,    String, default: '/usr/share/elasticsearch'
+property :path_conf,    String, default: '/etc/elasticsearch'
+property :path_data,    [String, Array], default: '/var/lib/elasticsearch'
+property :path_logs,    String, default: '/var/log/elasticsearch'
+property :path_pid,     String, default: '/var/run/elasticsearch'
+property :path_plugins, String, default: '/usr/share/elasticsearch/plugins'
+property :path_bin,     String, default: '/usr/share/elasticsearch/bin'
+
+property :template_elasticsearch_env, String, default: 'elasticsearch.in.sh.erb'
+property :cookbook_elasticsearch_env, String, default: 'elasticsearch'
+
+property :template_jvm_options, String, default: 'jvm_options.erb'
+property :cookbook_jvm_options, String, default: 'elasticsearch'
+
+property :template_elasticsearch_yml, String, default: 'elasticsearch.yml.erb'
+property :cookbook_elasticsearch_yml, String, default: 'elasticsearch'
+
+property :template_log4j2_properties, String, default: 'log4j2.properties.erb'
+property :cookbook_log4j2_properties, String, default: 'elasticsearch'
+
+property :logging, Hash, default: {}.freeze
+property :java_home, String
+
+# other settings in /etc/default or /etc/sysconfig
+property :memlock_limit, String, default: 'unlimited'
+property :max_map_count, String, default: '262144'
+property :nofile_limit, String, default: '65535'
+property :startup_sleep_seconds, [String, Integer], default: 5
+property :restart_on_upgrade, [true, false], default: false
+
+# Calculations for this are done in the provider, as we can't do them in the
+# resource definition. default is 50% of RAM or 31GB, which ever is smaller.
+property :allocated_memory, String
+
+property :jvm_options, Array, default:
+  %w(
+    8-13:-XX:+UseConcMarkSweepGC
+    8-13:-XX:CMSInitiatingOccupancyFraction=75
+    8-13:-XX:+UseCMSInitiatingOccupancyOnly
+    14-:-XX:+UseG1GC
+    -Djava.io.tmpdir=${ES_TMPDIR}
+    -XX:+HeapDumpOnOutOfMemoryError
+    9-:-XX:+ExitOnOutOfMemoryError
+    -XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
+    8:-XX:+PrintGCDetails
+    8:-XX:+PrintGCDateStamps
+    8:-XX:+PrintTenuringDistribution
+    8:-XX:+PrintGCApplicationStoppedTime
+    8:-Xloggc:/var/log/elasticsearch/gc.log
+    8:-XX:+UseGCLogFileRotation
+    8:-XX:NumberOfGCLogFiles=32
+    8:-XX:GCLogFileSize=64m
+    9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m
+  ).freeze
+
+# These are the default settings. Most of the time, you want to override
+# the `configuration` attribute below. If you do override the defaults, you
+# must supply ALL needed defaults, and don't use nil as a value in the hash.
+property :default_configuration, Hash, default: {
+  'cluster.name' => 'elasticsearch',
+  'node.name' => Chef::Config[:node_name],
+}
+
+# These settings are merged with the `default_configuration` attribute,
+# allowing you to override and set specific settings. Unless you intend to
+# wipe out all default settings, your configuration items should go here.
+#
+property :configuration, Hash, default: {}
+
+include ElasticsearchCookbook::Helpers
+
+action :manage do
+  # lookup existing ES resources
+  es_user = find_es_resource(Chef.run_context, :elasticsearch_user, new_resource)
+  es_svc = find_es_resource(Chef.run_context, :elasticsearch_service, new_resource)
+  es_install = find_es_resource(Chef.run_context, :elasticsearch_install, new_resource)
+
+  default_configuration = new_resource.default_configuration.dup
+  # if a subdir parameter is missing but dir is set, infer the subdir name
+  # then go and be sure it's also set in the YML hash if it wasn't given there
+  if new_resource.path_data && default_configuration['path.data'].nil?
+    default_configuration['path.data'] = new_resource.path_data
+  end
+
+  if new_resource.path_logs && default_configuration['path.logs'].nil?
+    default_configuration['path.logs'] = new_resource.path_logs
+  end
+
+  # Calculation for memory allocation; 50% or 31g, whatever is smaller
+  #
+  unless new_resource.allocated_memory
+    half = ((node['memory']['total'].to_i * 0.5).floor / 1024)
+    malloc_str = (half > 30_500 ? '30500m' : "#{half}m")
+    new_resource.allocated_memory malloc_str
+  end
+
+  # Create ES directories
+  #
+  [new_resource.path_conf, "#{new_resource.path_conf}/scripts"].each do |path|
+    directory path do
+      owner es_user.username
+      group es_user.groupname
+      mode '0750'
+      recursive true
+      action :create
+    end
+  end
+
+  directory new_resource.path_logs do
+    owner es_user.username
+    group es_user.groupname
+    mode '0750'
+    recursive true
+    action :create
+  end
+
+  if new_resource.path_data.is_a?(String)
+    directory new_resource.path_data do
+      owner es_user.username
+      group es_user.groupname
+      mode '0750'
+      recursive true
+      action :create
+    end
+  else
+    new_resource.path_data.each do |path|
+      directory path.strip do
+        owner es_user.username
+        group es_user.groupname
+        mode '0750'
+        recursive true
+        action :create
+      end
+    end
+  end
+
+  # Create elasticsearch shell variables file
+  #
+  # Valid values in /etc/sysconfig/elasticsearch or /etc/default/elasticsearch
+  # ES_HOME JAVA_HOME ES_PATH_CONF DATA_DIR LOG_DIR PID_DIR ES_JAVA_OPTS
+  # RESTART_ON_UPGRADE ES_USER ES_GROUP ES_STARTUP_SLEEP_TIME MAX_OPEN_FILES
+  # MAX_LOCKED_MEMORY MAX_MAP_COUNT
+  #
+  # We provide these values as resource attributes/parameters directly
+  params = {}
+  params[:ES_HOME] = new_resource.path_home
+  params[:JAVA_HOME] = new_resource.java_home
+  params[:ES_PATH_CONF] = new_resource.path_conf
+  params[:DATA_DIR] = new_resource.path_data
+  params[:LOG_DIR] = new_resource.path_logs
+  params[:PID_DIR] = new_resource.path_pid
+  params[:RESTART_ON_UPGRADE] = new_resource.restart_on_upgrade
+  params[:ES_USER] = es_user.username if es_install.type == 'tarball'
+  params[:ES_GROUP] = es_user.groupname if es_install.type == 'tarball'
+  params[:ES_STARTUP_SLEEP_TIME] = new_resource.startup_sleep_seconds.to_s
+  params[:MAX_OPEN_FILES] = new_resource.nofile_limit
+  params[:MAX_LOCKED_MEMORY] = new_resource.memlock_limit
+  params[:MAX_MAP_COUNT] = new_resource.max_map_count
+
+  default_config_name = es_svc.service_name || es_svc.instance_name || new_resource.instance_name || 'elasticsearch'
+
+  with_run_context :root do
+    template "elasticsearch.in.sh-#{default_config_name}" do
+      path platform_family?('rhel', 'amazon') ? "/etc/sysconfig/#{default_config_name}" : "/etc/default/#{default_config_name}"
+      source new_resource.template_elasticsearch_env
+      cookbook new_resource.cookbook_elasticsearch_env
+      mode '0644'
+      variables(params: params)
+      action :create
+    end
+  end
+
+  template "jvm_options-#{default_config_name}" do
+    path   "#{new_resource.path_conf}/jvm.options"
+    source new_resource.template_jvm_options
+    cookbook new_resource.cookbook_jvm_options
+    owner es_user.username
+    group es_user.groupname
+    mode '0644'
+    variables(jvm_options: [
+      "-Xms#{new_resource.allocated_memory}",
+      "-Xmx#{new_resource.allocated_memory}",
+      new_resource.jvm_options,
+    ].flatten.join("\n"))
+    action :create
+  end
+
+  template "log4j2_properties-#{default_config_name}" do
+    path   "#{new_resource.path_conf}/log4j2.properties"
+    source new_resource.template_log4j2_properties
+    cookbook new_resource.cookbook_log4j2_properties
+    owner es_user.username
+    group es_user.groupname
+    mode '0640'
+    variables(logging: new_resource.logging)
+    action :create
+  end
+
+  # Create ES elasticsearch.yml file
+  #
+  merged_configuration = default_configuration.merge(new_resource.configuration.dup)
+
+  # Warn if someone is using symbols. We don't support.
+  found_symbols = merged_configuration.keys.select { |s| s.is_a?(Symbol) }
+  unless found_symbols.empty?
+    Chef::Log.warn("Please change the following to strings in order to work with this Elasticsearch cookbook: #{found_symbols.join(',')}")
+  end
+
+  # workaround for https://github.com/sous-chefs/elasticsearch/issues/590
+  config_vars = ElasticsearchCookbook::HashAndMashBlender.new(merged_configuration).to_hash
+
+  with_run_context :root do
+    template "elasticsearch.yml-#{default_config_name}" do
+      path "#{new_resource.path_conf}/elasticsearch.yml"
+      source new_resource.template_elasticsearch_yml
+      cookbook new_resource.cookbook_elasticsearch_yml
+      owner es_user.username
+      group es_user.groupname
+      mode '0640'
+      helpers(ElasticsearchCookbook::Helpers)
+      variables(config: config_vars)
+      action :create
+    end
+  end
+end

cookbooks/elasticsearch/resources/install.rb

@@ -0,0 +1,47 @@
+unified_mode true
+use 'partial/_common'
+use 'partial/_package'
+use 'partial/_repository'
+
+property :type,
+        String,
+        equal_to: %w(package tarball repository),
+        default: 'repository'
+
+action :install do
+  case new_resource.type
+  when 'tarball'
+    raise 'Tarball method is not currently supported, due to no supporting systemd service'
+  when 'package'
+    elasticsearch_install_package "ElasticSearch #{new_resource.version}" do
+      version new_resource.version
+      instance_name new_resource.instance_name
+      download_url download_url
+      download_checksum download_checksum
+    end
+  when 'repository'
+    elasticsearch_install_repository "ElasticSearch #{new_resource.version}" do
+      version new_resource.version
+      instance_name new_resource.instance_name
+      enable_repository_actions new_resource.enable_repository_actions
+      package_options new_resource.package_options
+    end
+  else
+    raise "#{new_resource.type} is not a valid install type"
+  end
+end
+
+action :remove do
+  case new_resource.type
+  when 'package'
+    elasticsearch_install_package "ElasticSearch #{new_resource.version}" do
+      action :remove
+    end
+  when 'repository'
+    elasticsearch_install_repository "ElasticSearch #{new_resource.version}" do
+      action :remove
+    end
+  else
+    raise "#{install_type} is not a valid install type"
+  end
+end

cookbooks/elasticsearch/resources/install_package.rb

@@ -0,0 +1,41 @@
+include ElasticsearchCookbook::Helpers
+unified_mode true
+use 'partial/_common'
+use 'partial/_package'
+
+action :install do
+  remote_file "#{Chef::Config[:file_cache_path]}/#{filename_from_url}" do
+    source new_resource.download_url
+    checksum new_resource.download_checksum
+    mode '0644'
+    action :create
+  end
+
+  if platform_family?('debian')
+    dpkg_package filename_from_url do
+      options new_resource.package_options
+      source "#{Chef::Config[:file_cache_path]}/#{filename_from_url}"
+      action :install
+    end
+  else
+    package filename_from_url do
+      options new_resource.package_options
+      source "#{Chef::Config[:file_cache_path]}/#{filename_from_url}"
+      action :install
+    end
+  end
+end
+
+action :remove do
+  package "#{Chef::Config[:file_cache_path]}/#{filename_from_url}" do
+    action :temove
+  end
+end
+
+action_class do
+  include ElasticsearchCookbook::Helpers
+
+  def filename_from_url
+    new_resource.download_url.split('/').last
+  end
+end

cookbooks/elasticsearch/resources/install_repository.rb

@@ -0,0 +1,63 @@
+unified_mode true
+use 'partial/_common'
+use 'partial/_repository'
+
+include ElasticsearchCookbook::Helpers
+
+action :install do
+  major_version = new_resource.version.split('.')[0]
+
+  es_user = find_es_resource(Chef.run_context, :elasticsearch_user, new_resource)
+
+  unless es_user && es_user.username == 'elasticsearch' && es_user.groupname == 'elasticsearch'
+    raise 'Custom usernames/group names is not supported in Elasticsearch 6+ repository installation'
+  end
+
+  if new_resource.enable_repository_actions
+    if platform_family?('debian')
+      apt_repository "elastic-#{major_version}.x" do
+        uri 'https://artifacts.elastic.co/packages/7.x/apt'
+        key 'elasticsearch.asc'
+        cookbook 'elasticsearch'
+        components ['main']
+        distribution 'stable'
+      end
+    else
+      yum_repository "elastic-#{major_version}.x" do
+        baseurl "https://artifacts.elastic.co/packages/#{major_version}.x/yum"
+        gpgkey 'https://artifacts.elastic.co/GPG-KEY-elasticsearch'
+        action :create
+      end
+    end
+  end
+
+  package 'elasticsearch' do
+    options new_resource.package_options
+    version new_resource.version
+    action :install
+  end
+end
+
+action :remove do
+  if new_resource.enable_repository_actions
+    if platform_family?('debian')
+      apt_repository "elastic-#{new_resource.version}.x" do
+        action :remove
+      end
+    else
+      yum_repository "elastic-#{new_resource.version}.x" do
+        action :remove
+      end
+    end
+  end
+
+  package 'elasticsearch' do
+    options new_resource.package_options
+    version new_resource.version
+    action :remove
+  end
+end
+
+action_class do
+  include ElasticsearchCookbook::Helpers
+end

cookbooks/elasticsearch/resources/partial/_common.rb

@@ -0,0 +1,12 @@
+include ElasticsearchCookbook::Helpers
+include ElasticsearchCookbook::VersionHelpers
+
+property :instance_name,
+          String
+
+property :version,
+        String,
+        default: '7.17.9'
+
+property :package_options,
+        String

cookbooks/elasticsearch/resources/partial/_package.rb

@@ -0,0 +1,7 @@
+property :download_url,
+        String,
+        default: lazy { default_download_url(new_resource.version) }
+
+property :download_checksum,
+        String,
+        default: lazy { default_download_checksum(new_resource.version)[checksum_platform] }

cookbooks/elasticsearch/resources/partial/_repository.rb

@@ -0,0 +1,3 @@
+property :enable_repository_actions,
+          [true, false],
+          default: true

cookbooks/elasticsearch/resources/plugin.rb

@@ -0,0 +1,84 @@
+unified_mode true
+
+include ElasticsearchCookbook::Helpers
+
+property :plugin_name,
+         String,
+         name_property: true
+
+property :url,
+         String
+
+property :options,
+         String,
+         default: ''
+
+# this is what helps the various resources find each other
+property :instance_name,
+        String
+
+action :install do
+  execute "Install plugin #{new_resource.plugin_name}" do
+    command "#{es_conf.path_bin}/elasticsearch-plugin install #{new_resource.options} #{config[:plugin_name]}".chomp(' ')
+    not_if { plugin_exists? }
+    environment env
+    user config[:user] unless config[:install_type] == 'package' || config[:install_type] == 'repository'
+    group config[:group] unless config[:install_type] == 'package' || config[:install_type] == 'repository'
+  end
+end
+
+action :remove do
+  execute "Remove plugin #{new_resource.plugin_name}" do
+    command "#{es_conf.path_bin}/elasticsearch-plugin remove #{new_resource.options} #{config[:plugin_name]}".chomp(' ')
+    only_if { plugin_exists? }
+    environment env
+    user config[:user] unless config[:install_type] == 'package' || config[:install_type] == 'repository'
+    group config[:group] unless config[:install_type] == 'package' || config[:install_type] == 'repository'
+  end
+end
+
+action_class do
+  def es_user
+    find_es_resource(Chef.run_context, :elasticsearch_user, new_resource)
+  end
+
+  def es_install
+    find_es_resource(Chef.run_context, :elasticsearch_install, new_resource)
+  end
+
+  def es_conf
+    find_es_resource(Chef.run_context, :elasticsearch_configure, new_resource)
+  end
+
+  def env
+    include_file_resource = find_exact_resource(Chef.run_context, :template, "elasticsearch.in.sh-#{config[:name]}")
+    { 'ES_INCLUDE' => include_file_resource.path }
+  end
+
+  def config
+    {
+      name: new_resource.instance_name || es_conf.instance_name || 'elasticsearch',
+      plugin_name: new_resource.url || new_resource.plugin_name,
+      install_type: es_install.type,
+      user: es_user.username,
+      group: es_user.groupname,
+      path_conf: es_conf.path_conf,
+      path_plugins: es_conf.path_plugins,
+      path_bin: es_conf.path_bin,
+    }
+  end
+
+  def plugin_exists?
+    # This is quicker than shelling out to the plugin list command
+    # The plugin install command checks for the existsance of the plugin directory anyway
+    es_conf = find_es_resource(Chef.run_context, :elasticsearch_configure, new_resource)
+    path = es_conf.path_plugins
+
+    Dir.entries(path).any? do |plugin|
+      next if plugin =~ /^\./
+      config[:plugin_name] == plugin
+    end
+  rescue
+    false
+  end
+end

cookbooks/elasticsearch/resources/service.rb

@@ -0,0 +1,126 @@
+unified_mode true
+
+include ElasticsearchCookbook::Helpers
+
+# this is what helps the various resources find each other
+property :instance_name,
+        String
+
+property :service_name,
+        String,
+        name_property: true
+
+property :service_actions,
+        [Symbol, String, Array],
+        default: [:enable, :start]
+
+action :configure do
+  es_user = find_es_resource(Chef.run_context, :elasticsearch_user, new_resource)
+  es_conf = find_es_resource(Chef.run_context, :elasticsearch_configure, new_resource)
+  default_config_name = new_resource.service_name || new_resource.instance_name || es_conf.instance_name || 'elasticsearch'
+
+  directory "#{es_conf.path_pid}-#{default_config_name}" do
+    path es_conf.path_pid
+    owner es_user.username
+    group es_user.groupname
+    mode '0755'
+    recursive true
+    action :create
+  end
+
+  default_conf_dir = platform_family?('rhel', 'amazon') ? '/etc/sysconfig' : '/etc/default'
+
+  systemd_unit new_resource.service_name do
+    content(
+      Unit: {
+        Description: 'Elasticsearch',
+        Documentation: 'https://www.elastic.co',
+        Wants: 'network-online.target',
+        After: 'network-online.target',
+      },
+      Service: {
+        Type: 'notify',
+        RuntimeDirectory: 'elasticsearch',
+        PrivateTmp: 'true',
+        Environment: [
+          "ES_HOME=#{es_conf.path_home}",
+          'ES_PATH_CONF=/etc/elasticsearch',
+          "PID_DIR=#{es_conf.path_pid}",
+          'ES_SD_NOTIFY=true',
+        ],
+        EnvironmentFile: "-#{default_conf_dir}/#{new_resource.service_name}",
+        WorkingDirectory: "#{es_conf.path_home}",
+        User: es_user.username,
+        Group: es_user.groupname,
+        ExecStart: "#{es_conf.path_home}/bin/systemd-entrypoint -p ${PID_DIR}/elasticsearch.pid --quiet",
+        StandardOutput: 'journal',
+        StandardError: 'inherit',
+        LimitNOFILE: '65535',
+        LimitNPROC: '4096',
+        LimitAS: 'infinity',
+        LimitFSIZE: 'infinity',
+        TimeoutStopSec: '0',
+        KillSignal: 'SIGTERM',
+        KillMode: 'process',
+        SendSIGKILL: 'no',
+        SuccessExitStatus: '143',
+        TimeoutStartSec: '900',
+      },
+      Install: {
+        WantedBy: 'multi-user.target',
+      }
+    )
+    verify false
+    action :create
+    unit_name "#{new_resource.service_name}.service"
+  end
+
+  # flatten in an array here, in case the service_actions are a symbol vs. array
+  [new_resource.service_actions].flatten.each do |act|
+    passthrough_action(act)
+  end
+end
+
+# Passthrough actions to service[service_name]
+#
+action :enable do
+  passthrough_action(:enable)
+end
+
+action :disable do
+  passthrough_action(:disable)
+end
+
+action :start do
+  passthrough_action(:start)
+end
+
+action :stop do
+  passthrough_action(:stop)
+end
+
+action :restart do
+  passthrough_action(:restart)
+end
+
+action :status do
+  passthrough_action(:status)
+end
+
+action_class do
+  def passthrough_action(action)
+    svc_r = lookup_service_resource
+    svc_r.run_action(action)
+    new_resource.updated_by_last_action(true) if svc_r.updated_by_last_action?
+  end
+
+  def lookup_service_resource
+    rc = Chef.run_context.resource_collection
+    rc.find("service[#{new_resource.service_name}]")
+  rescue
+    service new_resource.service_name do
+      supports status: true, restart: true
+      action :nothing
+    end
+  end
+end

cookbooks/elasticsearch/resources/user.rb

@@ -0,0 +1,73 @@
+include ElasticsearchCookbook::Helpers
+
+unified_mode true
+
+property :instance_name,
+        String
+
+property :username,
+        String,
+        name_property: true
+
+property :groupname,
+        String,
+        default: lazy { username }
+
+property :shell,
+        String,
+        default: '/bin/bash'
+
+property :uid,
+        Integer
+
+property :comment,
+        String,
+        default: 'Elasticsearch User'
+
+property :gid,
+        Integer
+
+action :create do
+  group_r = group new_resource.groupname do
+    gid new_resource.gid
+    action :nothing
+    system true
+  end
+
+  group_r.run_action(:create)
+
+  new_resource.updated_by_last_action(true) if group_r.updated_by_last_action?
+
+  user_r = user new_resource.username do
+    comment new_resource.comment
+    shell   new_resource.shell
+    uid     new_resource.uid
+    gid     new_resource.groupname
+
+    manage_home false
+    action :nothing
+    system true
+  end
+
+  user_r.run_action(:create)
+
+  new_resource.updated_by_last_action(true) if user_r.updated_by_last_action?
+end
+
+action :remove do
+  user_r = user new_resource.username do
+    action :nothing
+  end
+
+  user_r.run_action(:remove)
+
+  new_resource.updated_by_last_action(true) if user_r.updated_by_last_action?
+
+  group_r = group new_resource.groupname do
+    action :nothing
+  end
+
+  group_r.run_action(:remove)
+
+  new_resource.updated_by_last_action(true) if group_r.updated_by_last_action?
+end