Vendor the external cookbooks

Knife-Zero doesn't include Berkshelf support, so vendoring everything in
the repo is convenient again

cookbooks/elasticsearch/libraries/helpers.rb

@@ -0,0 +1,156 @@
+module ElasticsearchCookbook
+  # Helper methods included by various providers and passed to the template engine
+  module Helpers
+    def find_es_resource(run_context, resource_type, resource)
+      resource_name = resource.name
+      instance_name = resource.instance_name
+
+      # if we are truly given a specific name to find
+      name_match = find_exact_resource(run_context, resource_type, resource_name) rescue nil
+      return name_match if name_match
+
+      # first try by instance name attribute
+      name_instance = find_instance_name_resource(run_context, resource_type, instance_name) rescue nil
+      return name_instance if name_instance
+
+      # otherwise try the defaults
+      name_default = find_exact_resource(run_context, resource_type, 'default') rescue nil
+      name_elasticsearch = find_exact_resource(run_context, resource_type, 'elasticsearch') rescue nil
+
+      # if we found exactly one default name that matched
+      return name_default if name_default && !name_elasticsearch
+      return name_elasticsearch if name_elasticsearch && !name_default
+
+      raise "Could not find exactly one #{resource_type} resource, and no specific resource or instance name was given"
+    end
+
+    # find exactly the resource name and type, but raise if there's multiple matches
+    # see https://github.com/chef/chef/blob/master/lib/chef/resource_collection/resource_set.rb#L80
+    def find_exact_resource(run_context, resource_type, resource_name)
+      rc = run_context.resource_collection
+      result = rc.find(resource_type => resource_name)
+
+      if result && result.is_a?(Array)
+        str = ''
+        str << "more than one #{resource_type} was found, "
+        str << 'you must specify a precise resource name'
+        raise str
+      end
+
+      result
+    end
+
+    def find_instance_name_resource(run_context, resource_type, instance_name)
+      results = []
+      rc = run_context.resource_collection
+
+      rc.each do |r|
+        next unless r.resource_name == resource_type && r.instance_name == instance_name
+        results << r
+      end
+
+      if !results.empty? && results.length > 1
+        str = ''
+        str << "more than one #{resource_type} was found, "
+        str << 'you must specify a precise instance name'
+        raise str
+      elsif !results.empty?
+        return results.first
+      end
+
+      nil # falsey
+    end
+
+    def determine_download_url(new_resource, node)
+      platform_family = node['platform_family']
+
+      version_key = 'download_urls'
+      if Gem::Version.new(new_resource.version) >= Gem::Version.new('7.0.0')
+        version_key = 'download_urls_v7'
+      end
+
+      url_string = nil
+      if new_resource.download_url
+        url_string = new_resource.download_url
+      elsif new_resource.type == 'tarball'
+        url_string = node['elasticsearch'][version_key]['tarball']
+      elsif new_resource.type == 'package' && node['elasticsearch']['download_urls'][platform_family]
+        url_string = node['elasticsearch'][version_key][platform_family]
+      end
+
+      if url_string && new_resource.version
+        return format(url_string, new_resource.version)
+      elsif url_string
+        return url_string
+      end
+    end
+
+    def determine_download_checksum(new_resource, node)
+      platform_family = node['platform_family']
+
+      # for the sake of finding correct attribute data, use rhel for amazon too
+      # See https://github.com/elastic/cookbook-elasticsearch/issues/609
+      platform_family = 'rhel' if platform_family == 'amazon'
+
+      install_type = new_resource.type
+      version = new_resource.version
+
+      if new_resource.download_checksum
+        new_resource.download_checksum
+      elsif install_type == 'tarball'
+        node && version &&
+          node['elasticsearch'] &&
+          node['elasticsearch']['checksums'] &&
+          node['elasticsearch']['checksums'][version] &&
+          node['elasticsearch']['checksums'][version]['tarball']
+      elsif install_type == 'package' && node['elasticsearch']['checksums'][version] && node['elasticsearch']['checksums'][version][platform_family]
+        node && version && platform_family &&
+          node['elasticsearch'] &&
+          node['elasticsearch']['checksums'] &&
+          node['elasticsearch']['checksums'][version] &&
+          node['elasticsearch']['checksums'][version][platform_family]
+      end
+    end
+
+    # proxy helper for chef sets JVM 8 proxy options
+    def get_java_proxy_arguments(enabled = true)
+      return '' unless enabled
+      require 'uri'
+      output = ''
+
+      if Chef::Config[:http_proxy] && !Chef::Config[:http_proxy].empty?
+        parsed_uri = URI(Chef::Config[:http_proxy])
+        output += "-Dhttp.proxyHost=#{parsed_uri.host} -Dhttp.proxyPort=#{parsed_uri.port} "
+      end
+
+      if Chef::Config[:https_proxy] && !Chef::Config[:https_proxy].empty?
+        parsed_uri = URI(Chef::Config[:https_proxy])
+        output += "-Dhttps.proxyHost=#{parsed_uri.host} -Dhttps.proxyPort=#{parsed_uri.port} "
+      end
+
+      output
+    rescue
+      ''
+    end
+  end
+
+  class HashAndMashBlender
+    attr_accessor :target
+    def initialize(hash_or_mash_or_whatever)
+      self.target = hash_or_mash_or_whatever
+    end
+
+    def to_hash
+      target.each_with_object({}) do |(k, v), hsh|
+        hsh[k] =
+          if v.respond_to?(:to_hash)
+            self.class.new(v).to_hash
+          elsif v.respond_to?(:to_a)
+            v.to_a
+          else
+            v
+          end
+      end
+    end
+  end
+end

cookbooks/elasticsearch/libraries/matchers.rb

@@ -0,0 +1,73 @@
+# ChefSpec is a tool to unit test cookbooks in conjunction with rspec
+# Learn more on the README or at https://github.com/sethvargo/chefspec.
+if defined?(ChefSpec)
+  ChefSpec.define_matcher(:elasticsearch_configure)
+  ChefSpec.define_matcher(:elasticsearch_install)
+  ChefSpec.define_matcher(:elasticsearch_plugin)
+  ChefSpec.define_matcher(:elasticsearch_service)
+  ChefSpec.define_matcher(:elasticsearch_user)
+
+  def create_elasticsearch_user(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_user, :create, resource_name)
+  end
+
+  def remove_elasticsearch_user(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_user, :remove, resource_name)
+  end
+
+  def install_elasticsearch(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_install, :install, resource_name)
+  end
+
+  def remove_elasticsearch(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_install, :remove, resource_name)
+  end
+
+  def manage_elasticsearch_configure(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_configure, :manage, resource_name)
+  end
+
+  def remove_elasticsearch_configure(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_configure, :remove, resource_name)
+  end
+
+  def configure_elasticsearch_service(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_service, :configure, resource_name)
+  end
+
+  def remove_elasticsearch_service(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_service, :remove, resource_name)
+  end
+
+  def enable_elasticsearch_service(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_service, :enable, resource_name)
+  end
+
+  def disable_elasticsearch_service(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_service, :disable, resource_name)
+  end
+
+  def start_elasticsearch_service(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_service, :start, resource_name)
+  end
+
+  def stop_elasticsearch_service(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_service, :stop, resource_name)
+  end
+
+  def restart_elasticsearch_service(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_service, :restart, resource_name)
+  end
+
+  def status_elasticsearch_service(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_service, :status, resource_name)
+  end
+
+  def install_elasticsearch_plugin(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_plugin, :install, resource_name)
+  end
+
+  def remove_elasticsearch_plugin(resource_name)
+    ChefSpec::Matchers::ResourceMatcher.new(:elasticsearch_plugin, :remove, resource_name)
+  end
+end

cookbooks/elasticsearch/libraries/provider_configure.rb

@@ -0,0 +1,164 @@
+# Chef Provider for configuring an elasticsearch instance
+class ElasticsearchCookbook::ConfigureProvider < Chef::Provider::LWRPBase
+  include ElasticsearchCookbook::Helpers
+
+  provides :elasticsearch_configure
+
+  def whyrun_supported?
+    true # we only use core Chef resources that also support whyrun
+  end
+
+  def action_manage
+    # lookup existing ES resources
+    es_user = find_es_resource(Chef.run_context, :elasticsearch_user, new_resource)
+    es_svc = find_es_resource(Chef.run_context, :elasticsearch_service, new_resource)
+    es_install = find_es_resource(Chef.run_context, :elasticsearch_install, new_resource)
+
+    default_configuration = new_resource.default_configuration.dup
+    # if a subdir parameter is missing but dir is set, infer the subdir name
+    # then go and be sure it's also set in the YML hash if it wasn't given there
+    if new_resource.path_data && default_configuration['path.data'].nil?
+      default_configuration['path.data'] = new_resource.path_data
+    end
+
+    if new_resource.path_logs && default_configuration['path.logs'].nil?
+      default_configuration['path.logs'] = new_resource.path_logs
+    end
+
+    # calculation for memory allocation; 50% or 31g, whatever is smaller
+    unless new_resource.allocated_memory
+      half = ((node['memory']['total'].to_i * 0.5).floor / 1024)
+      malloc_str = (half > 30_500 ? '30500m' : "#{half}m")
+      new_resource.allocated_memory malloc_str
+    end
+
+    # Create ES directories
+    #
+    [new_resource.path_conf, "#{new_resource.path_conf}/scripts"].each do |path|
+      d = directory path do
+        owner es_user.username
+        group es_user.groupname
+        mode '0750'
+        recursive true
+        action :nothing
+      end
+      d.run_action(:create)
+      new_resource.updated_by_last_action(true) if d.updated_by_last_action?
+    end
+
+    # Create data path directories
+    #
+    data_paths = new_resource.path_data.is_a?(Array) ? new_resource.path_data : new_resource.path_data.split(',')
+    data_paths = data_paths << new_resource.path_logs
+
+    data_paths.each do |path|
+      d = directory path.strip do
+        owner es_user.username
+        group es_user.groupname
+        mode '0755'
+        recursive true
+        action :nothing
+      end
+      d.run_action(:create)
+      new_resource.updated_by_last_action(true) if d.updated_by_last_action?
+    end
+
+    # Create elasticsearch shell variables file
+    #
+    # Valid values in /etc/sysconfig/elasticsearch or /etc/default/elasticsearch
+    # ES_HOME JAVA_HOME ES_PATH_CONF DATA_DIR LOG_DIR PID_DIR ES_JAVA_OPTS
+    # RESTART_ON_UPGRADE ES_USER ES_GROUP ES_STARTUP_SLEEP_TIME MAX_OPEN_FILES
+    # MAX_LOCKED_MEMORY MAX_MAP_COUNT
+    #
+    # We provide these values as resource attributes/parameters directly
+
+    params = {}
+    params[:ES_HOME] = new_resource.path_home
+    params[:JAVA_HOME] = new_resource.java_home
+    params[:ES_PATH_CONF] = new_resource.path_conf
+    params[:DATA_DIR] = new_resource.path_data
+    params[:LOG_DIR] = new_resource.path_logs
+    params[:PID_DIR] = new_resource.path_pid
+    params[:RESTART_ON_UPGRADE] = new_resource.restart_on_upgrade
+    params[:ES_USER] = es_user.username if es_install.type == 'tarball'
+    params[:ES_GROUP] = es_user.groupname if es_install.type == 'tarball'
+    params[:ES_STARTUP_SLEEP_TIME] = new_resource.startup_sleep_seconds.to_s
+    params[:MAX_OPEN_FILES] = new_resource.nofile_limit
+    params[:MAX_LOCKED_MEMORY] = new_resource.memlock_limit
+    params[:MAX_MAP_COUNT] = new_resource.max_map_count
+
+    default_config_name = es_svc.service_name || es_svc.instance_name || new_resource.instance_name || 'elasticsearch'
+
+    shell_template = template "elasticsearch.in.sh-#{default_config_name}" do
+      path %w[rhel amazon].include?(node['platform_family']) ? "/etc/sysconfig/#{default_config_name}" : "/etc/default/#{default_config_name}"
+      source new_resource.template_elasticsearch_env
+      cookbook new_resource.cookbook_elasticsearch_env
+      mode '0644'
+      variables(params: params)
+      action :nothing
+    end
+    shell_template.run_action(:create)
+    new_resource.updated_by_last_action(true) if shell_template.updated_by_last_action?
+
+    # Create jvm.options file
+    #
+    jvm_options_template = template "jvm_options-#{default_config_name}" do
+      path   "#{new_resource.path_conf}/jvm.options"
+      source new_resource.template_jvm_options
+      cookbook new_resource.cookbook_jvm_options
+      owner es_user.username
+      group es_user.groupname
+      mode '0644'
+      variables(jvm_options: [
+        "-Xms#{new_resource.allocated_memory}",
+        "-Xmx#{new_resource.allocated_memory}",
+        new_resource.jvm_options,
+      ].flatten.join("\n"))
+      action :nothing
+    end
+    jvm_options_template.run_action(:create)
+    new_resource.updated_by_last_action(true) if jvm_options_template.updated_by_last_action?
+
+    # Create ES logging file
+    #
+    logging_template = template "log4j2_properties-#{default_config_name}" do
+      path   "#{new_resource.path_conf}/log4j2.properties"
+      source new_resource.template_log4j2_properties
+      cookbook new_resource.cookbook_log4j2_properties
+      owner es_user.username
+      group es_user.groupname
+      mode '0640'
+      variables(logging: new_resource.logging)
+      action :nothing
+    end
+    logging_template.run_action(:create)
+    new_resource.updated_by_last_action(true) if logging_template.updated_by_last_action?
+
+    # Create ES elasticsearch.yml file
+    #
+    merged_configuration = default_configuration.merge(new_resource.configuration.dup)
+
+    # warn if someone is using symbols. we don't support.
+    found_symbols = merged_configuration.keys.select { |s| s.is_a?(Symbol) }
+    unless found_symbols.empty?
+      Chef::Log.warn("Please change the following to strings in order to work with this Elasticsearch cookbook: #{found_symbols.join(',')}")
+    end
+
+    # workaround for https://github.com/elastic/cookbook-elasticsearch/issues/590
+    config_vars = ElasticsearchCookbook::HashAndMashBlender.new(merged_configuration).to_hash
+
+    yml_template = template "elasticsearch.yml-#{default_config_name}" do
+      path "#{new_resource.path_conf}/elasticsearch.yml"
+      source new_resource.template_elasticsearch_yml
+      cookbook new_resource.cookbook_elasticsearch_yml
+      owner es_user.username
+      group es_user.groupname
+      mode '0640'
+      helpers(ElasticsearchCookbook::Helpers)
+      variables(config: config_vars)
+      action :nothing
+    end
+    yml_template.run_action(:create)
+    new_resource.updated_by_last_action(true) if yml_template.updated_by_last_action?
+  end
+end

cookbooks/elasticsearch/libraries/provider_install.rb

@@ -0,0 +1,238 @@
+# Chef Provider for installing or removing Elasticsearch from package or tarball
+# downloaded from elasticsearch.org and installed by package manager or ark resource
+class ElasticsearchCookbook::InstallProvider < Chef::Provider::LWRPBase
+  include ElasticsearchCookbook::Helpers
+  include Chef::DSL::IncludeRecipe
+  provides :elasticsearch_install
+
+  def whyrun_supported?
+    true # we only use core Chef resources that also support whyrun
+  end
+
+  def action_install
+    if new_resource.type == 'tarball'
+      install_tarball_wrapper_action
+    elsif new_resource.type == 'package'
+      install_package_wrapper_action
+    elsif new_resource.type == 'repository'
+      install_repo_wrapper_action
+    else
+      raise "#{install_type} is not a valid install type"
+    end
+  end
+
+  def action_remove
+    if new_resource.type == 'tarball'
+      remove_tarball_wrapper_action
+    elsif new_resource.type == 'package'
+      remove_package_wrapper_action
+    elsif new_resource.type == 'repository'
+      remove_repo_wrapper_action
+    else
+      raise "#{install_type} is not a valid install type"
+    end
+  end
+
+  protected
+
+  def install_repo_wrapper_action
+    es_user = find_es_resource(Chef.run_context, :elasticsearch_user, new_resource)
+    unless es_user && es_user.username == 'elasticsearch' && es_user.groupname == 'elasticsearch'
+      raise 'Custom usernames/group names is not supported in Elasticsearch 6+ repository installation'
+    end
+
+    if new_resource.enable_repository_actions
+      if node['platform_family'] == 'debian'
+        apt_r = apt_repo_resource
+        apt_r.run_action(:add)
+        new_resource.updated_by_last_action(true) if apt_r.updated_by_last_action?
+      else
+        yr_r = yum_repo_resource
+        yr_r.run_action(:create)
+        new_resource.updated_by_last_action(true) if yr_r.updated_by_last_action?
+      end
+    end
+
+    if !new_resource.version.nil? && %w[rhel amazon].include?(node['platform_family']) && !new_resource.version.include?('-')
+      # NB: yum repo packages are broken in Chef if you don't specify a release
+      #     https://github.com/chef/chef/issues/4103
+      new_resource.version = "#{new_resource.version}-1"
+    end
+
+    pkg_r = package 'elasticsearch' do
+      options new_resource.package_options
+      version new_resource.version
+      action :nothing
+    end
+
+    pkg_r.run_action(:install)
+    new_resource.updated_by_last_action(true) if pkg_r.updated_by_last_action?
+  end
+
+  def remove_repo_wrapper_action
+    if new_resource.enable_repository_actions
+      if node['platform_family'] == 'debian'
+        apt_r = apt_repo_resource
+        apt_r.run_action(:remove)
+        new_resource.updated_by_last_action(true) if apt_r.updated_by_last_action?
+      else
+        yr_r = yum_repo_resource
+        yr_r.run_action(:delete)
+        new_resource.updated_by_last_action(true) if yr_r.updated_by_last_action?
+      end
+    end
+
+    pkg_r = package 'elasticsearch' do
+      options new_resource.package_options
+      version new_resource.version
+      action :nothing
+    end
+    pkg_r.run_action(:remove)
+    new_resource.updated_by_last_action(true) if pkg_r.updated_by_last_action?
+  end
+
+  def install_package_wrapper_action
+    es_user = find_es_resource(Chef.run_context, :elasticsearch_user, new_resource)
+    unless es_user && es_user.username == 'elasticsearch' && es_user.groupname == 'elasticsearch'
+      raise 'Custom usernames/group names is not supported in Elasticsearch 6+ package installation'
+    end
+
+    found_download_url = determine_download_url(new_resource, node)
+    unless found_download_url
+      raise 'Could not determine download url for package on this platform'
+    end
+
+    filename = found_download_url.split('/').last
+    checksum = determine_download_checksum(new_resource, node)
+    package_options = new_resource.package_options
+
+    unless checksum
+      Chef::Log.warn("No checksum was provided for #{found_download_url}, this may download a new package on every chef run!")
+    end
+
+    remote_file_r = remote_file "#{Chef::Config[:file_cache_path]}/#{filename}" do
+      source found_download_url
+      checksum checksum
+      mode '0644'
+      action :nothing
+    end
+    remote_file_r.run_action(:create)
+    new_resource.updated_by_last_action(true) if remote_file_r.updated_by_last_action?
+
+    pkg_r = if node['platform_family'] == 'debian'
+              dpkg_package "#{Chef::Config[:file_cache_path]}/#{filename}" do
+                options package_options
+                action :nothing
+              end
+            else
+              package "#{Chef::Config[:file_cache_path]}/#{filename}" do
+                options package_options
+                action :nothing
+              end
+            end
+
+    pkg_r.run_action(:install)
+    new_resource.updated_by_last_action(true) if pkg_r.updated_by_last_action?
+  end
+
+  def remove_package_wrapper_action
+    package_url = determine_download_url(new_resource, node)
+    filename = package_url.split('/').last
+
+    pkg_r = if node['platform_family'] == 'debian'
+              dpkg_package "#{Chef::Config[:file_cache_path]}/#{filename}" do
+                action :nothing
+              end
+            else
+              package "#{Chef::Config[:file_cache_path]}/#{filename}" do
+                action :nothing
+              end
+            end
+
+    pkg_r.run_action(:remove)
+    new_resource.updated_by_last_action(true) if pkg_r.updated_by_last_action?
+  end
+
+  def install_tarball_wrapper_action
+    include_recipe 'ark'
+
+    es_user = find_es_resource(Chef.run_context, :elasticsearch_user, new_resource)
+    found_download_url = determine_download_url(new_resource, node)
+    unless found_download_url
+      raise 'Could not determine download url for tarball on this platform'
+    end
+
+    ark_r = ark 'elasticsearch' do
+      url   found_download_url
+      owner es_user.username
+      group es_user.groupname
+      version new_resource.version
+      has_binaries ['bin/elasticsearch', 'bin/elasticsearch-plugin']
+      checksum determine_download_checksum(new_resource, node)
+      prefix_root   new_resource.dir
+      prefix_home   new_resource.dir
+
+      not_if do
+        link   = "#{new_resource.dir}/elasticsearch"
+        target = "#{new_resource.dir}/elasticsearch-#{new_resource.version}"
+        binary = "#{target}/bin/elasticsearch"
+
+        ::File.directory?(link) && ::File.symlink?(link) && ::File.readlink(link) == target && ::File.exist?(binary)
+      end
+      action :nothing
+    end
+    ark_r.run_action(:install)
+    new_resource.updated_by_last_action(true) if ark_r.updated_by_last_action?
+
+    # destroy the sample config directory for tarball installs, or it will
+    # take precedence beyond the default stuff in /etc/elasticsearch and within
+    # /etc/sysconfig or /etc/default
+    sample_r = directory "#{new_resource.dir}/elasticsearch/config" do
+      action :nothing
+      recursive true
+    end
+    sample_r.run_action(:delete)
+    new_resource.updated_by_last_action(true) if sample_r.updated_by_last_action?
+  end
+
+  def remove_tarball_wrapper_action
+    # remove the symlink to this version
+    link_r = link "#{new_resource.dir}/elasticsearch" do
+      only_if do
+        link   = "#{new_resource.dir}/elasticsearch"
+        target = "#{new_resource.dir}/elasticsearch-#{new_resource.version}"
+
+        ::File.directory?(link) && ::File.symlink?(link) && ::File.readlink(link) == target
+      end
+      action :nothing
+    end
+    link_r.run_action(:delete)
+    new_resource.updated_by_last_action(true) if link_r.updated_by_last_action?
+
+    # remove the specific version
+    d_r = directory "#{new_resource.dir}/elasticsearch-#{new_resource.version}" do
+      recursive true
+      action :nothing
+    end
+    d_r.run_action(:delete)
+    new_resource.updated_by_last_action(true) if d_r.updated_by_last_action?
+  end
+
+  def yum_repo_resource
+    yum_repository "elastic-#{new_resource.version.to_i}.x" do
+      baseurl "https://artifacts.elastic.co/packages/#{new_resource.version.to_i}.x/yum"
+      gpgkey 'https://artifacts.elastic.co/GPG-KEY-elasticsearch'
+      action :nothing # :add, remove
+    end
+  end
+
+  def apt_repo_resource
+    apt_repository "elastic-#{new_resource.version.to_i}.x" do
+      uri "https://artifacts.elastic.co/packages/#{new_resource.version.to_i}.x/apt"
+      key 'https://artifacts.elastic.co/GPG-KEY-elasticsearch'
+      components ['main']
+      distribution 'stable'
+      action :nothing # :create, :delete
+    end
+  end
+end

cookbooks/elasticsearch/libraries/provider_plugin.rb

@@ -0,0 +1,108 @@
+# Chef Provider for installing an elasticsearch plugin
+class ElasticsearchCookbook::PluginProvider < Chef::Provider::LWRPBase
+  include ElasticsearchCookbook::Helpers
+  include Chef::Mixin::ShellOut
+
+  provides :elasticsearch_plugin
+
+  def whyrun_supported?
+    true # we only use core Chef resources that also support whyrun, or guard
+  end
+
+  def action_install
+    return if plugin_exists(new_resource.plugin_name)
+
+    # since install can take a URL argument instead
+    url_or_name = new_resource.url || new_resource.plugin_name
+    manage_plugin("install #{url_or_name}")
+  end # action
+
+  def action_remove
+    return unless plugin_exists(new_resource.plugin_name)
+
+    manage_plugin("remove #{new_resource.plugin_name}")
+  end # action
+
+  def manage_plugin(arguments)
+    es_user = find_es_resource(Chef.run_context, :elasticsearch_user, new_resource)
+    es_install = find_es_resource(Chef.run_context, :elasticsearch_install, new_resource)
+    es_conf = find_es_resource(Chef.run_context, :elasticsearch_configure, new_resource)
+
+    assert_state_is_valid(es_user, es_install, es_conf) unless whyrun_mode?
+
+    # shell_out! automatically raises on error, logs command output
+    # required for package installs that show up with parent dir owned by root
+    plugin_dir_exists = ::File.exist?(es_conf.path_plugins)
+    unless plugin_dir_exists
+      cmd_str = "mkdir -p #{es_conf.path_plugins}"
+      if whyrun_mode?
+        Chef::Log.info("Would run command: #{cmd_str}")
+      else
+        shell_out_as_user!(cmd_str, Chef.run_context)
+        new_resource.updated_by_last_action(true)
+      end
+    end
+
+    unless plugin_exists(new_resource.plugin_name)
+      cmd_str = "#{es_conf.path_bin}/elasticsearch-plugin #{arguments.chomp(' ')} #{new_resource.options}".chomp(' ')
+      if whyrun_mode?
+        Chef::Log.info("Would run command: #{cmd_str}")
+      else
+        command_array = cmd_str.split(' ')
+        shell_out_as_user!(command_array, Chef.run_context)
+        new_resource.updated_by_last_action(true)
+      end
+    end
+  end
+
+  def plugin_exists(name)
+    es_conf = find_es_resource(Chef.run_context, :elasticsearch_configure, new_resource)
+    path = es_conf.path_plugins
+
+    Dir.entries(path).any? do |plugin|
+      next if plugin =~ /^\./
+      name == plugin
+    end
+  rescue
+    false
+  end
+
+  def assert_state_is_valid(_es_user, _es_install, es_conf)
+    unless es_conf.path_plugins # we do not check existence (may not exist if no plugins installed)
+      raise "Could not determine the plugin directory (#{es_conf.path_plugins}). Please check elasticsearch_configure[#{es_conf.name}]."
+    end
+
+    unless es_conf.path_bin && ::File.exist?(es_conf.path_bin)
+      raise "Could not determine the binary directory (#{es_conf.path_bin}). Please check elasticsearch_configure[#{es_conf.name}]."
+    end
+
+    true
+  end
+
+  def shell_out_as_user!(command, run_ctx)
+    es_install = find_es_resource(run_ctx, :elasticsearch_install, new_resource)
+    es_conf = find_es_resource(run_ctx, :elasticsearch_configure, new_resource)
+    es_svc = find_es_resource(run_ctx, :elasticsearch_service, new_resource)
+
+    # we need to figure out the env file path to set environment for plugins
+    default_config_name = es_svc.service_name || es_svc.instance_name || es_conf.instance_name || 'elasticsearch'
+    include_file_resource = find_exact_resource(run_ctx, :template, "elasticsearch.in.sh-#{default_config_name}")
+    env = { 'ES_INCLUDE' => include_file_resource.path }
+
+    # Add HTTP Proxy vars unless explicitly told not to
+    if new_resource.chef_proxy
+      env['ES_JAVA_OPTS'] = "#{ENV['ES_JAVA_OPTS']} #{get_java_proxy_arguments}"
+    end
+
+    # See this link for an explanation:
+    # https://www.elastic.co/guide/en/elasticsearch/plugins/2.1/plugin-management.html
+    if es_install.type == 'package' || es_install.type == 'repository'
+      # package installations should install plugins as root
+      shell_out!(command, :env => env, :timeout => 1200)
+    else
+      # non-package installations should install plugins as the ES user
+      es_user = find_es_resource(run_ctx, :elasticsearch_user, new_resource)
+      shell_out!(command, user: es_user.username, group: es_user.groupname, :env => env, :timeout => 1200)
+    end
+  end
+end # provider

cookbooks/elasticsearch/libraries/provider_service.rb

@@ -0,0 +1,140 @@
+# Chef Provider for configuring an elasticsearch service in the init system
+class ElasticsearchCookbook::ServiceProvider < Chef::Provider::LWRPBase
+  provides :elasticsearch_service
+  include ElasticsearchCookbook::Helpers
+
+  def whyrun_supported?
+    true # we only use core Chef resources that also support whyrun
+  end
+
+  def action_remove
+    raise "#{new_resource} remove not currently implemented"
+  end
+
+  def action_configure
+    es_user = find_es_resource(Chef.run_context, :elasticsearch_user, new_resource)
+    es_install = find_es_resource(Chef.run_context, :elasticsearch_install, new_resource)
+    es_conf = find_es_resource(Chef.run_context, :elasticsearch_configure, new_resource)
+    default_config_name = new_resource.service_name || new_resource.instance_name || es_conf.instance_name || 'elasticsearch'
+
+    d_r = directory "#{es_conf.path_pid}-#{default_config_name}" do
+      path es_conf.path_pid
+      owner es_user.username
+      group es_user.groupname
+      mode '0755'
+      recursive true
+      action :nothing
+    end
+    d_r.run_action(:create)
+    new_resource.updated_by_last_action(true) if d_r.updated_by_last_action?
+
+    # Create service for init and systemd
+    #
+    if new_resource.init_source
+      init_r = template "/etc/init.d/#{new_resource.service_name}" do
+        source new_resource.init_source
+        cookbook new_resource.init_cookbook
+        owner 'root'
+        mode '0755'
+        variables(
+          # we need to include something about #{progname} fixed in here.
+          program_name: new_resource.service_name,
+          install_type: es_install.type
+        )
+        only_if { ::File.exist?('/etc/init.d') }
+        action :nothing
+      end
+      init_r.run_action(:create)
+      new_resource.updated_by_last_action(true) if init_r.updated_by_last_action?
+    end
+
+    if new_resource.systemd_source
+      systemd_parent_r = directory "/usr/lib/systemd/system-#{default_config_name}" do
+        path '/usr/lib/systemd/system'
+        action :nothing
+        only_if { ::File.exist?('/usr/lib/systemd') }
+      end
+      systemd_parent_r.run_action(:create)
+      new_resource.updated_by_last_action(true) if systemd_parent_r.updated_by_last_action?
+
+      default_conf_dir = %w[rhel amazon].include?(node['platform_family']) ? '/etc/sysconfig' : '/etc/default'
+      systemd_r = template "/usr/lib/systemd/system/#{new_resource.service_name}.service" do
+        source new_resource.systemd_source
+        cookbook new_resource.systemd_cookbook
+        owner 'root'
+        mode '0644'
+        variables(
+          # we need to include something about #{progname} fixed in here.
+          program_name: new_resource.service_name,
+          default_dir: default_conf_dir,
+          path_home: es_conf.path_home,
+          es_user: es_user.username,
+          es_group: es_user.groupname,
+          nofile_limit: es_conf.nofile_limit,
+          install_type: es_install.type
+        )
+        only_if 'which systemctl'
+        action :nothing
+      end
+      systemd_r.run_action(:create)
+      # special case here -- must reload unit files if we modified one
+      if systemd_r.updated_by_last_action?
+        new_resource.updated_by_last_action(systemd_r.updated_by_last_action?)
+
+        reload_r = execute "reload-systemd-#{new_resource.service_name}" do
+          command 'systemctl daemon-reload'
+          action :nothing
+          only_if 'which systemctl'
+        end
+        reload_r.run_action(:run)
+      end
+    end
+
+    # flatten in an array here, in case the service_actions are a symbol vs. array
+    [new_resource.service_actions].flatten.each do |act|
+      passthrough_action(act)
+    end
+  end
+
+  # Passthrough actions to service[service_name]
+  #
+  def action_enable
+    passthrough_action(:enable)
+  end
+
+  def action_disable
+    passthrough_action(:disable)
+  end
+
+  def action_start
+    passthrough_action(:start)
+  end
+
+  def action_stop
+    passthrough_action(:stop)
+  end
+
+  def action_restart
+    passthrough_action(:restart)
+  end
+
+  def action_status
+    passthrough_action(:status)
+  end
+
+  def passthrough_action(action)
+    svc_r = lookup_service_resource
+    svc_r.run_action(action)
+    new_resource.updated_by_last_action(true) if svc_r.updated_by_last_action?
+  end
+
+  def lookup_service_resource
+    rc = Chef.run_context.resource_collection
+    rc.find("service[#{new_resource.service_name}]")
+  rescue
+    service new_resource.service_name do
+      supports status: true, restart: true
+      action :nothing
+    end
+  end
+end

cookbooks/elasticsearch/libraries/provider_user.rb

@@ -0,0 +1,48 @@
+# Chef Provider for creating a user and group for Elasticsearch
+class ElasticsearchCookbook::UserProvider < Chef::Provider::LWRPBase
+  include ElasticsearchCookbook::Helpers
+
+  provides :elasticsearch_user
+
+  def whyrun_supported?
+    true # we only use core Chef resources that also support whyrun
+  end
+
+  def action_create
+    group_r = group new_resource.groupname do
+      gid new_resource.gid
+      action :nothing
+      system true
+    end
+    group_r.run_action(:create)
+    new_resource.updated_by_last_action(true) if group_r.updated_by_last_action?
+
+    user_r = user new_resource.username do
+      comment new_resource.comment
+      shell   new_resource.shell
+      uid     new_resource.uid
+      gid     new_resource.groupname
+
+      manage_home false
+      action :nothing
+      system true
+    end
+    user_r.run_action(:create)
+    new_resource.updated_by_last_action(true) if user_r.updated_by_last_action?
+  end
+
+  def action_remove
+    # delete user before deleting the group
+    user_r = user new_resource.username do
+      action :nothing
+    end
+    user_r.run_action(:remove)
+    new_resource.updated_by_last_action(true) if user_r.updated_by_last_action?
+
+    group_r = group new_resource.groupname do
+      action :nothing
+    end
+    group_r.run_action(:remove)
+    new_resource.updated_by_last_action(true) if group_r.updated_by_last_action?
+  end
+end

cookbooks/elasticsearch/libraries/resource_configure.rb

@@ -0,0 +1,92 @@
+# Chef Resource for configuring an Elasticsearch node
+class ElasticsearchCookbook::ConfigureResource < Chef::Resource::LWRPBase
+  resource_name :elasticsearch_configure
+  provides :elasticsearch_configure
+
+  actions(:manage, :remove)
+  default_action :manage
+
+  # this is what helps the various resources find each other
+  attribute(:instance_name, kind_of: String, default: nil)
+
+  # if you override one of these, you should probably override them all
+  attribute(:path_home,    kind_of: String, default: '/usr/share/elasticsearch')
+  attribute(:path_conf,    kind_of: String, default: '/etc/elasticsearch')
+  attribute(:path_data,    kind_of: String, default: '/var/lib/elasticsearch')
+  attribute(:path_logs,    kind_of: String, default: '/var/log/elasticsearch')
+  attribute(:path_pid,     kind_of: String, default: '/var/run/elasticsearch')
+  attribute(:path_plugins, kind_of: String, default: '/usr/share/elasticsearch/plugins')
+  attribute(:path_bin,     kind_of: String, default: '/usr/share/elasticsearch/bin')
+
+  attribute(:template_elasticsearch_env, kind_of: String, default: 'elasticsearch.in.sh.erb')
+  attribute(:cookbook_elasticsearch_env, kind_of: String, default: 'elasticsearch')
+
+  attribute(:template_jvm_options, kind_of: String, default: 'jvm_options.erb')
+  attribute(:cookbook_jvm_options, kind_of: String, default: 'elasticsearch')
+
+  attribute(:template_elasticsearch_yml, kind_of: String, default: 'elasticsearch.yml.erb')
+  attribute(:cookbook_elasticsearch_yml, kind_of: String, default: 'elasticsearch')
+
+  attribute(:template_log4j2_properties, kind_of: String, default: 'log4j2.properties.erb')
+  attribute(:cookbook_log4j2_properties, kind_of: String, default: 'elasticsearch')
+
+  attribute(:logging, kind_of: Hash, default: {}.freeze)
+  attribute(:java_home, kind_of: String, default: nil)
+
+  # other settings in /etc/default or /etc/sysconfig
+  attribute(:memlock_limit, kind_of: String, default: 'unlimited')
+  attribute(:max_map_count, kind_of: String, default: '262144')
+  attribute(:nofile_limit, kind_of: String, default: '65536')
+  attribute(:startup_sleep_seconds, kind_of: [String, Integer], default: 5)
+  attribute(:restart_on_upgrade, kind_of: [TrueClass, FalseClass], default: false)
+
+  # Calculations for this are done in the provider, as we can't do them in the
+  # resource definition. default is 50% of RAM or 31GB, which ever is smaller.
+  attribute(:allocated_memory, kind_of: String)
+
+  attribute(:jvm_options, kind_of: Array, default:
+    %w(
+      -XX:+UseConcMarkSweepGC
+      -XX:CMSInitiatingOccupancyFraction=75
+      -XX:+UseCMSInitiatingOccupancyOnly
+      -XX:+AlwaysPreTouch
+      -server
+      -Xss1m
+      -Djava.awt.headless=true
+      -Dfile.encoding=UTF-8
+      -Djna.nosys=true
+      -XX:-OmitStackTraceInFastThrow
+      -Dio.netty.noUnsafe=true
+      -Dio.netty.noKeySetOptimization=true
+      -Dio.netty.recycler.maxCapacityPerThread=0
+      -Dlog4j.shutdownHookEnabled=false
+      -Dlog4j2.disable.jmx=true
+      -XX:+HeapDumpOnOutOfMemoryError
+    ).freeze)
+
+  # These are the default settings. Most of the time, you want to override
+  # the `configuration` attribute below. If you do override the defaults, you
+  # must supply ALL needed defaults, and don't use nil as a value in the hash.
+  attribute(:default_configuration, kind_of: Hash, default: {
+    # === NAMING
+    'cluster.name' => 'elasticsearch',
+    # can't access node.name, so expect to have to set set this
+    'node.name' => Chef::Config[:node_name],
+
+    # if omitted or nil, these will be populated from attributes above
+    'path.data' => nil, # see path_data above
+    'path.logs' => nil, # see path_logs above
+
+    # Refer to ES documentation on how to configure these to a
+    # specific node role/type instead of using the defaults
+    #
+    # 'node.data' => ?,
+    # 'node.master' => ?,
+  }.freeze)
+
+  # These settings are merged with the `default_configuration` attribute,
+  # allowing you to override and set specific settings. Unless you intend to
+  # wipe out all default settings, your configuration items should go here.
+  #
+  attribute(:configuration, kind_of: Hash, default: {}.freeze)
+end

cookbooks/elasticsearch/libraries/resource_install.rb

@@ -0,0 +1,31 @@
+# Chef Resource for installing or removing Elasticsearch from package or source
+class ElasticsearchCookbook::InstallResource < Chef::Resource::LWRPBase
+  resource_name :elasticsearch_install
+  provides :elasticsearch_install
+
+  actions(:install, :remove)
+  default_action :install
+
+  # this is what helps the various resources find each other
+  attribute(:instance_name, kind_of: String)
+
+  # if this version parameter is not set by the caller, we look at
+  # `attributes/default.rb` for a default value to use, or we raise
+  attribute(:version, kind_of: String, default: '7.3.0')
+
+  # we allow a string or symbol for this value
+  attribute(:type, kind_of: String, equal_to: %w(package tarball repository), default: 'repository')
+
+  # these use `attributes/default.rb` for default values per platform and install type
+  attribute(:download_url, kind_of: String)
+  attribute(:download_checksum, kind_of: String) # sha256
+
+  # where to install?
+  attribute(:dir, kind_of: String, default: '/usr/share')
+
+  # attributes used by the package-flavor provider
+  attribute(:package_options, kind_of: String)
+
+  # attributes for the repository-option install
+  attribute(:enable_repository_actions, kind_of: [TrueClass, FalseClass], default: true)
+end

cookbooks/elasticsearch/libraries/resource_plugin.rb

@@ -0,0 +1,19 @@
+# Chef Resource for installing an elasticsearch plugin
+class ElasticsearchCookbook::PluginResource < Chef::Resource::LWRPBase
+  resource_name :elasticsearch_plugin
+  provides :elasticsearch_plugin
+
+  include ElasticsearchCookbook::Helpers
+
+  actions(:install, :remove)
+  default_action :install
+
+  # if the name or url are different from the resource name
+  attribute(:plugin_name, kind_of: String, name_attribute: true)
+  attribute(:url, kind_of: String, default: nil)
+  attribute(:chef_proxy, kind_of: [TrueClass, FalseClass], default: false)
+  attribute(:options, kind_of: String, default: '')
+
+  # this is what helps the various resources find each other
+  attribute(:instance_name, kind_of: String, default: nil)
+end

cookbooks/elasticsearch/libraries/resource_service.rb

@@ -0,0 +1,28 @@
+# Chef Resource for declaring a service for Elasticsearch
+class ElasticsearchCookbook::ServiceResource < Chef::Resource::LWRPBase
+  resource_name :elasticsearch_service
+  provides :elasticsearch_service
+
+  actions(
+    :configure, :remove, # our custom actions
+    :enable, :disable, :start, :stop, :restart, :status # passthrough to service resource
+  )
+  default_action :configure
+
+  # this is what helps the various resources find each other
+  attribute(:instance_name, kind_of: String, default: nil)
+
+  attribute(:service_name, kind_of: String, name_attribute: true)
+  attribute(:args, kind_of: String, default: '-d')
+
+  # service actions
+  attribute(:service_actions, kind_of: [Symbol, String, Array], default: [:enable, :start].freeze)
+
+  # allow overridable init script
+  attribute(:init_source, kind_of: String, default: 'initscript.erb')
+  attribute(:init_cookbook, kind_of: String, default: 'elasticsearch')
+
+  # allow overridable systemd unit
+  attribute(:systemd_source, kind_of: String, default: 'systemd_unit.erb')
+  attribute(:systemd_cookbook, kind_of: String, default: 'elasticsearch')
+end

cookbooks/elasticsearch/libraries/resource_user.rb

@@ -0,0 +1,19 @@
+# Chef Resource for declaring a user and group for Elasticsearch
+class ElasticsearchCookbook::UserResource < Chef::Resource::LWRPBase
+  resource_name :elasticsearch_user
+  provides :elasticsearch_user
+
+  actions(:create, :remove)
+  default_action :create
+
+  # this is what helps the various resources find each other
+  attribute(:instance_name, kind_of: String, default: nil)
+
+  attribute(:username, kind_of: String, name_attribute: true) # default to resource name
+  attribute(:uid, kind_of: Integer)
+  attribute(:shell, kind_of: String, default: '/bin/bash')
+  attribute(:comment, kind_of: String, default: 'Elasticsearch User')
+
+  attribute(:groupname, kind_of: String, name_attribute: true) # default to resource name
+  attribute(:gid, kind_of: Integer)
+end