Merge pull request #407 from engineyard/fb-522-ruby-jemalloc

[FB-522] Support for jemalloc-enabled Ruby

cookbooks/emerge/definitions/package_use.rb

@@ -1,6 +1,7 @@
-define :package_use, :flags => nil do
+define :package_use, :flags => nil, :use_file => nil do
   name = params[:name]
   flags = params[:flags]
+  use_file = params[:use_file] || 'local'
   full_name = name + (" #{flags}" if flags)
 
   file "/etc/portage/package.use" do
@@ -13,14 +14,14 @@
     action :create
   end
 
-  execute "touch /etc/portage/package.use/local" do
+  execute "touch /etc/portage/package.use/#{use_file}" do
     action :run
-    not_if { FileTest.exists?("/etc/portage/package.use/local") }
+    not_if { FileTest.exists?("/etc/portage/package.use/#{use_file}") }
   end
 
-  update_file "local portage package.use" do
-    path "/etc/portage/package.use/local"
+  update_file "#{use_file} portage package.use" do
+    path "/etc/portage/package.use/#{use_file}"
     body full_name
-    not_if "grep '#{full_name}' /etc/portage/package.use/local"
+    not_if "grep '^#{full_name}$' /etc/portage/package.use/#{use_file}"
   end
 end

cookbooks/emerge/definitions/package_use_clear.rb

@@ -0,0 +1,9 @@
+define :package_use_clear do
+  use_file = params[:use_file]
+
+  file "/etc/portage/package.use/#{use_file}" do
+    action :delete
+
+    only_if { FileTest.exists?("/etc/portage/package.use/#{use_file}") }
+  end
+end

cookbooks/ruby/attributes/versions.rb

@@ -6,6 +6,7 @@
 
 default[:ruby_dependencies] = {}
 default[:do_upgrade_eselect_ruby] = false
+default[:ruby_jemalloc_available] = false
 
 if ruby2x?(0, ruby_version)
   default[:ruby_dependencies] = {
@@ -51,6 +52,7 @@
     "dev-ruby/xmlrpc"       => "0.2.1",
     "virtual/rubygems"      => "12",
   }
+  default[:ruby_jemalloc_available] = true
 elsif ruby2x?(5, ruby_version)
   default[:do_upgrade_eselect_ruby] = true
   default[:ruby_dependencies] = {
@@ -68,6 +70,7 @@
     "dev-ruby/xmlrpc"       => "0.3.0",
     "virtual/rubygems"      => "13",
   }
+  default[:ruby_jemalloc_available] = true
 elsif ruby2x?(6, ruby_version)
   default[:ruby_dependencies] = {
     "dev-ruby/bundler"      => "1.17.3",
@@ -85,4 +88,5 @@
     "dev-ruby/kpeg"         => "1.1.0-r2",
     "virtual/rubygems"      => "14",
   }
+  default[:ruby_jemalloc_available] = true
 end

cookbooks/ruby/libraries/helpers.rb

@@ -1,4 +1,8 @@
 module RubyHelpers
+  def ruby_package_atom(package, version)
+    "=#{package}-#{version}"
+  end
+
   def ruby2x?(x, version_string)
     version_string =~ /^2\.#{x}\.[\d\.]*\b/
   end
@@ -98,7 +102,7 @@ def install_ruby_and_deps(reset_and_retry=true)
     })
     package_atoms = packages.map { |package_name, package_version| "=#{package_name}-#{package_version}" }
     execute 'install ruby and its dependencies' do
-      command %Q{emerge --read-news=n -g -n --color n --nospinner --quiet #{package_atoms.join(' ')}}
+      command %Q{emerge --read-news=n -g -n --changed-use --color n --nospinner --quiet #{package_atoms.join(' ')}}
       action :run
     end
   end
@@ -165,6 +169,16 @@ def ensure_rubygems_version
       reinstall_rubygems_os_package
     end
   end
+
+  def is_ruby_jemalloc_enabled(app_data)
+    env_vars = fetch_environment_variables(app_data)
+    env_vars.each do |ev|
+      if /^EY_RUBY_JEMALLOC/.match(ev[:name]) && /TRUE/i.match(ev[:value])
+        return true
+      end
+    end
+    return false
+  end
 end
 
 class Chef::Recipe
@@ -174,3 +188,7 @@ class Chef::Recipe
 class Chef::Node
   include RubyHelpers
 end
+
+class Chef::Resource
+  include RubyHelpers
+end

cookbooks/ruby/recipes/jemalloc.rb

@@ -0,0 +1,58 @@
+component = node.engineyard.environment.ruby
+ruby_jemalloc_available = node.default[:ruby_jemalloc_available]
+
+ruby_pkg_name = component[:package].split('/')[1]
+
+pkg_download_url = "http://distfiles.engineyard.com/EYGL2016.06-#{ruby_pkg_name}-#{component[:version]}-jemalloc.tbz2"
+pkg_destination_directory = "/engineyard/portage/packages/dev-lang"
+pkg_destination_filename = "#{ruby_pkg_name}-#{component[:version]}.tbz2"
+pkg_destination_path = File.join(pkg_destination_directory, pkg_destination_filename)
+
+# Check all apps on the env for ruby+jemalloc
+# If all of them have it enabled, then activate jemalloc
+ruby_jemalloc_enabled = node['dna']['engineyard']['environment']['apps'].all? do |app_data|
+  is_ruby_jemalloc_enabled(app_data)
+end
+
+if ruby_jemalloc_available and ruby_jemalloc_enabled
+  package_use ruby_package_atom(component[:package], component[:version]) do
+    flags 'jemalloc'
+    use_file 'ruby-jemalloc'
+  end
+
+  directory "create the #{ruby_pkg_name} binary directory" do
+    path pkg_destination_directory
+    recursive true
+    action :create
+  end
+
+  ruby_block "download the #{ruby_pkg_name} binary package" do
+    block do
+      wget_cmd = Mixlib::ShellOut.new(
+        "wget -O #{pkg_destination_path} #{pkg_download_url}"
+      )
+      wget_cmd.run_command
+      if wget_cmd.error?
+        Chef::Log.info "The #{pkg_destination_filename} (with jemalloc) binary package does not exist."
+        Chef::Log.info wget_cmd.stderr
+        Mixlib::ShellOut.new("rm -f #{pkg_destination_path}").run_command
+      else # successfully downloaded
+        Mixlib::ShellOut.new("emaint --fix binhost && eix-update").run_command
+      end
+    end
+  end
+else
+  package_use_clear 'remove jemalloc USE-flag' do
+    use_file 'ruby-jemalloc'
+  end
+
+  bash 'remove the binary packages' do
+    code <<-EOH
+    rm -f #{File.join(pkg_destination_directory, "#{ruby_pkg_name}-*.tbz2")}
+    emaint --fix binhost
+    eix-update
+    EOH
+    action :run
+  end
+end
+

cookbooks/ruby/recipes/ruby.rb

@@ -26,4 +26,5 @@
   end
 end
 
+include_recipe 'ruby::jemalloc'
 include_recipe 'ruby::common'