Add support for Cloud Foundry

Signed-off-by: Natalie Tay <ntay@pivotal.io>
Signed-off-by: Alan Yeo <ayeo@pivotal.io>
Signed-off-by: Benjamin Tan <btan@pivotal.io>

Author: gamov

README.md

@@ -333,6 +333,21 @@ filesystem gets recreated from the git sources on each instance refresh. To use
 
 To upload your local values to Heroku you could ran `bundle exec rake config:heroku`.
 
+### Working with Cloud Foundry
+
+Cloud Foundry integration will generate a manifest from your CF manifest with the defined ENV variables added 
+under the `env` section. **ENV variables will be added to all applications specified in the manifest.** By default, 
+it uses `manifest.yml` and the current `Rails.env`:
+
+    bundle exec rake config:cf
+    
+You may optionally pass target environment _and_ the name of your CF manifest file (in that case, both are compulsory):
+
+    bundle exec rake config:cf[target_env, your_manifest.yml]
+    
+The result of this command will create a new manifest file, name suffixed with '-merged'. You can then push your app 
+with the generated manifest.    
+
 ### Fine-tuning
 
 You can customize how environment variables are processed:

lib/config/integrations/cloud_foundry.rb

@@ -0,0 +1,28 @@
+require 'bundler'
+require 'yaml'
+require_relative '../../../lib/config/integrations/helpers/cf_manifest_merger'
+
+module Config
+  module Integrations
+    class CloudFoundry < Struct.new(:target_env, :file_path)
+
+      def invoke
+
+        manifest_path = file_path
+        file_name, _ext = manifest_path.split('.yml')
+
+        manifest_hash = YAML.load(IO.read(File.join(::Rails.root, manifest_path)))
+
+        puts "Generating manifest... (base cf manifest: #{manifest_path})"
+
+        merged_hash = Config::CFManifestMerger.new(target_env, manifest_hash).add_to_env
+
+        target_manifest_path = File.join(::Rails.root, "#{file_name}-merged.yml")
+        IO.write(target_manifest_path, merged_hash.to_yaml)
+
+        puts "File #{target_manifest_path} generated."
+      end
+
+    end
+  end
+end

lib/config/integrations/helpers/cf_manifest_merger.rb

@@ -0,0 +1,39 @@
+require_relative 'helpers'
+
+module Config
+  class CFManifestMerger
+    include Integrations::Helpers
+
+    def initialize(target_env, manifest_hash)
+      @manifest_hash = manifest_hash.dup
+
+      raise ArgumentError.new('Target environment & manifest path must be specified') unless target_env && @manifest_hash
+
+      config_root = File.join(Rails.root, 'config')
+      config_setting_files = Config.setting_files(config_root, target_env)
+      @settings_hash = Config.load_files(config_setting_files).to_hash.stringify_keys
+    end
+
+    def add_to_env
+
+      prefix_keys_with_const_name_hash = to_dotted_hash(@settings_hash, namespace: Config.const_name)
+
+      apps = @manifest_hash['applications']
+
+      apps.each do |app|
+        check_conflicting_keys(app['env'], @settings_hash)
+        app['env'].merge!(prefix_keys_with_const_name_hash)
+      end
+
+      @manifest_hash
+    end
+
+    private
+
+    def check_conflicting_keys(env_hash, settings_hash)
+      conflicting_keys = env_hash.keys & settings_hash.keys
+      raise ArgumentError.new("Conflicting keys: #{conflicting_keys.join(', ')}") if conflicting_keys.any?
+    end
+
+  end
+end

lib/config/integrations/helpers/helpers.rb

@@ -0,0 +1,21 @@
+module Config::Integrations::Helpers
+
+  def to_dotted_hash(source, target: {}, namespace: nil)
+    raise ArgumentError, "target must be a hash (given: #{target.class.name})" unless target.kind_of? Hash
+    prefix = "#{namespace}." if namespace
+    case source
+      when Hash
+        source.each do |key, value|
+          to_dotted_hash(value, target: target, namespace: "#{prefix}#{key}")
+        end
+      when Array
+        source.each_with_index do |value, index|
+          to_dotted_hash(value, target: target, namespace: "#{prefix}#{index}")
+        end
+      else
+        target[namespace] = source
+    end
+    target
+  end
+
+end

lib/config/integrations/heroku.rb

@@ -1,8 +1,11 @@
 require 'bundler'
+require_relative 'helpers/helpers'
 
 module Config
   module Integrations
     class Heroku < Struct.new(:app)
+      include Integrations::Helpers
+
       def invoke
         puts 'Setting vars...'
         heroku_command = "config:set #{vars}"
@@ -14,13 +17,13 @@ def invoke
       def vars
         # Load only local options to Heroku
         Config.load_and_set_settings(
-            Rails.root.join("config", "settings.local.yml").to_s,
-            Rails.root.join("config", "settings", "#{environment}.local.yml").to_s,
-            Rails.root.join("config", "environments", "#{environment}.local.yml").to_s
+            ::Rails.root.join("config", "settings.local.yml").to_s,
+            ::Rails.root.join("config", "settings", "#{environment}.local.yml").to_s,
+            ::Rails.root.join("config", "environments", "#{environment}.local.yml").to_s
         )
 
         out = ''
-        dotted_hash = to_dotted_hash Kernel.const_get(Config.const_name).to_hash, {}, Config.const_name
+        dotted_hash = to_dotted_hash Kernel.const_get(Config.const_name).to_hash, namespace: Config.const_name
         dotted_hash.each {|key, value| out += " #{key}=#{value} "}
         out
       end
@@ -38,22 +41,6 @@ def `(command)
         Bundler.with_clean_env { super }
       end
 
-      def to_dotted_hash(source, target = {}, namespace = nil)
-        prefix = "#{namespace}." if namespace
-        case source
-          when Hash
-            source.each do |key, value|
-              to_dotted_hash(value, target, "#{prefix}#{key}")
-            end
-          when Array
-            source.each_with_index do |value, index|
-              to_dotted_hash(value, target, "#{prefix}#{index}")
-            end
-          else
-            target[namespace] = source
-        end
-        target
-      end
     end
   end
 end

lib/config/integrations/rails/railtie.rb

@@ -15,7 +15,7 @@ def preload
 
         # Load rake tasks (eg. Heroku)
         rake_tasks do
-          Dir[File.join(File.dirname(__FILE__), '../tasks/*.rake')].each { |f| load f }
+          Dir[File.join(File.dirname(__FILE__), '../../tasks/*.rake')].each { |f| load f }
         end
 
         config.before_configuration { preload }

lib/config/tasks/cloud_foundry.rake

@@ -0,0 +1,16 @@
+require 'config/integrations/cloud_foundry'
+
+namespace 'config' do
+
+  desc 'Create a cf manifest with the env variables defined by config under current environment'
+  task 'cf', [:target_env, :file_path] => [:environment] do |_, args|
+
+    raise ArgumentError, 'Both target_env and file_path arguments must be specified' if args.length == 1
+
+    default_args = {:target_env => Rails.env, :file_path => 'manifest.yml'}
+    merged_args = default_args.merge(args)
+
+    Config::Integrations::CloudFoundry.new(*merged_args.values).invoke
+  end
+
+end

lib/config/tasks/heroku.rake

@@ -1,7 +1,10 @@
 require 'config/integrations/heroku'
 
 namespace 'config' do
+
+  desc 'Upload to Heroku all env variables defined by config under current environment'
   task :heroku, [:app] => :environment do |_, args|
     Config::Integrations::Heroku.new(args[:app]).invoke
   end
+
 end

spec/fixtures/cf/cf_manifest.yml

@@ -0,0 +1,11 @@
+applications:
+- name: some-cf-app
+  instances: 1
+  env:
+    DEFAULT_HOST: host
+    DEFAULT_PORT: port
+    FOO: BAR
+
+- name: app_name
+  env:
+    DEFAULT_HOST: host

spec/fixtures/cf/config/settings/conflict_settings.yml

@@ -0,0 +1,2 @@
+DEFAULT_HOST: host
+DEFAULT_PORT: port

spec/fixtures/cf/config/settings/multilevel_settings.yml

@@ -0,0 +1,13 @@
+world:
+  capitals:
+    europe:
+      germany: 'Berlin'
+      poland: 'Warsaw'
+  array:
+  - name: 'Alan'
+  - name: 'Gam'
+  array_with_index:
+    0:
+      name: 'Bob'
+    1:
+      name: 'William'

spec/integrations/helpers/cf_manifest_merger_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+require_relative '../../../lib/config/integrations/helpers/cf_manifest_merger'
+
+describe Config::CFManifestMerger do
+
+  let(:mocked_rails_root_path) { "#{fixture_path}/cf/" }
+  let(:manifest_hash) { load_manifest('cf_manifest.yml') }
+
+  it 'raises an argument error if you do not specify a target environment' do
+    expect {
+      Config::CFManifestMerger.new(nil, manifest_hash)
+    }.to raise_error(ArgumentError, 'Target environment & manifest path must be specified')
+  end
+
+  it 'returns the cf manifest unmodified if no settings are available' do
+    merger = Config::CFManifestMerger.new('test', manifest_hash)
+
+    resulting_hash = merger.add_to_env
+    expect(resulting_hash).to eq(manifest_hash)
+  end
+
+  it 'adds the settings for the target_env to the manifest_hash' do
+    allow(Rails).to receive(:root).and_return(mocked_rails_root_path)
+
+    # we use the target_env to load the proper settings file
+    merger = Config::CFManifestMerger.new('multilevel_settings', manifest_hash)
+
+    resulting_hash = merger.add_to_env
+    expect(resulting_hash).to eq({
+                                     'applications' => [
+                                         {
+                                             'name' => 'some-cf-app',
+                                             'instances' => 1,
+                                             'env' => {
+                                                 'DEFAULT_HOST' => 'host',
+                                                 'DEFAULT_PORT' => 'port',
+                                                 'FOO' => 'BAR',
+                                                 'Settings.world.capitals.europe.germany' => 'Berlin',
+                                                 'Settings.world.capitals.europe.poland' => 'Warsaw',
+                                                 'Settings.world.array.0.name' => 'Alan',
+                                                 'Settings.world.array.1.name' => 'Gam',
+                                                 'Settings.world.array_with_index.0.name' => 'Bob',
+                                                 'Settings.world.array_with_index.1.name' => 'William'
+                                             }
+                                         },
+                                         {
+                                             'name' => 'app_name',
+                                             'env' => {
+                                                 'DEFAULT_HOST' => 'host',
+                                                 'Settings.world.capitals.europe.germany' => 'Berlin',
+                                                 'Settings.world.capitals.europe.poland' => 'Warsaw',
+                                                 'Settings.world.array.0.name' => 'Alan',
+                                                 'Settings.world.array.1.name' => 'Gam',
+                                                 'Settings.world.array_with_index.0.name' => 'Bob',
+                                                 'Settings.world.array_with_index.1.name' => 'William'
+                                             }
+                                         }
+                                     ]
+                                 })
+  end
+
+  it 'raises an exception if there is conflicting keys' do
+    allow(Rails).to receive(:root).and_return(mocked_rails_root_path)
+
+    merger = Config::CFManifestMerger.new('conflict_settings', manifest_hash)
+
+    # Config.load_and_set_settings "#{fixture_path}/cf/conflict_settings.yml"
+    expect {
+      merger.add_to_env
+    }.to raise_error(ArgumentError, 'Conflicting keys: DEFAULT_HOST, DEFAULT_PORT')
+  end
+
+  def load_manifest filename
+    YAML.load(IO.read("#{fixture_path}/cf/#{filename}"))
+  end
+end

spec/integrations/helpers/helpers_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require_relative '../../../lib/config/integrations/helpers/helpers'
+
+describe 'Helpers' do
+
+  subject { Class.new.send(:include, Config::Integrations::Helpers).new }
+
+  describe '#to_dotted_hash' do
+
+    context 'only the source is specified' do
+
+      it 'returns a hash with a nil key (default)' do
+        expect(subject.to_dotted_hash 3).to eq({nil => 3})
+      end
+    end
+
+    context 'with invalid arguments' do
+      it 'raises an error' do
+        expect { subject.to_dotted_hash(3, target: [1, 2, 7], namespace: 2) }
+            .to raise_error(ArgumentError, 'target must be a hash (given: Array)')
+      end
+    end
+
+    context 'all arguments specified' do
+
+      it 'returns a hash with the namespace as the key' do
+        expect(subject.to_dotted_hash(3, namespace: 'ns')).to eq({'ns' => 3})
+      end
+
+      it 'returns a new hash with a dotted string key prefixed with namespace' do
+        expect(subject.to_dotted_hash({hello: {cruel: 'world'}}, namespace: 'ns'))
+            .to eq({'ns.hello.cruel' => 'world'})
+      end
+
+      it 'returns the same hash as passed as a parameter' do
+        target = {something: 'inside'}
+        target_id = target.object_id
+        result = subject.to_dotted_hash(2, target: target, namespace: 'ns')
+        expect(result).to eq({:something => 'inside', 'ns' => 2})
+        expect(result.object_id).to eq target_id
+      end
+
+      it 'returns a hash when given a source with mixed nested types (hashes & arrays)' do
+        expect(subject.to_dotted_hash(
+            {hello: {evil: [:cruel, 'world', and: {dark: 'universe'}]}}, namespace: 'ns'))
+            .to eq(
+                    {"ns.hello.evil.0" => :cruel,
+                     "ns.hello.evil.1" => "world",
+                     "ns.hello.evil.2.and.dark" => "universe"}
+                )
+      end
+    end
+  end
+end

spec/tasks/cloud_foundry_spec.rb

@@ -0,0 +1,80 @@
+require 'spec_helper'
+
+describe 'config:cf' do
+  include_context 'rake'
+  let(:test_settings_file) { "#{fixture_path}/cf/config/settings/multilevel_settings.yml" }
+  let(:test_manifest_file) { "#{fixture_path}/cf/cf_manifest.yml" }
+  let(:development_settings_file) { "#{fixture_path}/development.yml" }
+  let(:settings_dir) { Rails.root.join('config', 'settings') }
+
+  def setup_temp_rails_root
+    allow(Rails).to receive(:root).and_return(Pathname.new Dir.mktmpdir)
+  end
+
+  before :all do
+    load File.expand_path('../../../lib/config/tasks/cloud_foundry.rake', __FILE__)
+    Rake::Task.define_task(:environment)
+  end
+
+  before { allow($stdout).to receive(:puts) } # suppressing console output during testing
+
+  after :all do
+    Settings.reload_from_files("#{fixture_path}/settings.yml")
+  end
+
+  it 'raises an error if the manifest file is missing' do
+    expect {
+      Rake::Task['config:cf'].execute
+    }.to raise_error(SystemCallError)
+  end
+
+  it 'raises an error if the settings file is missing' do
+    expect {
+      Rake::Task['config:cf'].execute({target_env: 'not_existing_env', file_path: 'manifest.yml'})
+    }.to raise_error(SystemCallError)
+  end
+
+  describe 'without arguments' do
+    it 'creates the merged manifest file with the settings ENV variables included for all cf applications' do
+      setup_temp_rails_root
+      settings_dir.mkpath
+      FileUtils.cp(test_settings_file, settings_dir.join('test.yml'))
+      FileUtils.cp(test_manifest_file, Rails.root.join('manifest.yml'))
+
+      Rake::Task['config:cf'].execute
+
+      merged_manifest_file = File.join(Rails.root, 'manifest-merged.yml')
+      merged_manifest_file_contents = YAML.load(IO.read(merged_manifest_file))
+
+      expect(merged_manifest_file_contents['applications'][0]['name']).to eq 'some-cf-app'
+      expect(merged_manifest_file_contents['applications'][0]['env']['DEFAULT_HOST']).to eq 'host'
+      expect(merged_manifest_file_contents['applications'][0]['env']['Settings.world.array.0.name']).to eq 'Alan'
+
+      expect(merged_manifest_file_contents['applications'][1]['name']).to eq 'app_name'
+      expect(merged_manifest_file_contents['applications'][1]['env']['Settings.world.array.0.name']).to eq 'Alan'
+    end
+  end
+
+  describe 'with arguments' do
+    it 'raises an error if only one argument is provided' do
+      expect {
+        Rake::Task['config:cf'].execute({target_env:'target_env_name'})
+      }.to raise_error(ArgumentError)
+    end
+
+    it 'takes in account the provided arguments' do
+      setup_temp_rails_root
+      settings_dir.mkpath
+      FileUtils.cp(test_manifest_file, Rails.root.join('cf_manifest.yml'))
+      FileUtils.cp(development_settings_file, settings_dir.join('development.yml'))
+      FileUtils.cp(test_settings_file, settings_dir.join('test.yml'))
+
+      Rake::Task['config:cf'].execute({target_env: 'development', file_path: 'cf_manifest.yml'})
+
+      merged_manifest_file = File.join(Rails.root, 'cf_manifest-merged.yml')
+      merged_manifest_file_contents = YAML.load(IO.read(merged_manifest_file))
+
+      expect(merged_manifest_file_contents['applications'][0]['env']['Settings.size']).to eq 2
+    end
+  end
+end

spec/tasks/db_spec.rb

@@ -3,6 +3,8 @@
 describe 'db:create' do
   include_context 'rake'
 
+  before { allow($stdout).to receive(:puts) } # suppressing console output during testing
+
   it 'has access to Settings object and can read databases from settings.yml file' do
     Rake::Task['db:create'].invoke
   end

spec/tasks/heroku_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe 'config:heroku' do
+  include_context 'rake'
+
+  before do
+    load File.expand_path("../../../lib/config/tasks/heroku.rake", __FILE__)
+    Rake::Task.define_task(:environment)
+  end
+
+  it 'includes the helper module that defines to_dotted_hash' do
+    h = Config::Integrations::Heroku.new
+    expect(h.public_methods(:true)).to include(:to_dotted_hash)
+  end
+end