Class | BoxGrinder::S3Plugin |
In: |
lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb
lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb |
Parent: | BasePlugin |
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 30 30: def after_init 31: register_supported_os("fedora", ['13', '14', '15', '16']) 32: register_supported_os("centos", ['5', '6']) 33: register_supported_os("rhel", ['5', '6']) 34: register_supported_os("sl", ['5', '6']) 35: 36: @ami_build_dir = "#{@dir.base}/ami" 37: @ami_manifest = "#{@ami_build_dir}/#{@appliance_config.name}.ec2.manifest.xml" 38: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 30 30: def after_init 31: register_supported_os("fedora", ['13', '14', '15', '16']) 32: register_supported_os("centos", ['5', '6']) 33: register_supported_os("rhel", ['5', '6']) 34: register_supported_os("sl", ['5', '6']) 35: 36: @ami_build_dir = "#{@dir.base}/ami" 37: @ami_manifest = "#{@ami_build_dir}/#{@appliance_config.name}.ec2.manifest.xml" 38: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 201 201: def ami_by_manifest_key(ami_manifest_key) 202: ami = @ec2.images.with_owner(@plugin_config['account_number']). 203: filter("manifest-location","#{@plugin_config['bucket']}/#{ami_manifest_key.key}") 204: return nil unless ami.any? 205: ami.first 206: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 201 201: def ami_by_manifest_key(ami_manifest_key) 202: ami = @ec2.images.with_owner(@plugin_config['account_number']). 203: filter("manifest-location","#{@plugin_config['bucket']}/#{ami_manifest_key.key}") 204: return nil unless ami.any? 205: ami.first 206: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 208 208: def ami_key(appliance_name, path) 209: base_path = "#{@s3helper.parse_path(path)}#{appliance_name}/#{@appliance_config.os.name}/#{@appliance_config.os.version}/#{@appliance_config.version}.#{@appliance_config.release}" 210: 211: return "#{base_path}/#{@appliance_config.hardware.arch}" unless @plugin_config['snapshot'] 212: 213: @log.info "Determining snapshot name" 214: snapshot = 1 215: while @s3helper.stub_s3obj(@bucket, "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}/").exists? 216: snapshot += 1 217: end 218: # Reuse the last key (if there was one) 219: snapshot -=1 if snapshot > 1 and @plugin_config['overwrite'] 220: 221: "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}" 222: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 208 208: def ami_key(appliance_name, path) 209: base_path = "#{@s3helper.parse_path(path)}#{appliance_name}/#{@appliance_config.os.name}/#{@appliance_config.os.version}/#{@appliance_config.version}.#{@appliance_config.release}" 210: 211: return "#{base_path}/#{@appliance_config.hardware.arch}" unless @plugin_config['snapshot'] 212: 213: @log.info "Determining snapshot name" 214: snapshot = 1 215: while @s3helper.stub_s3obj(@bucket, "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}/").exists? 216: snapshot += 1 217: end 218: # Reuse the last key (if there was one) 219: snapshot -=1 if snapshot > 1 and @plugin_config['overwrite'] 220: 221: "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}" 222: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 139 139: def asset_bucket(create_if_missing = true, permissions = :private) 140: @s3helper.bucket(:bucket => @plugin_config['bucket'], 141: :create_if_missing => create_if_missing, 142: :acl => permissions, 143: :location_constraint => @s3_endpoints[@plugin_config['region']][:location] 144: ) 145: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 139 139: def asset_bucket(create_if_missing = true, permissions = :private) 140: @s3helper.bucket(:bucket => @plugin_config['bucket'], 141: :create_if_missing => create_if_missing, 142: :acl => permissions, 143: :location_constraint => @s3_endpoints[@plugin_config['region']][:location] 144: ) 145: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 147 147: def bundle_image(deliverables) 148: if @plugin_config['snapshot'] 149: @log.debug "Removing bundled image from local disk..." 150: FileUtils.rm_rf(@ami_build_dir) 151: end 152: 153: return if File.exists?(@ami_build_dir) 154: 155: @log.info "Bundling AMI..." 156: 157: FileUtils.mkdir_p(@ami_build_dir) 158: 159: cmd_str = "euca-bundle-image --ec2cert #{File.dirname(__FILE__)}/src/cert-ec2.pem " << 160: "-i #{deliverables[:disk]} " << 161: "--kernel #{@plugin_config['kernel'] || @s3_endpoints[@plugin_config['region']][:kernel][@appliance_config.hardware.base_arch.intern][:aki]} " << 162: "-c #{@plugin_config['cert_file']} "<< 163: "-k #{@plugin_config['key_file']} " << 164: "-u #{@plugin_config['account_number']} " << 165: "-r #{@appliance_config.hardware.base_arch} " << 166: "-d #{@ami_build_dir} " 167: 168: cmd_str << "--ramdisk #{@plugin_config['ramdisk']}" if @plugin_config['ramdisk'] 169: 170: @exec_helper.execute(cmd_str, :redacted => [@plugin_config['account_number'], @plugin_config['key_file'], @plugin_config['cert_file']]) 171: 172: @log.info "Bundling AMI finished." 173: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 147 147: def bundle_image(deliverables) 148: if @plugin_config['snapshot'] 149: @log.debug "Removing bundled image from local disk..." 150: FileUtils.rm_rf(@ami_build_dir) 151: end 152: 153: return if File.exists?(@ami_build_dir) 154: 155: @log.info "Bundling AMI..." 156: 157: FileUtils.mkdir_p(@ami_build_dir) 158: 159: cmd_str = "euca-bundle-image --ec2cert #{File.dirname(__FILE__)}/src/cert-ec2.pem " << 160: "-i #{deliverables[:disk]} " << 161: "--kernel #{@plugin_config['kernel'] || @s3_endpoints[@plugin_config['region']][:kernel][@appliance_config.hardware.base_arch.intern][:aki]} " << 162: "-c #{@plugin_config['cert_file']} "<< 163: "-k #{@plugin_config['key_file']} " << 164: "-u #{@plugin_config['account_number']} " << 165: "-r #{@appliance_config.hardware.base_arch} " << 166: "-d #{@ami_build_dir} " 167: 168: cmd_str << "--ramdisk #{@plugin_config['ramdisk']}" if @plugin_config['ramdisk'] 169: 170: @exec_helper.execute(cmd_str, :redacted => [@plugin_config['account_number'], @plugin_config['key_file'], @plugin_config['cert_file']]) 171: 172: @log.info "Bundling AMI finished." 173: end
US constraint is often represented as ’’ or nil
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 225 225: def constraint_equal?(a, b) 226: [a, b].collect!{|c| c.nil? ? '': c} 227: a == b 228: end
US constraint is often represented as ’’ or nil
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 225 225: def constraint_equal?(a, b) 226: [a, b].collect!{|c| c.nil? ? '': c} 227: a == b 228: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 191 191: def deregister_image(ami_manifest_key) 192: if ami = ami_by_manifest_key(ami_manifest_key) 193: @log.info "Preexisting image '#{ami.location}' for #{@appliance_config.name} will be de-registered, it had id: #{ami.id} (region: #{@plugin_config['region']})." 194: ami.deregister 195: @ec2helper.wait_for_image_death(ami) 196: else # This occurs when the AMI is de-registered externally but the file structure is left intact in S3. In this instance, we simply overwrite and register the image as if it were "new". 197: @log.debug "Possible dangling/unregistered AMI skeleton structure in S3, there is nothing to deregister" 198: end 199: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 191 191: def deregister_image(ami_manifest_key) 192: if ami = ami_by_manifest_key(ami_manifest_key) 193: @log.info "Preexisting image '#{ami.location}' for #{@appliance_config.name} will be de-registered, it had id: #{ami.id} (region: #{@plugin_config['region']})." 194: ami.deregister 195: @ec2helper.wait_for_image_death(ami) 196: else # This occurs when the AMI is de-registered externally but the file structure is left intact in S3. In this instance, we simply overwrite and register the image as if it were "new". 197: @log.debug "Possible dangling/unregistered AMI skeleton structure in S3, there is nothing to deregister" 198: end 199: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 85 85: def execute 86: case @type 87: when :s3 88: upload_to_bucket(@previous_deliverables) 89: when :cloudfront 90: upload_to_bucket(@previous_deliverables, :public_read) 91: when :ami 92: ami_dir = ami_key(@appliance_config.name, @plugin_config['path']) 93: ami_manifest_key = @s3helper.stub_s3obj(@bucket, "#{ami_dir}/#{@appliance_config.name}.ec2.manifest.xml") 94: 95: @log.debug "Going to check whether s3 object exists" 96: 97: if ami_manifest_key.exists? and @plugin_config['overwrite'] 98: @log.info "Object exists, attempting to deregister an existing image" 99: deregister_image(ami_manifest_key) # Remove existing image 100: @s3helper.delete_folder(@bucket, ami_dir) # Avoid triggering dupe detection 101: end 102: 103: if !ami_manifest_key.exists? or @plugin_config['snapshot'] 104: @log.info "Doing bundle/snapshot" 105: bundle_image(@previous_deliverables) 106: upload_image(ami_dir) 107: end 108: register_image(ami_manifest_key) 109: end 110: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 85 85: def execute 86: case @type 87: when :s3 88: upload_to_bucket(@previous_deliverables) 89: when :cloudfront 90: upload_to_bucket(@previous_deliverables, :public_read) 91: when :ami 92: ami_dir = ami_key(@appliance_config.name, @plugin_config['path']) 93: ami_manifest_key = @s3helper.stub_s3obj(@bucket, "#{ami_dir}/#{@appliance_config.name}.ec2.manifest.xml") 94: 95: @log.debug "Going to check whether s3 object exists" 96: 97: if ami_manifest_key.exists? and @plugin_config['overwrite'] 98: @log.info "Object exists, attempting to deregister an existing image" 99: deregister_image(ami_manifest_key) # Remove existing image 100: @s3helper.delete_folder(@bucket, ami_dir) # Avoid triggering dupe detection 101: end 102: 103: if !ami_manifest_key.exists? or @plugin_config['snapshot'] 104: @log.info "Doing bundle/snapshot" 105: bundle_image(@previous_deliverables) 106: upload_image(ami_dir) 107: end 108: register_image(ami_manifest_key) 109: end 110: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 181 181: def register_image(ami_manifest_key) 182: if ami = ami_by_manifest_key(ami_manifest_key) 183: @log.info "Image for #{@appliance_config.name} is already registered under id: #{ami.id} (region: #{@plugin_config['region']})." 184: else 185: ami = @ec2.images.create(:image_location => "#{@plugin_config['bucket']}/#{ami_manifest_key.key}") 186: @ec2helper.wait_for_image_state(:available, ami) 187: @log.info "Image for #{@appliance_config.name} successfully registered under id: #{ami.id} (region: #{@plugin_config['region']})." 188: end 189: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 181 181: def register_image(ami_manifest_key) 182: if ami = ami_by_manifest_key(ami_manifest_key) 183: @log.info "Image for #{@appliance_config.name} is already registered under id: #{ami.id} (region: #{@plugin_config['region']})." 184: else 185: ami = @ec2.images.create(:image_location => "#{@plugin_config['bucket']}/#{ami_manifest_key.key}") 186: @ec2helper.wait_for_image_state(:available, ami) 187: @log.info "Image for #{@appliance_config.name} successfully registered under id: #{ami.id} (region: #{@plugin_config['region']})." 188: end 189: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 175 175: def upload_image(ami_dir) 176: @log.info "Uploading #{@appliance_config.name} AMI to bucket '#{@plugin_config['bucket']}'..." 177: 178: @exec_helper.execute("euca-upload-bundle -U #{@plugin_config['url'].nil? ? "http://#{@s3_endpoints[@plugin_config['region']][:endpoint]}" : @plugin_config['url']} -b #{@plugin_config['bucket']}/#{ami_dir} -m #{@ami_manifest} -a #{@plugin_config['access_key']} -s #{@plugin_config['secret_access_key']}", :redacted => [@plugin_config['access_key'], @plugin_config['secret_access_key']]) 179: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 175 175: def upload_image(ami_dir) 176: @log.info "Uploading #{@appliance_config.name} AMI to bucket '#{@plugin_config['bucket']}'..." 177: 178: @exec_helper.execute("euca-upload-bundle -U #{@plugin_config['url'].nil? ? "http://#{@s3_endpoints[@plugin_config['region']][:endpoint]}" : @plugin_config['url']} -b #{@plugin_config['bucket']}/#{ami_dir} -m #{@ami_manifest} -a #{@plugin_config['access_key']} -s #{@plugin_config['secret_access_key']}", :redacted => [@plugin_config['access_key'], @plugin_config['secret_access_key']]) 179: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 112 112: def upload_to_bucket(previous_deliverables, permissions = :private) 113: register_deliverable( 114: :package => "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz" 115: ) 116: 117: # quick and dirty workaround to use @deliverables[:package] later in code 118: FileUtils.mv(@target_deliverables[:package], @deliverables[:package]) if File.exists?(@target_deliverables[:package]) 119: 120: PackageHelper.new(@config, @appliance_config, :log => @log, :exec_helper => @exec_helper).package(File.dirname(previous_deliverables[:disk]), @deliverables[:package]) 121: 122: remote_path = "#{@s3helper.parse_path(@plugin_config['path'])}#{File.basename(@deliverables[:package])}" 123: size_m = File.size(@deliverables[:package])/1024**2 124: s3_obj = @s3helper.stub_s3obj(@bucket,remote_path.gsub(/^\//, '').gsub(/\/\//, '')) 125: # Does it really exist? 126: obj_exists = s3_obj.exists? 127: 128: if !obj_exists or @plugin_config['overwrite'] 129: @log.info "Will overwrite existing file #{remote_path}" if obj_exists and @plugin_config['overwrite'] 130: @log.info "Uploading #{File.basename(@deliverables[:package])} (#{size_m}MB) to '#{@plugin_config['bucket']}#{remote_path}' path..." 131: s3_obj.write(:file => @deliverables[:package], 132: :acl => permissions) 133: @log.info "Appliance #{@appliance_config.name} uploaded to S3." 134: else 135: @log.info "File '#{@plugin_config['bucket']}#{remote_path}' already uploaded, skipping." 136: end 137: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 112 112: def upload_to_bucket(previous_deliverables, permissions = :private) 113: register_deliverable( 114: :package => "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz" 115: ) 116: 117: # quick and dirty workaround to use @deliverables[:package] later in code 118: FileUtils.mv(@target_deliverables[:package], @deliverables[:package]) if File.exists?(@target_deliverables[:package]) 119: 120: PackageHelper.new(@config, @appliance_config, :log => @log, :exec_helper => @exec_helper).package(File.dirname(previous_deliverables[:disk]), @deliverables[:package]) 121: 122: remote_path = "#{@s3helper.parse_path(@plugin_config['path'])}#{File.basename(@deliverables[:package])}" 123: size_m = File.size(@deliverables[:package])/1024**2 124: s3_obj = @s3helper.stub_s3obj(@bucket,remote_path.gsub(/^\//, '').gsub(/\/\//, '')) 125: # Does it really exist? 126: obj_exists = s3_obj.exists? 127: 128: if !obj_exists or @plugin_config['overwrite'] 129: @log.info "Will overwrite existing file #{remote_path}" if obj_exists and @plugin_config['overwrite'] 130: @log.info "Uploading #{File.basename(@deliverables[:package])} (#{size_m}MB) to '#{@plugin_config['bucket']}#{remote_path}' path..." 131: s3_obj.write(:file => @deliverables[:package], 132: :acl => permissions) 133: @log.info "Appliance #{@appliance_config.name} uploaded to S3." 134: else 135: @log.info "File '#{@plugin_config['bucket']}#{remote_path}' already uploaded, skipping." 136: end 137: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 40 40: def validate 41: set_default_config_value('overwrite', false) 42: set_default_config_value('kernel', false) 43: set_default_config_value('ramdisk', false) 44: set_default_config_value('path', '/') 45: set_default_config_value('region', 'us-east-1') 46: validate_plugin_config(['bucket', 'access_key', 'secret_access_key'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin') 47: 48: subtype(:ami) do 49: set_default_config_value('snapshot', false) 50: validate_plugin_config(['cert_file', 'key_file', 'account_number'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin') 51: 52: raise PluginValidationError, "AWS certificate file doesn't exists, please check the path: '#{@plugin_config['cert_file']}'." unless File.exists?(File.expand_path(@plugin_config['cert_file'])) 53: raise PluginValidationError, "AWS key file doesn't exists, please check the path: '#{@plugin_config['key_file']}'." unless File.exists?(File.expand_path(@plugin_config['key_file'])) 54: end 55: 56: @s3_endpoints = S3Helper::endpoints 57: raise PluginValidationError, "Invalid region specified: #{@plugin_config['region']}. This plugin is only aware of the following regions: #{@s3_endpoints.keys.join(", ")}." unless @s3_endpoints.has_key?(@plugin_config['region']) 58: 59: @plugin_config['account_number'] = @plugin_config['account_number'].to_s.gsub(/-/, '') 60: 61: # Set global AWS configuration 62: AWS.config(:access_key_id => @plugin_config['access_key'], 63: :secret_access_key => @plugin_config['secret_access_key'], 64: :ec2_endpoint => EC2Helper::endpoints[@plugin_config['region']][:endpoint], 65: :s3_endpoint => @s3_endpoints[@plugin_config['region']][:endpoint], 66: :max_retries => 5, 67: :use_ssl => @plugin_config['use_ssl']) 68: 69: @ec2 = AWS::EC2.new 70: @s3 = AWS::S3.new 71: @s3helper = S3Helper.new(@ec2, @s3, :log => @log) 72: @ec2helper = EC2Helper.new(@ec2, :log => @log) 73: 74: subtype(:ami) do 75: # If there is an existing bucket, determine whether its location_constraint matches the region selected 76: if existing_bucket = asset_bucket(false) 77: raise PluginValidationError, "Existing bucket #{@plugin_config['bucket']} has a location constraint that does not match the region selected. " << 78: "AMI region and bucket location constraint must match." unless constraint_equal?(@s3_endpoints[@plugin_config['region']][:location], existing_bucket.location_constraint) 79: end 80: end 81: 82: @bucket = asset_bucket(true) 83: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 40 40: def validate 41: set_default_config_value('overwrite', false) 42: set_default_config_value('kernel', false) 43: set_default_config_value('ramdisk', false) 44: set_default_config_value('path', '/') 45: set_default_config_value('region', 'us-east-1') 46: validate_plugin_config(['bucket', 'access_key', 'secret_access_key'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin') 47: 48: subtype(:ami) do 49: set_default_config_value('snapshot', false) 50: validate_plugin_config(['cert_file', 'key_file', 'account_number'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin') 51: 52: raise PluginValidationError, "AWS certificate file doesn't exists, please check the path: '#{@plugin_config['cert_file']}'." unless File.exists?(File.expand_path(@plugin_config['cert_file'])) 53: raise PluginValidationError, "AWS key file doesn't exists, please check the path: '#{@plugin_config['key_file']}'." unless File.exists?(File.expand_path(@plugin_config['key_file'])) 54: end 55: 56: @s3_endpoints = S3Helper::endpoints 57: raise PluginValidationError, "Invalid region specified: #{@plugin_config['region']}. This plugin is only aware of the following regions: #{@s3_endpoints.keys.join(", ")}." unless @s3_endpoints.has_key?(@plugin_config['region']) 58: 59: @plugin_config['account_number'] = @plugin_config['account_number'].to_s.gsub(/-/, '') 60: 61: # Set global AWS configuration 62: AWS.config(:access_key_id => @plugin_config['access_key'], 63: :secret_access_key => @plugin_config['secret_access_key'], 64: :ec2_endpoint => EC2Helper::endpoints[@plugin_config['region']][:endpoint], 65: :s3_endpoint => @s3_endpoints[@plugin_config['region']][:endpoint], 66: :max_retries => 5, 67: :use_ssl => @plugin_config['use_ssl']) 68: 69: @ec2 = AWS::EC2.new 70: @s3 = AWS::S3.new 71: @s3helper = S3Helper.new(@ec2, @s3, :log => @log) 72: @ec2helper = EC2Helper.new(@ec2, :log => @log) 73: 74: subtype(:ami) do 75: # If there is an existing bucket, determine whether its location_constraint matches the region selected 76: if existing_bucket = asset_bucket(false) 77: raise PluginValidationError, "Existing bucket #{@plugin_config['bucket']} has a location constraint that does not match the region selected. " << 78: "AMI region and bucket location constraint must match." unless constraint_equal?(@s3_endpoints[@plugin_config['region']][:location], existing_bucket.location_constraint) 79: end 80: end 81: 82: @bucket = asset_bucket(true) 83: end