display ec2-name and ip-address as ssh-config or hosts file

Inspired by this instagram engineering blog, I wrote a python script to printout ec2 hostname and ip-address as ssh-config or hosts file format.

In our environment, unfortunately we don't have dns for ec2 hosts, for now. 

I use /etc/hosts. Some people use ssh-config file.

This script may help others who are in similar environment where using ssh-config or hosts file to manage ec2-host mapping.

Usage

	# display entries as ssh-config file format
	./ec2-printconfig -k <aws_key> -s <aws_secret> -r us-west-2
	# display entries as hosts file format
	./ec2-printconfig --format hosts
	# add prefix "my-" to instance name. (ex: my-foo, my-bar, ...)
	./ec2-printconfig --prefix my-
	# add domain "mydomain.com" to instance name. (ex: foo.mydomain.com, bar.mydomain.com, ...)
	./ec2-printconfig --domain mydomain.com
	# use elastic IP instead of private IP
	./ec2-printconfig --use-elastic-ip
	# filter instances. (ex: only running and monitored instances. )
	./ec2-printconfig --filter monitoring-state=enabled --filter instance-state-name=running
	# specify tag for ec2 instance name. (ex: assume there is "alias" tag)
	./ec2-printconfig -t alias

view raw usage hosted with ❤ by GitHub

ec2-printconfig script

	#! /usr/bin/env python
	import argparse
	import os
	import sys
	import boto.ec2
	def parse_args():
	parser = argparse.ArgumentParser(
	description='Display EC2 hosts in ssh-config or hosts file format',
	formatter_class=argparse.ArgumentDefaultsHelpFormatter)
	parser.add_argument('-k', '--aws-key', type=str,
	default=os.environ.get("AWS_ACCESS_KEY_ID"),
	help='Amazon EC2 Key, defaults to ENV[AWS_ACCESS_KEY_ID]')
	parser.add_argument('-s', '--aws-secret', type=str,
	default=os.environ.get("AWS_SECRET_ACCESS_KEY"),
	help='Amazon EC2 Secret, defaults to ENV[AWS_SECRET_ACCESS_KEY]')
	parser.add_argument('-r', '--region', type=str,
	default=os.environ.get("AWS_EC2_REGION"),
	help='Amazon EC2 Region, defaults to us-east-1 or ENV[AWS_EC2_REGION]')
	parser.add_argument('-t', '--tag', type=str,
	default='Name', dest='name_tag',
	help="Amazon EC2 Tag for instance name, defaults to 'Name'")
	parser.add_argument('--prefix', type=str, default='',
	help='prefix to the instance name')
	parser.add_argument('--suffix', type=str, default='',
	help='suffix to the instance name')
	parser.add_argument('--domain', type=str, default='',
	help='domain to the instance name')
	parser.add_argument('--format', choices=['ssh', 'hosts'], default='ssh',
	help='output format. ssh-config or hosts file style. ')
	parser.add_argument('--filter', type=str, action='append', default=[],
	help=('Amazon EC2 API filter to limit the result returned. '
	'(Example: --filter instance-state-name=running)'))
	parser.add_argument('--use-elastic-ip', action='store_true',
	help='use elastic IP instead of private IP')
	return parser.parse_args()
	def main(opts):
	aws_key = opts['aws_key']
	aws_secret = opts['aws_secret']
	region = opts['region']
	name_tag = opts['name_tag']
	prefix = opts['prefix']
	suffix = opts['suffix']
	domain = opts['domain']
	format = opts['format']
	filter = opts['filter']
	use_elastic_ip = opts['use_elastic_ip']
	filters = dict([f.split('=', 1) for f in filter])
	if domain and not domain.startswith('.'):
	domain = '.' + domain
	ip_addr_attr = 'ip_address' if use_elastic_ip else 'private_ip_address'
	# validation
	if not aws_key or not aws_secret:
	if not aws_key:
	print >> sys.stderr,\
	"AWS_ACCESS_KEY_ID not set in environment and not",\
	"specified by -k AWS_KEY or --aws-key AWS_KEY"
	if not aws_secret:
	print >> sys.stderr,\
	"AWS_SECRET_ACCESS_KEY not set in envoronment and not",\
	"specified by -s AWS_SECRET or --aws-secret AWS_SECRET"
	sys.exit(2)
	region = region and boto.ec2.get_region(region,
	aws_access_key_id=aws_key,
	aws_secret_access_key=aws_secret)
	conn = boto.ec2.connection.EC2Connection(aws_key, aws_secret,
	region=region)
	# list of (instance_name, ip_address)
	instances = get_ec2_instances(conn, name_tag, ip_addr_attr, filters)
	# sort by name
	instances = sorted(instances)
	# print out
	print_fn = print_ssh_config if format == 'ssh' else print_hosts
	print_fn(instances, prefix, suffix, domain)
	def get_ec2_instances(conn, name_tag, ip_addr_attr, filters):
	instances = [] # (instance_name, ip_address)
	reservations = conn.get_all_instances(filters=filters)
	for reservation in reservations:
	for instance in reservation.instances:
	instance_name = instance.tags.get(name_tag)
	ip_address = getattr(instance, ip_addr_attr)
	if instance_name and ip_address:
	pair = (instance_name, ip_address)
	instances.append(pair)
	return instances
	def print_ssh_config(instances, prefix, suffix, domain):
	""" Print out as ssh-config file format """
	for instance_name, ip_address in instances:
	instance_name = prefix + instance_name + suffix + domain
	# double quote if name contains space
	instance_name = '"{0}"'.format(
	instance_name) if ' ' in instance_name else instance_name
	print "Host %s" % instance_name
	print "Hostname %s" % ip_address
	print ""
	def print_hosts(instances, prefix, suffix, domain):
	""" Print out as hosts file format """
	for instance_name, ip_address in instances:
	if ' ' in instance_name:
	continue # skip if name contains space.
	instance_name = prefix + instance_name + suffix + domain
	print "%s\t%s" % (ip_address, instance_name)
	if __name__ == '__main__':
	args = vars(parse_args())
	main(args)

view raw ec2-printconfig.py hosted with ❤ by GitHub

util method that runs TestNG test classes and verify the result

I wrote a utility class for my project.

The method does:

• run TestNG tests programmatically
• verify the failures happened in TestNG tests run
• If there is a failure, combine all assertion messages including stack trace, then make a single AssertionError

(I'm also using google-guava)

	public class TestNGUtils {
	private static final String LINE_SEPARATOR = System.getProperty("line.separator");
	public static void runAndVerify(Class<?>... testClasses) {
	if (ObjectUtils.isEmpty(testClasses)) {
	throw new AssertionError("no test class specified.");
	}
	// run testng tests
	TestListenerAdapter tla = new TestListenerAdapter();
	TestNG testNG = new TestNG();
	testNG.setTestClasses(testClasses);
	testNG.addListener(tla);
	testNG.run();
	List<ITestResult> failedTests = Lists.newArrayList();
	failedTests.addAll(tla.getFailedTests());
	failedTests.addAll(tla.getConfigurationFailures());
	if (!failedTests.isEmpty()) {
	String header = String.format("Combined Messages (Total:%d)", failedTests.size());
	List<String> errorMessages = Lists.newArrayList();
	errorMessages.add(header);
	errorMessages.addAll(Lists.transform(failedTests, new Function<ITestResult, String>() {
	int i = 1;
	@Override
	public String apply(ITestResult testResult) {
	String stackTraceString = Throwables.getStackTraceAsString(testResult.getThrowable());
	String template = "Message-%d: %n %s";
	return String.format(template, i++, stackTraceString);
	}
	}));
	// transform messages to a single combined string
	String message = Joiner.on(LINE_SEPARATOR).join(errorMessages);
	throw new AssertionError(message);
	}
	}
	}

view raw TestNGUtils.java hosted with ❤ by GitHub

run jetty on dependency resolved war in gradle

Background:

• test classes(such as selenium tests) are in an independent module(maven project, etc.).
• have an external dependency(mave, ivy, etc) to war file(s)
• want to run jetty on dependent war file(s)

Gradle can start up jetty on dependency resolved war file(s).

	// Run Jetty on dependency resolved war file
	//
	// you can run multiple wars with different configuration by
	// adding more task definitions.
	//
	import org.gradle.api.plugins.jetty.JettyRunWar
	apply plugin: 'java'
	apply plugin: 'jetty'
	repositories {
	mavenCentral()
	}
	configurations {
	// define custom dependency configuration for war file
	myWebApp
	}
	dependencies {
	// add war file dependency to my custom configuration
	myWebApp "org.some:my-app:1.0@war"
	}
	task runMyWebApp(type: JettyRunWar) {
	// specify the resolved war file
	webApp = configurations.myWebApp.singleFile
	// more configuration
	httpPort = 8001
	contextPath = 'myWebApp'
	}

view raw build.gradle hosted with ❤ by GitHub

When you want to run your tests after jetty started, set "daemon=true" and make the test task depends on your new run-war task.

run multiple jetty in gradle

Background:
I have a selenium test suite for end-to-end web testing.
Currently, it is running on jenkins box but it is taking so long to finish all the tests.
So, next step is to parallelize the selenium tests.
As a first step, I need to start multiple jetty servers using different port.

With Gradle jetty plugin, I camp up how to run the multiple jetty servers before tests.

	// start multiple jetties before test
	//
	// useful for parallel web end-to-end testing
	//
	import org.gradle.api.plugins.jetty.JettyRun
	apply plugin: 'java'
	apply plugin: 'jetty'
	repositories {
	mavenCentral()
	}
	// configuration for each jetty server
	def jettyConfigs = [
	[port:8001],
	[port:8002]
	]
	def jettyMultiRunTasks = []
	// create jetty tasks and assign them
	jettyConfigs.eachWithIndex {jettyConfig, index ->
	jettyMultiRunTasks << task("jettyMultiRun-${index}", type: JettyRun) {
	httpPort = jettyConfig.port
	daemon = true
	// more configuration to specify each jetty to use different env...
	}
	}
	test {
	// start jetties before running tests
	dependsOn(jettyMultiRunTasks)
	// more test options such as maxParallelForks, etc.
	}

view raw build.gradle hosted with ❤ by GitHub