newspaint

Documenting Problems That Were Difficult To Find The Answer To

Node.JS Utility To Launch Processes Remotely Using ssh2 Library

This Node.JS utility was written with the sole purpose of running many processes on remote servers and collating the output (stdout and stderr) line by line such that no two lines would get mixed.

It uses the ssh2 library available for Node.JS.

Note that this opens a single ssh connection to each target server and multiplexes the file transfers and command executions through channels within this single ssh connection. By default OpenSSH allowed 10 simultaneous channels in a ssh connection but versions 5.1 and subsequent introduced a MaxSessions parameter which allows this to be greatly increased (I’ve had it in the thousands).

// node.js SSH process launcher and line logger

var fs = require('fs');
var util = require('util');
var ssh2 = require('/usr/local/node/node_modules/ssh2');

/***
  help_and_exit() - print out help for command line, then exit
***/
function help_and_exit() {
    var helpstr = (
        "Usage:\n" +
        "    node launcher.js -s <ip> [-s <ip> ...] \\\n" +
        "        {-i <identity_key> or -w <password>}\\\n" +
        "        [-u <user{root}>] \\\n" +
        "        [-p <port{22}>] \\\n" +
        "        [-c <command_line>] \\\n" +
        "        [-n <num_of_commands>] \\\n" +
        "        [-g <num_of_seconds>] \\\n" +
        "        [-f <file> -t <target_file>] [-f <file> -t <target_file> ...] \\\n" +
        "        [-x <connect_timeout_seconds>] \\\n" +
        "        [-k <keyword_file>] \\\n" +
        "        [-r] \\\n" +
        "        [-d]\n" +
        "\n" +
        "Notes:\n" +
        "  -g <sec> - spread start of n commands over this many seconds\n" +
        "  -f <file> -t <target_file> - uploads specified in source/dest pairs\n" +
        "  -k <file> - keyword file consists of CSV lines, one line per command\n" +
        "  -c - string may use \${1} \${2} ... \${99} to replace from fields in keyword file\n" +
        "       \${time} - timestamp in milliseconds\n" +
        "       \${index} - unique number per command executed\n" +
        "  -r - randomise starting line from keyword file\n" +
        "\n"
    );

    console.log( "%s", helpstr );
    process.exit( 1 );
}

/***
  process_command_line() - handle command line arguments
***/
function process_command_line() {
    if ( process.argv.length < 3 ) {
        help_and_exit();
    }

    var servers = new Array();
    var identity_file = null;
    var user = "root";
    var password = null;
    var port = 22;
    var command = null;
    var command_count = 1;
    var command_spread = 1; // milliseconds to wait between launches
    var command_spread_period = null;
    var files = new Array();
    var debugging = false;
    var timeout_connect = null;
    var keyword_file = null;
    var keyword_line_randomise = false;

    var idx;
    for ( idx = 2; idx < process.argv.length; idx++ ) {
        if ( process.argv[idx] === "-s" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -s option" );
                help_and_exit();
            }
            servers.push( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-i" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -i option" );
                help_and_exit();
            }
            if ( ! fs.existsSync( process.argv[idx] ) ) {
                console.error( "Error, identity file \"%s\" does not exist", process.argv[idx] );
                help_and_exit();
            }
            identity_file = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-u" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -u option" );
                help_and_exit();
            }
            user = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-w" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -w option" );
                help_and_exit();
            }
            password = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-p" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -p option" );
                help_and_exit();
            }
            if (
                ( parseInt( process.argv[idx] ) < 1 ) ||
                ( parseInt( process.argv[idx] ) > 65535 )
            ) {
                console.error( "Invalid port number supplied to -p option" );
                help_and_exit();
            }
            port = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-c" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -c option" );
                help_and_exit();
            }
            command = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-n" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -n option" );
                help_and_exit();
            }
            if ( parseInt( process.argv[idx] ) < 1 ) {
                console.error( "Invalid command count supplied to -n option" );
                help_and_exit();
            }
            command_count = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-g" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -g option" );
                help_and_exit();
            }
            if ( parseInt( process.argv[idx] ) < 1 ) {
                console.error( "Invalid number of seconds supplied to -g option" );
                help_and_exit();
            }
            command_spread_period = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-f" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -f option" );
                help_and_exit();
            }
            if ( ! fs.existsSync( process.argv[idx] ) ) {
                console.error( "Error, source file \"%s\" does not exist", process.argv[idx] );
                help_and_exit();
            }
            var source_file = process.argv[idx];

            idx++;
            if ( process.argv[idx] !== "-t" ) {
                console.error( "Error, must provide -t option following -f option" );
                help_and_exit();
            }

            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -t option" );
                help_and_exit();
            }
            var target_file = process.argv[idx];

            var collection = { "source": source_file, "target": target_file };
            files.push( collection );

            continue;
        }

        if ( process.argv[idx] === "-d" ) {
            debugging = true;
            continue;
        }

        if ( process.argv[idx] === "-x" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -x option" );
                help_and_exit();
            }
            if ( parseInt( process.argv[idx] ) < 1 ) {
                console.error( "Invalid timeout supplied to -x option" );
                help_and_exit();
            }
            timeout_connect = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-k" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -k option" );
                help_and_exit();
            }
            if ( ! fs.existsSync( process.argv[idx] ) ) {
                console.error( "Error, keyword file \"%s\" does not exist", process.argv[idx] );
                help_and_exit();
            }
            keyword_file = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-r" ) {
            keyword_line_randomise = true;
            continue;
        }
    }

    if ( command_spread_period !== null ) {
        command_spread = parseInt( ( command_spread_period * 1000 ) / command_count );
    }

    if ( servers.length < 1 ) {
        console.error( "Must specify a minimum of one server" );
        help_and_exit();
    }

    if ( ( identity_file === null ) && ( password === null ) ) {
        console.error( "Must specify an identity file or login password" );
        help_and_exit();
    }

    var serverstates = { };
    for ( idx = 0; idx < servers.length; idx++ ) {
        serverstates[ servers[idx] ] = "initial";
    }

    var keyword_lines = null;
    if ( keyword_file ) {
        keyword_lines = [ ];

        // load whole file into memory (simplest)
        var keyword_buffer = fs.readFileSync( keyword_file, { encoding: 'utf8' } );
        // split into array of lines and words
        var regexpLine = new RegExp( "([^\r\n]+)[\r\n]+", "g" );
        while ( 1 ) {
            var mymatch = regexpLine.exec( keyword_buffer );
            if ( mymatch === null )
                break;

            var line = mymatch[1];

            if ( debugging ) {
                console.error( "  - processing line \"%s\"", line );
            }

            // process the line of text
            var words = [ ];
            while ( 1 ) {
                var next = line.indexOf( "," );
                if ( next < 0 ) {
                    words.push( line );
                    break;
                }

                words.push( line.substring( 0, next ) );
                line = line.substring( next + 1 );
            }

            keyword_lines.push( words );
        }
    }

    var keyword_line_number = 0;
    if ( keyword_line_randomise ) {
        keyword_line_number = Math.floor(Math.random() * keyword_lines.length);
    }

    var options = {
        "servers": serverstates,
        "identity": identity_file,
        "user": user,
        "password": password,
        "port": port,
        "command": command,
        "command_count": command_count,
        "command_spread": command_spread,
        "files": files,
        "debug": debugging,
        "timeout": timeout_connect,
        "keyword_file": keyword_file,
        "keyword_lines": keyword_lines,
        "keyword_line_number": keyword_line_number,
        "conns": { }
    };

    return( options );
}

/***
  log_server_state() - print out the states of all the servers
***/
function log_server_state( serverlist ) {
    var idx;
    for ( var key in serverlist ) {
        console.error( "    %s: %s", key, serverlist[key] );
    }
}

/***
  die_if_not_connected() - scan serverlist and die if any not connected
***/
function die_if_not_connected( serverlist ) {
    var idx;
    var connected = true;
    var disconnected_list = new Array();
    for ( var key in serverlist ) {
        if (
            ( serverlist[key] === "initial" ) ||
            ( serverlist[key] === "connected" ) ||
            ( serverlist[key] === "error" )
        ) {
            connected = false;
            disconnected_list.push( key );
        }
    }

    if ( connected )
        return;

    console.error( "ERROR: timeout, server%s failed to connect", ( disconnected_list.length === 1 ? "" : "s" ) );

    log_server_state( serverlist );

    process.exit( 2 );
}

/***
  interpret_command() - replace variables in command string with literals
***/
function interpret_command( options, index ) {
    var cmd = options.command;

    if ( options.keyword_lines ) {
        if ( options.keyword_line_number >= options.keyword_lines.length )
            options.keyword_line_number = 0;

        var words = options.keyword_lines[options.keyword_line_number];

        options.keyword_line_number++;

        var find_substitution = new RegExp( "[$][{]([0-9]+)[}]" );
        while ( 1 ) {
            var result = find_substitution.exec( cmd );
            if ( ! result )
                break;

            var start = result.index;
            var end = start + result[0].length;

            var wanted_idx = result[1] - 1;
            var substitution;
            if ( wanted_idx >= words.length ) {
                substitution = "";
            } else {
                substitution = words[wanted_idx];
            }

            cmd = cmd.substring( 0, start ) + substitution + cmd.substring( end );
        }

        var find_time = new RegExp( "[$][{](time)[}]" );
        while ( 1 ) {
            var result = find_time.exec( cmd );
            if ( ! result )
                break;

            var start = result.index;
            var end = start + result[0].length;

            var substitution = (new Date()).getTime();

            cmd = cmd.substring( 0, start ) + substitution + cmd.substring( end );
        }

        var find_index = new RegExp( "[$][{](index)[}]" );
        while ( 1 ) {
            var result = find_index.exec( cmd );
            if ( ! result )
                break;

            var start = result.index;
            var end = start + result[0].length;

            var substitution = index;

            cmd = cmd.substring( 0, start ) + substitution + cmd.substring( end );
        }
    }

    return( cmd );
}

/***
  ip2num() - convert dotted IP address to integer
***/
function ip2num( ip ) {
    var elements = ip.split( '.' );
    var result = (
        parseInt( elements[0] ) * 256 * 256 * 256 +
        parseInt( elements[1] ) * 256 * 256 +
        parseInt( elements[2] ) * 256 +
        parseInt( elements[3] )
    );

    return( result );
}

/***
   getconnidx() - get index of this connection in array of connections
***/
function getconnidx( options, conn ) {
    var idx = 0;
    for ( var tmpip in options.conns ) {
        if ( conn == options.conns[tmpip] ) {
            return idx;
        }
        idx++;
    }

    return( undefined );
}

/***
  do_exec() - execute command via established SSH connection
***/
function do_exec( conn, options, ip, instance_num ) {
    var uniqueindex = getconnidx( options, conn ) * options.command_count + instance_num;

    var cmdline = interpret_command( options, uniqueindex );

    if ( options.debug ) {
        console.error( "  - command line is \"%s\"", cmdline );
    }

    conn.exec(
        cmdline,
        function ( err, stream ) {
            if ( err ) {
                console.error( "[%s]/%d Error executing \"%s\": %s", ip, instance_num, cmdline, err );
                options.servers[ip] = "error";
                options.command_state[ip][instance_num] = "error";
                check_states( options );
                return;
            } else {
                options.command_state[ip][instance_num] = "exec";
            }

            var buffer_stdout = "";
            var buffer_stderr = "";

            var regexpLine = new RegExp( "([^\r\n]+)[\r\n]+" );

            stream.on(
                'data',
                function (data, extended) {
                    if ( extended === 'stderr' ) {
                        buffer_stderr += data;
                    } else {
                        buffer_stdout += data;
                    }

                    if ( options.debug ) {
                        console.error( "[%s]/%d Data received = %d", ip, instance_num, data.length );
                    }

                    while ( 1 ) {
                        var mymatch = regexpLine.exec( buffer_stderr );
                        if ( mymatch == null )
                            break;

                        var start = mymatch.index;
                        var end = start + mymatch[0].length;
                        var line = mymatch[1];

                        buffer_stderr = buffer_stderr.substring( end );

                        console.log( "[%s]/%d stderr: %s", ip, instance_num, line );
                    }

                    while ( 1 ) {
                        var mymatch = regexpLine.exec( buffer_stdout );
                        if ( mymatch == null )
                            break;

                        var start = mymatch.index;
                        var end = start + mymatch[0].length;
                        var line = mymatch[1];

                        buffer_stdout = buffer_stdout.substring( end );

                        console.log( "[%s]/%d stdout: %s", ip, instance_num, line );
                    }
                }
            );

            stream.on(
                'end',
                function () {
                    if ( options.debug ) {
                        console.error( "[%s]/%d stream end", ip, instance_num );
                    }
                    options.command_state[ip][instance_num] = "completed";
                    check_states( options );
                }
            );

            stream.on(
                'close',
                function () {
                    if ( options.debug ) {
                        console.error( "[%s]/%d stream closed", ip, instance_num );
                    }
                }
            );
            stream.on(
                'exit',
                function (retval, signalName, didCoreDump, description) {
                    if ( retval != null ) {
                        if ( options.debug ) {
                            console.error( "[%s]/%d stream exit = %d", ip, instance_num, retval );
                        }
                        stream.end();
                    } else {
                        if ( options.debug ) {
                            console.error( "[%s]/%d stream exit signal %s%s [%s]", ip, instance_num, signalName, didCoreDump ? " (dumped core)" : "", description );
                        }
                    }
                }
            );
        }
    );
}

/***
  check_states() - state machine engine, checking states, calling next action when ready
***/
function check_states( options ) {
    var all_done = true;

    if ( options.debug ) {
        for ( var sip in options.servers ) {
            console.error( "  %s: %s", sip, options.servers[sip] );
        }
    }

    if ( options.command_state ) {
        for ( var ip in options.command_state ) {
            if (
                ( options.servers[ip] === "closed" ) ||
                ( options.servers[ip] === "error" )
            ) {
                continue;
            }

            if ( options.debug ) {
                var strstate = "";
                var statectr = 0;
                for ( statectr = 0; statectr < options.command_state[ip].length; statectr++ ) {
                    if ( strstate.length != 0 ) {
                        strstate += ", ";
                    }
                    strstate += "\n  " + statectr + ":" + options.command_state[ip][statectr];
                }
                console.error( "[%s] command_state = %s", ip, strstate );
            }

            var all_completed = true;
            var instance = 0;
            for ( inst = 0; inst < options.command_state[ip].length; inst++ ) {
                var state = options.command_state[ip][inst];
                if (
                    ( state === "error" ) ||
                    ( state === "completed" )
                ) {
                    ;
                } else {
                    all_completed = false;
                }
            }

            if ( all_completed ) {
                options.conns[ip].end();
                options.servers[ip] = "closed";
            }
        }
    }

    for ( var ip in options.servers ) {
        if (
            ( options.servers[ip] === "error" ) ||
            ( options.servers[ip] === "closed" )
        ) {
            ;
        } else {
            all_done = false;
        }
    }

    if ( all_done ) {
        if ( options.debug ) {
            console.error( "- all completed" );
        }
        process.exit( 0 );
    }

    var all_awaiting_cmd = true;
    var any_closed = false;
    var any_awaiting = false;

    for ( var ip in options.servers ) {
        if (
            ( options.servers[ip] === "awaiting_cmd" )
        ) {
            any_awaiting = true;
        } else {
            all_awaiting_cmd = false;
            if (
                ( options.servers[ip] === "closed" ) ||
                ( options.servers[ip] === "error" )
            ) {
                // connection closed before "awaiting_command"
                any_closed = true;
            }
        }
    }

    if ( all_awaiting_cmd ) {
        options["command_state"] = { };
        for ( var ip in options.servers ) {
            options.servers[ip] = "executing";

            if ( options.command != null ) {
                var conn = options.conns[ip];
                if ( ! conn ) {
                    console.error( "Programmer error, conn should exist" );
                    process.exit( 98 );
                }

                // execute command number of times
                var cmdidx;
                options.command_state[ip] = new Array( options.command_count );
                for ( cmdidx = 0; cmdidx < options.command_count; cmdidx++ ) {
                    options.command_state[ip][cmdidx] = "initial";
                }
                var timeoffset = 0;
                for ( cmdidx = 0; cmdidx < options.command_count; cmdidx++ ) {
                    setTimeout(
                        function (conn, options, ip, cmdidx) {
                            do_exec( conn, options, ip, cmdidx );
                        },
                        timeoffset,
                        conn, options, ip, cmdidx
                    );
                    timeoffset += options.command_spread;
                }
            } else {
                options.servers[ip] = "closed";
                check_states( options );
            }
        }
    } else if ( any_closed && any_awaiting ) {
        console.error( "Error, connection has been prematurely closed" );
        process.exit( 2 );
    }
}

/***
  do_sftp() - perform file upload via established SSH connection
***/
function do_sftp( conn, options, ip, filesremaining, callback ) {
    if ( options.debug ) {
        console.error( "[%s] doing sftp %d", ip, filesremaining.length );
    }

    if ( filesremaining.length == 0 ) {
        callback();
        return;
    }

    var nextfile = filesremaining.shift();
    if ( options.debug ) {
        console.error( "[%s] next file is %s", ip, nextfile.source );
    }

    conn.sftp(
        function (err, sftp) {
            if ( err ) {
                console.error( "[%s] Error, problem starting SFTP: %s", ip, err );
                process.exit( 3 );
            }

            if ( options.debug ) {
                console.error( "[%s] SFTP started", ip );
            }

            // start upload
            var readStream = fs.createReadStream( nextfile.source );
            var writeStream = sftp.createWriteStream(
                nextfile.target,
                { flags: 'w', encoding: null, mode: 0666 }
            );

            writeStream.on(
                'close',
                function () {
                    if ( options.debug ) {
                        console.error( "[%s] end transfer of \"%s\"", ip, nextfile.source );
                    }
                    sftp.end();
                    do_sftp( conn, options, ip, filesremaining, callback );
                }
            );

            readStream.pipe( writeStream );
        }
    );
}

/***
  do_ready_action_awaiting_cmd() - change state to waiting for cmd and check state machine
***/
function do_ready_action_awaiting_cmd( conn, options, ip ) {
    options.servers[ip] = "awaiting_cmd";
    check_states( options );
}

/***
 do_ready_action_upload_files() - upload files if required
***/
function do_ready_action_upload_files( conn, options, ip ) {
    if ( options.files.length > 0 ) {
        var filesremaining = options.files.slice( 0 );

        do_sftp(
            conn, options, ip, filesremaining,
            function () {
                do_ready_action_awaiting_cmd( conn, options, ip );
            }
        );
    } else {
        // no files to transfer, change state
        do_ready_action_awaiting_cmd( conn, options, ip );
    }
}

/***
  connect_to_server() - set up a connection to a server
***/
function connect_to_server( options, ip ) {
    var conn = new ssh2();

    conn.setMaxListeners(0);

    conn.on(
        'connect',
        function () {
            if ( options.debug ) {
                console.error( "[%s] connect", ip );
            }
            options.servers[ip] = "connected";
        }
    );

    conn.on(
        'ready',
        function () {
            if ( options.debug ) {
                console.error( "[%s] ready", ip );
            }
            options.servers[ip] = "ready";
            options.conns[ip] = conn;

            do_ready_action_upload_files(
                conn, options, ip
            );
        }
    );

    conn.on(
        'error',
        function (err) {
            if ( options.debug ) {
                console.error( "[%s] error: %s", ip, err );
            }
            options.servers[ip] = "error";
            check_states( options );
        }
    );

    conn.on(
        'end',
        function () {
            if ( options.debug ) {
                console.error( "[%s] end", ip );
            }
            options.servers[ip] = "closed";
            check_states( options );
        }
    );

    conn.on(
        'close',
        function (had_error) {
            if ( options.debug ) {
                console.error( "[%s] end: %s", ip, had_error );
            }
            options.servers[ip] = "closed";
            check_states( options );
        }
    );

    var connectoptions = {
        "host": ip,
        "port": options.port,
        "username": options.user
    };

    if ( options.identity !== null ) {
        connectoptions["privateKey"] = fs.readFileSync( options.identity );
    } else if ( options.password !== null ) {
        connectoptions["password"] = options.password;
    } else {
        console.error( "No authentication option for SSH provided" );
        process.exit( 1 );
    }

    conn.connect( connectoptions );
}

/***
  connect_to_servers() - connect to all the servers
***/
function connect_to_servers( options ) {
    var idx;
    for ( var ip in options.servers ) {
        connect_to_server( options, ip );
    }
}

/***
  main() - process command line, then connect to servers
***/
function main() {
    var options = process_command_line();

    // establish ssh connections to all servers, wait until this is achieved
    if ( options.debug ) {
        console.error( "options = %s", util.inspect( options ) );
    }

    // timeout to blow up if connections haven't been established
    if ( options.timeout ) {
        setTimeout(
            function () {
                die_if_not_connected( options.servers );
            },
            options.timeout * 1000
        );
    }

    // establish connection to servers
    connect_to_servers( options );
}

main();

Example Usage

One could have a local jar file they wish to run remotely on four servers. And they want to run 20 instances of this on each server but with the instances started spread out over 60 seconds. In total this means that 80 instances of the program will be started spread out over 60 seconds.

The jar file exists in ~/myjar.jar on the local server and before executing this jar file it must be uploaded to /tmp/ on the remote servers. Command execution won’t begin until the file has been successfully transferred to all the remote servers.

/usr/local/node/bin/node sshlauncher.js \
  -s test_server1.my.net \
  -s test_server2.my.net \
  -s test_server3.my.net \
  -s test_server4.my.net \
  -u testuser \
  -w testpassword \
  -x 15 \
  -g 60 \
  -n 20 \
  -f ~/myjar.jar -t /tmp/myjar.jar \
  -c 'java -jar /tmp/myjar.jar'

Debugging mode can be enabled with a -d flag so you can see when connections take place and file transfers, etc.

You may wish to provide a text comma-separated value file so that you can provide unique or different parameters to your executed command instances (e.g. for testing login processes to a site).

If you wanted to launch 4 instances of a program with unique usernames you could have a file named ~/keywords.txt containing:

user1,password1
user2,password2
user3,password3
user4,password4

and a command line of:

/usr/local/node/bin/node sshlauncher.js \
  -s test_server1.my.net \
  -s test_server2.my.net \
  -s test_server3.my.net \
  -s test_server4.my.net \
  -u testuser \
  -w testpassword \
  -x 15 \
  -g 60 \
  -n 20 \
  -f ~/myjar.jar -t /tmp/myjar.jar \
  -c 'java -jar /tmp/myjar.jar ${1} ${2}' \
  -k ~/keywords.txt

If your CSV (comma-separated value) keyword file is long enough (say, a file full of dictionary words) you can provide the -r flag to start reading from a random line in the file (e.g. when trying to test a cached server and you want to feed it unique words to circumvent caching).

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s

%d bloggers like this: