newspaint

Documenting Problems That Were Difficult To Find The Answer To

Category Archives: NodeJS

Getting Statistics from HG612 Revision 3B VDSL Modem Using Node

The HG612 Revision 3B VDSL modem commonly provided by BT OpenReach in the United Kingdom for fibre-to-the-cabinet (FTTC) connected households can be unlocked with alternative firmware.

For those modems that are unlocked they can be connected to on the modem’s default IP address of 192.168.1.1 with a username of admin, password of admin, using web (if enabled), telnet, and ssh.

Such a session entails getting into the shell, then running a Linux command to show the DSL statistics, like the following:

me@home:~$ ssh admin@192.168.1.1
admin@192.168.1.1's password: admin
PTY allocation request failed on channel 0

ATP>sh
sh


BusyBox v1.9.1 (2010-10-15 17:59:06 CST) built-in shell (ash)
Enter 'help' for a list of built-in commands.

# xdslcmd info --stats
xdslcmd: ADSL driver and PHY status
Status: Showtime
Retrain Reason: 0
Max:    Upstream rate = 30992 Kbps, Downstream rate = 90516 Kbps
Path:   0, Upstream rate = 20000 Kbps, Downstream rate = 79999 Kbps
...
# exit
exit

ATP>exit
exit
exit from configuration console.
[6408] Jan 03 23:45:49 exit after auth (admin): child failed
Connection to 192.168.1.1 closed.

I wanted a way of dumping this information on a regular basis automatically but it wasn’t as simple as merely executing a command from the Linux command line, e.g. ssh admin@192.168.1.1 “sh\nxdslcmd info –stats\nexit\nexit”.

So I wrote the following Node.JS script to open a ssh connection to the modem, and execute the appropriate instructions, and dump the output from the xdslcmd command.

'use strict';

/*

  You will need to run the following in the directory you place this script:
    npm install ssh2
    npm install bluebird

*/

var Promise = require("bluebird");
var ssh2 = require("ssh2");

var modem_ip = "192.168.1.1";
var modem_user = "admin";
var modem_pass = "admin";

var debugging = false;

function debug( msg ) {
  if ( debugging ) {
    console.log( msg );
  }
}

function connect() {
  return new Promise( function( resolve, reject ) {
    var conn = new ssh2();

    conn.setMaxListeners(0);

    conn.on(
      'connect',
      function () {
        debug( "ssh connect" );
      }
    );

    conn.on(
      'ready',
      function () {
        debug( "ssh ready" );

        resolve( conn );
        return;
      }
    );

    conn.on(
      'error',
      function (err) {
        debug( "error: " + err );

        reject();
        return;
      }
    );

    conn.on(
      'end',
      function () {
        debug( "end" );

        //reject();
      }
    );

    conn.on(
      'close',
      function (had_error) {
        debug( "close: " + had_error );

        reject();
        return;
      }
    );

    conn.connect(
      {
        "host": modem_ip,
        "port": 22,
        "username": modem_user,
        "password": modem_pass,
        "debug": function ( msg ) { debug( msg ); },
        "algorithms": {
          "kex": [
            'ecdh-sha2-nistp256',
            'ecdh-sha2-nistp384',
            'ecdh-sha2-nistp521',
            'diffie-hellman-group-exchange-sha256',
            'diffie-hellman-group14-sha1',
            'diffie-hellman-group1-sha1', /* required for modem */
          ],
          "cipher": [
            'aes128-ctr',
            'aes192-ctr',
            'aes256-ctr',
            'aes128-gcm',
            'aes128-gcm@openssh.com',
            'aes256-gcm',
            'aes256-gcm@openssh.com',
            '3des-cbc', /* required for modem */
          ],
        },
      }
    );

    debug( "Starting..." );
  } );
}

function interactive( conn ) {
  return new Promise( function( resolve, reject ) {
    debug( "Starting shell..." );
    conn.exec( '',
      function( err, stream ) {
        if ( err ) {
          reject( err );
          return;
        }

        var data = "";

        stream.on(
          'close',
          function( code, signal ) {
            debug( "close stream " + code + " " + signal );
            conn.end();
            resolve( data );
            return;
          }
        );

        /* use a state machine */
        var stage = "ATPprompt";

        stream.on(
          'data',
          function ( chunk ) {
            debug( "Chunk: " + chunk );
            data += chunk;

            /* determine state we are in and if expected prompt is seen */
            if ( ( stage == "ATPprompt" ) && /^ATP>/m.test( data ) ) {
              debug( "writing sh" );
              data = ""; /* clear buffer */
              stream.write( "sh\n" );
              stage = "shprompt";
              return;
            }

            if ( ( stage == "shprompt" ) && /^#/m.test( data ) ) {
              debug( "writing xdslcmd info --stats" );
              data = ""; /* clear buffer */
              stream.write( "xdslcmd info --stats\n" );
              stage = "shprompt2";
              return;
            }

            if ( ( stage == "shprompt2" ) && /^#/m.test( data ) ) {
              debug( "writing exit" );
              console.log( data );
              data = ""; /* clear buffer */
              stream.write( "exit\n" );
              stage = "ATPprompt2";
              return;
            }

            if ( ( stage == "ATPprompt2" ) && /^ATP>/m.test( data ) ) {
              debug( "writing exit" );
              data = ""; /* clear buffer */
              stream.write( "exit\n" );
              stage = "none";

              /* we now expect connection to close, so resolve when we do */
              conn.on(
                'close',
                function (had_error) {
                  resolve();
                  return;
                }
              );

              return;
            }
          }
        );
      }
    );
  } );
}

/* set out abort timer in case something goes wrong */
setTimeout(
  function () {
    console.error( "ERROR: Timeout" );
    process.exit( 1 );
  },
  5000 /* 5.000 seconds */
);

/* main routine as a chain of promises */
connect().then(
  function ( conn ) {
    return interactive( conn );
  }
).then(
  function () {
    debug( "Done." );
    process.exit();
  }
).catch(
  function ( err ) {
    debug( "Failed to connect." );
    process.exit();
  }
);

Which can be run simply as node xdslcmd.js.

Thread Pool Pattern for Node.JS using Q.js Promises

I wanted to emulate the Thread Pool Pattern in Node.JS using Q.js promises.

Of course Node.JS does not have multiple threads for the user to use. Which is why Q.js is used in the first place – to make concurrency programming easier using the event/callback model that Node.JS implements.

Given an array of functions which return promises I wanted a pattern function that provided a customisable n number of workers. These workers would run asynchronously processing one function from the list at a time, and when each returned promise resolved it would then fetch the next function from the list – until no functions were left to process. When the last worker finished processing it would then resolve the promise returned by the pattern function to the caller. Usually there would be many tasks and limited number of workers.

Why would I want to do this? Let’s say I have a hundred or thousand URLs I want to fetch. If I put them all into a list and called Q.allSettled then Node.JS would attempt to fetch them all at the same time – this could result in excessive delays, a peak in network traffic, or might be plain impolite on the target webserver. Instead it may be better to download a maximum of 4 web pages at any one time.

For simplicity’s sake here is an example. Let’s create a function that returns a promise:

// delayms() - returns a promise that resolves approximately n milliseconds into the future
//   - ms - milliseconds to wait before resolving promise
function delayms( ms ) {
  var deferred = Q.defer();

  setTimeout( function () { deferred.resolve( "Finished after " + ms + "ms" ); }, ms);
  return deferred.promise;
}

Now let’s create a list of functions that return a promise (we could call these functions “tasks”). It’s important that the array contains functions, not the result of a function that returns a promise (i.e. a promise).

For example:

// create array of functions to process
var tasklist = [ ];
for ( var item = 0; item < 12; item++ ) {
  tasklist.push(
    function () {
      console.log( "Going to wait 1 second..." );
      return delayms( 1000 ); // return promise as last statement
    }
  );
}

// let's throw in an error, too
tasklist.push(
  function () {
    throw new Error( "Deliberate error!" );
  }
);

Now we have an array of tasks to process. How about we process them all – but a maximum of 3 running at any one time? We would want to do something like:

var Q = require('./q.js'); // load Q.js library

Q.fcall(
  // call a function, this function resolves immediately
  function () {
    console.log( "Starting" );
  }
).then(
  // do the list of tasks, max concurrency of 3
  function () {
    // return the promise
    return workermodel( 3, tasks );
  }
).then(
  // output the list of results
  function ( results ) {
    console.log( "End: " + JSON.stringify( results, null, 2 ) );
  }
).fail(
  function ( reason ) {
    console.error( "Error: " + reason );
  }
).done();

Well all we need now is the workermodel() function to run through our list of tasks:

// workermodel() - returns a promise that resolves after task list processes with max concurrency
//   - workers - maximum concurrency or number of workers
//   - tasklist - array of functions that return a promise
var debug = true;
function workermodel( workers, tasklist ) {
  var taskidx = 0;
  var workersfree = 0;

  // if no tasks to perform return a completed promise with empty array
  if ( tasklist.length == 0 ) {
    return Q.fcall( function () { return []; } );
  }

  var deferred = Q.defer();

  // if less tasks than workers, limit workers to task size
  if ( tasklist.length < workers ) {
    workers = tasklist.length;
  }

  // results will go into this array
  var resultsarray = new Array( workers );

  var getNextTaskGenerator = function () { }; // place holder

  var startNextTask = function ( innerworkeridx, innertaskidx ) {
    if ( debug )
      console.error( new Date() + " task[" + innertaskidx + "/" + tasklist.length + "] assigned to worker[" + innerworkeridx + "]" );
    var nextTask = Q.fcall(
      function () {
        return tasklist[innertaskidx]();
      }
    );
    nextTask.then(
      function ( value ) {
        if ( debug )
          console.error( new Date() + " task[" + innertaskidx + "] resolved on worker[" + innerworkeridx + "]" );
        resultsarray[innertaskidx] = { state: "fulfilled", value: value };
        return getNextTaskGenerator( innerworkeridx )();
      },
      function ( reason ) {
        if ( debug )
          console.log( new Date() + " task[" + innertaskidx + "] rejected on worker[" + innerworkeridx + "]" );
        resultsarray[innertaskidx] = { state: "rejected", reason: reason };
        return getNextTaskGenerator( innerworkeridx )();
      }
    );

    return nextTask;
  };

  getNextTaskGenerator = function ( workeridx ) {
    return function () {
      if ( debug )
        console.error( new Date() + " getnext task[" + taskidx + "] for worker[" + workeridx + "]" );
      if ( taskidx < tasklist.length ) {
        var nextTask = startNextTask( workeridx, taskidx );
        taskidx++;
        return nextTask;
      } else {
        workersfree++;
        if ( workersfree == workers ) {
          if ( debug )
            console.error( new Date() + " workermodel RESOLVE" );
          deferred.resolve( resultsarray );
        } else {
          if ( debug )
            console.error( new Date() + " no more work but " + ( workers - workersfree ) + " workers busy" );
        }
      }
    };
  };

  // start workers
  for ( var workeridx = 0; workeridx < workers; workeridx++ ) {
    startNextTask( workeridx, taskidx );
    taskidx++;
  }

  if ( debug )
    console.error( new Date() + " RETURNING PROMISE" );
  return deferred.promise;
}

Like Q.allSettled() this function always resolves and when it does the value is a list of associative arrays, each row (related to the function in the tasklist provided) contains a state field of “fulfilled” or “rejected” and if “fulfilled” then the “value” field will contain the result. Otherwise the “reason” field will contain the error.

What is the output of this script (with debugging turned on)?

Starting
Sat Sep 12 2015 19:48:29 GMT+0100 (BST) task[0/13] assigned to worker[0]
Sat Sep 12 2015 19:48:29 GMT+0100 (BST) task[1/13] assigned to worker[1]
Sat Sep 12 2015 19:48:29 GMT+0100 (BST) task[2/13] assigned to worker[2]
Sat Sep 12 2015 19:48:29 GMT+0100 (BST) RETURNING PROMISE
Going to wait 1 second...
Going to wait 1 second...
Going to wait 1 second...
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[0] resolved on worker[0]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) getnext task[3] for worker[0]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[3/13] assigned to worker[0]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[1] resolved on worker[1]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) getnext task[4] for worker[1]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[4/13] assigned to worker[1]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[2] resolved on worker[2]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) getnext task[5] for worker[2]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[5/13] assigned to worker[2]
Going to wait 1 second...
Going to wait 1 second...
Going to wait 1 second...
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[3] resolved on worker[0]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) getnext task[6] for worker[0]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[6/13] assigned to worker[0]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[4] resolved on worker[1]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) getnext task[7] for worker[1]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[7/13] assigned to worker[1]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[5] resolved on worker[2]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) getnext task[8] for worker[2]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[8/13] assigned to worker[2]
Going to wait 1 second...
Going to wait 1 second...
Going to wait 1 second...
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[6] resolved on worker[0]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) getnext task[9] for worker[0]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[9/13] assigned to worker[0]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[7] resolved on worker[1]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) getnext task[10] for worker[1]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[10/13] assigned to worker[1]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[8] resolved on worker[2]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) getnext task[11] for worker[2]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[11/13] assigned to worker[2]
Going to wait 1 second...
Going to wait 1 second...
Going to wait 1 second...
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) task[9] resolved on worker[0]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) getnext task[12] for worker[0]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) task[12/13] assigned to worker[0]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) task[10] resolved on worker[1]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) getnext task[13] for worker[1]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) no more work but 2 workers busy
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) task[11] resolved on worker[2]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) getnext task[13] for worker[2]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) no more work but 1 workers busy
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) task[12] rejected on worker[0]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) getnext task[13] for worker[0]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) workermodel RESOLVE
End: [
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "rejected",
    "reason": {}
  }
]

All of the above is my own work. It is free for others to use.

See Also

  • bluebird Javascript promises library has a concurrency option which would appear to be similar in functionality

How To Get Cookies From Node.JS HTTP Response

So you make a HTTP request using the http library and you want the cookies returned by the remote webserver (maybe for a later request, a session cookie for example)?

The approach I ended up taking in Node.JS was to inspect the response.headers[“set-cookie”] field which is set to an array of cookies if they have been given in the server response.

e.g.

var http = require( "http" );
var url = require( "url" );

var urlstring = "http://www.voa.com/";
var parsedurl = url.parse( urlstring );
var options = {
  hostname: parsedurl.hostname,
  port: ( parsedurl.port || 80 ), // 80 by default
  method: 'GET',
  path: parsedurl.path,
  headers: { },
};

var request = http.request(
  options,
  function ( response ) {
    // display returned cookies in header
    var setcookie = response.headers["set-cookie"];
    if ( setcookie ) {
      setcookie.forEach(
        function ( cookiestr ) {
          console.log( "COOKIE:" + cookiestr );
        }
      );
    }

    var data = "";
    response.on(
      "data",
      function ( chunk ) { data += chunk; }
    );

    response.on(
      "end",
      function () {
        console.log( "STATUS:" + response.statusCode );
        console.log( "  DATA:" + data );
      }
    );
  }
);

request.on(
  "error",
  function( err ) {
    console.error( "ERROR:" + err );
  }
);

request.end(); // let request know it is finished sending

This will output cookie information like:

COOKIE:JSESSIONID=7bcc3a128; Path=/

If you want to send cookies with your request add a header to the options you pass the request, e.g.:

var cookies = [
  "JSESSIONID=c2aa6fa21",
  "site_ip=10.18.32.12",
];

options.headers["Cookie"] = cookies.join( "; " );

Node.JS Utility To Launch Processes Remotely Using ssh2 Library

This Node.JS utility was written with the sole purpose of running many processes on remote servers and collating the output (stdout and stderr) line by line such that no two lines would get mixed.

It uses the ssh2 library available for Node.JS.

Note that this opens a single ssh connection to each target server and multiplexes the file transfers and command executions through channels within this single ssh connection. By default OpenSSH allowed 10 simultaneous channels in a ssh connection but versions 5.1 and subsequent introduced a MaxSessions parameter which allows this to be greatly increased (I’ve had it in the thousands).

// node.js SSH process launcher and line logger

var fs = require('fs');
var util = require('util');
var ssh2 = require('/usr/local/node/node_modules/ssh2');

/***
  help_and_exit() - print out help for command line, then exit
***/
function help_and_exit() {
    var helpstr = (
        "Usage:\n" +
        "    node launcher.js -s <ip> [-s <ip> ...] \\\n" +
        "        {-i <identity_key> or -w <password>}\\\n" +
        "        [-u <user{root}>] \\\n" +
        "        [-p <port{22}>] \\\n" +
        "        [-c <command_line>] \\\n" +
        "        [-n <num_of_commands>] \\\n" +
        "        [-g <num_of_seconds>] \\\n" +
        "        [-f <file> -t <target_file>] [-f <file> -t <target_file> ...] \\\n" +
        "        [-x <connect_timeout_seconds>] \\\n" +
        "        [-k <keyword_file>] \\\n" +
        "        [-r] \\\n" +
        "        [-d]\n" +
        "\n" +
        "Notes:\n" +
        "  -g <sec> - spread start of n commands over this many seconds\n" +
        "  -f <file> -t <target_file> - uploads specified in source/dest pairs\n" +
        "  -k <file> - keyword file consists of CSV lines, one line per command\n" +
        "  -c - string may use \${1} \${2} ... \${99} to replace from fields in keyword file\n" +
        "       \${time} - timestamp in milliseconds\n" +
        "       \${index} - unique number per command executed\n" +
        "  -r - randomise starting line from keyword file\n" +
        "\n"
    );

    console.log( "%s", helpstr );
    process.exit( 1 );
}

/***
  process_command_line() - handle command line arguments
***/
function process_command_line() {
    if ( process.argv.length < 3 ) {
        help_and_exit();
    }

    var servers = new Array();
    var identity_file = null;
    var user = "root";
    var password = null;
    var port = 22;
    var command = null;
    var command_count = 1;
    var command_spread = 1; // milliseconds to wait between launches
    var command_spread_period = null;
    var files = new Array();
    var debugging = false;
    var timeout_connect = null;
    var keyword_file = null;
    var keyword_line_randomise = false;

    var idx;
    for ( idx = 2; idx < process.argv.length; idx++ ) {
        if ( process.argv[idx] === "-s" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -s option" );
                help_and_exit();
            }
            servers.push( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-i" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -i option" );
                help_and_exit();
            }
            if ( ! fs.existsSync( process.argv[idx] ) ) {
                console.error( "Error, identity file \"%s\" does not exist", process.argv[idx] );
                help_and_exit();
            }
            identity_file = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-u" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -u option" );
                help_and_exit();
            }
            user = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-w" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -w option" );
                help_and_exit();
            }
            password = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-p" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -p option" );
                help_and_exit();
            }
            if (
                ( parseInt( process.argv[idx] ) < 1 ) ||
                ( parseInt( process.argv[idx] ) > 65535 )
            ) {
                console.error( "Invalid port number supplied to -p option" );
                help_and_exit();
            }
            port = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-c" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -c option" );
                help_and_exit();
            }
            command = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-n" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -n option" );
                help_and_exit();
            }
            if ( parseInt( process.argv[idx] ) < 1 ) {
                console.error( "Invalid command count supplied to -n option" );
                help_and_exit();
            }
            command_count = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-g" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -g option" );
                help_and_exit();
            }
            if ( parseInt( process.argv[idx] ) < 1 ) {
                console.error( "Invalid number of seconds supplied to -g option" );
                help_and_exit();
            }
            command_spread_period = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-f" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -f option" );
                help_and_exit();
            }
            if ( ! fs.existsSync( process.argv[idx] ) ) {
                console.error( "Error, source file \"%s\" does not exist", process.argv[idx] );
                help_and_exit();
            }
            var source_file = process.argv[idx];

            idx++;
            if ( process.argv[idx] !== "-t" ) {
                console.error( "Error, must provide -t option following -f option" );
                help_and_exit();
            }

            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -t option" );
                help_and_exit();
            }
            var target_file = process.argv[idx];

            var collection = { "source": source_file, "target": target_file };
            files.push( collection );

            continue;
        }

        if ( process.argv[idx] === "-d" ) {
            debugging = true;
            continue;
        }

        if ( process.argv[idx] === "-x" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -x option" );
                help_and_exit();
            }
            if ( parseInt( process.argv[idx] ) < 1 ) {
                console.error( "Invalid timeout supplied to -x option" );
                help_and_exit();
            }
            timeout_connect = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-k" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -k option" );
                help_and_exit();
            }
            if ( ! fs.existsSync( process.argv[idx] ) ) {
                console.error( "Error, keyword file \"%s\" does not exist", process.argv[idx] );
                help_and_exit();
            }
            keyword_file = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-r" ) {
            keyword_line_randomise = true;
            continue;
        }
    }

    if ( command_spread_period !== null ) {
        command_spread = parseInt( ( command_spread_period * 1000 ) / command_count );
    }

    if ( servers.length < 1 ) {
        console.error( "Must specify a minimum of one server" );
        help_and_exit();
    }

    if ( ( identity_file === null ) && ( password === null ) ) {
        console.error( "Must specify an identity file or login password" );
        help_and_exit();
    }

    var serverstates = { };
    for ( idx = 0; idx < servers.length; idx++ ) {
        serverstates[ servers[idx] ] = "initial";
    }

    var keyword_lines = null;
    if ( keyword_file ) {
        keyword_lines = [ ];

        // load whole file into memory (simplest)
        var keyword_buffer = fs.readFileSync( keyword_file, { encoding: 'utf8' } );
        // split into array of lines and words
        var regexpLine = new RegExp( "([^\r\n]+)[\r\n]+", "g" );
        while ( 1 ) {
            var mymatch = regexpLine.exec( keyword_buffer );
            if ( mymatch === null )
                break;

            var line = mymatch[1];

            if ( debugging ) {
                console.error( "  - processing line \"%s\"", line );
            }

            // process the line of text
            var words = [ ];
            while ( 1 ) {
                var next = line.indexOf( "," );
                if ( next < 0 ) {
                    words.push( line );
                    break;
                }

                words.push( line.substring( 0, next ) );
                line = line.substring( next + 1 );
            }

            keyword_lines.push( words );
        }
    }

    var keyword_line_number = 0;
    if ( keyword_line_randomise ) {
        keyword_line_number = Math.floor(Math.random() * keyword_lines.length);
    }

    var options = {
        "servers": serverstates,
        "identity": identity_file,
        "user": user,
        "password": password,
        "port": port,
        "command": command,
        "command_count": command_count,
        "command_spread": command_spread,
        "files": files,
        "debug": debugging,
        "timeout": timeout_connect,
        "keyword_file": keyword_file,
        "keyword_lines": keyword_lines,
        "keyword_line_number": keyword_line_number,
        "conns": { }
    };

    return( options );
}

/***
  log_server_state() - print out the states of all the servers
***/
function log_server_state( serverlist ) {
    var idx;
    for ( var key in serverlist ) {
        console.error( "    %s: %s", key, serverlist[key] );
    }
}

/***
  die_if_not_connected() - scan serverlist and die if any not connected
***/
function die_if_not_connected( serverlist ) {
    var idx;
    var connected = true;
    var disconnected_list = new Array();
    for ( var key in serverlist ) {
        if (
            ( serverlist[key] === "initial" ) ||
            ( serverlist[key] === "connected" ) ||
            ( serverlist[key] === "error" )
        ) {
            connected = false;
            disconnected_list.push( key );
        }
    }

    if ( connected )
        return;

    console.error( "ERROR: timeout, server%s failed to connect", ( disconnected_list.length === 1 ? "" : "s" ) );

    log_server_state( serverlist );

    process.exit( 2 );
}

/***
  interpret_command() - replace variables in command string with literals
***/
function interpret_command( options, index ) {
    var cmd = options.command;

    if ( options.keyword_lines ) {
        if ( options.keyword_line_number >= options.keyword_lines.length )
            options.keyword_line_number = 0;

        var words = options.keyword_lines[options.keyword_line_number];

        options.keyword_line_number++;

        var find_substitution = new RegExp( "[$][{]([0-9]+)[}]" );
        while ( 1 ) {
            var result = find_substitution.exec( cmd );
            if ( ! result )
                break;

            var start = result.index;
            var end = start + result[0].length;

            var wanted_idx = result[1] - 1;
            var substitution;
            if ( wanted_idx >= words.length ) {
                substitution = "";
            } else {
                substitution = words[wanted_idx];
            }

            cmd = cmd.substring( 0, start ) + substitution + cmd.substring( end );
        }

        var find_time = new RegExp( "[$][{](time)[}]" );
        while ( 1 ) {
            var result = find_time.exec( cmd );
            if ( ! result )
                break;

            var start = result.index;
            var end = start + result[0].length;

            var substitution = (new Date()).getTime();

            cmd = cmd.substring( 0, start ) + substitution + cmd.substring( end );
        }

        var find_index = new RegExp( "[$][{](index)[}]" );
        while ( 1 ) {
            var result = find_index.exec( cmd );
            if ( ! result )
                break;

            var start = result.index;
            var end = start + result[0].length;

            var substitution = index;

            cmd = cmd.substring( 0, start ) + substitution + cmd.substring( end );
        }
    }

    return( cmd );
}

/***
  ip2num() - convert dotted IP address to integer
***/
function ip2num( ip ) {
    var elements = ip.split( '.' );
    var result = (
        parseInt( elements[0] ) * 256 * 256 * 256 +
        parseInt( elements[1] ) * 256 * 256 +
        parseInt( elements[2] ) * 256 +
        parseInt( elements[3] )
    );

    return( result );
}

/***
   getconnidx() - get index of this connection in array of connections
***/
function getconnidx( options, conn ) {
    var idx = 0;
    for ( var tmpip in options.conns ) {
        if ( conn == options.conns[tmpip] ) {
            return idx;
        }
        idx++;
    }

    return( undefined );
}

/***
  do_exec() - execute command via established SSH connection
***/
function do_exec( conn, options, ip, instance_num ) {
    var uniqueindex = getconnidx( options, conn ) * options.command_count + instance_num;

    var cmdline = interpret_command( options, uniqueindex );

    if ( options.debug ) {
        console.error( "  - command line is \"%s\"", cmdline );
    }

    conn.exec(
        cmdline,
        function ( err, stream ) {
            if ( err ) {
                console.error( "[%s]/%d Error executing \"%s\": %s", ip, instance_num, cmdline, err );
                options.servers[ip] = "error";
                options.command_state[ip][instance_num] = "error";
                check_states( options );
                return;
            } else {
                options.command_state[ip][instance_num] = "exec";
            }

            var buffer_stdout = "";
            var buffer_stderr = "";

            var regexpLine = new RegExp( "([^\r\n]+)[\r\n]+" );

            stream.on(
                'data',
                function (data, extended) {
                    if ( extended === 'stderr' ) {
                        buffer_stderr += data;
                    } else {
                        buffer_stdout += data;
                    }

                    if ( options.debug ) {
                        console.error( "[%s]/%d Data received = %d", ip, instance_num, data.length );
                    }

                    while ( 1 ) {
                        var mymatch = regexpLine.exec( buffer_stderr );
                        if ( mymatch == null )
                            break;

                        var start = mymatch.index;
                        var end = start + mymatch[0].length;
                        var line = mymatch[1];

                        buffer_stderr = buffer_stderr.substring( end );

                        console.log( "[%s]/%d stderr: %s", ip, instance_num, line );
                    }

                    while ( 1 ) {
                        var mymatch = regexpLine.exec( buffer_stdout );
                        if ( mymatch == null )
                            break;

                        var start = mymatch.index;
                        var end = start + mymatch[0].length;
                        var line = mymatch[1];

                        buffer_stdout = buffer_stdout.substring( end );

                        console.log( "[%s]/%d stdout: %s", ip, instance_num, line );
                    }
                }
            );

            stream.on(
                'end',
                function () {
                    if ( options.debug ) {
                        console.error( "[%s]/%d stream end", ip, instance_num );
                    }
                    options.command_state[ip][instance_num] = "completed";
                    check_states( options );
                }
            );

            stream.on(
                'close',
                function () {
                    if ( options.debug ) {
                        console.error( "[%s]/%d stream closed", ip, instance_num );
                    }
                }
            );
            stream.on(
                'exit',
                function (retval, signalName, didCoreDump, description) {
                    if ( retval != null ) {
                        if ( options.debug ) {
                            console.error( "[%s]/%d stream exit = %d", ip, instance_num, retval );
                        }
                        stream.end();
                    } else {
                        if ( options.debug ) {
                            console.error( "[%s]/%d stream exit signal %s%s [%s]", ip, instance_num, signalName, didCoreDump ? " (dumped core)" : "", description );
                        }
                    }
                }
            );
        }
    );
}

/***
  check_states() - state machine engine, checking states, calling next action when ready
***/
function check_states( options ) {
    var all_done = true;

    if ( options.debug ) {
        for ( var sip in options.servers ) {
            console.error( "  %s: %s", sip, options.servers[sip] );
        }
    }

    if ( options.command_state ) {
        for ( var ip in options.command_state ) {
            if (
                ( options.servers[ip] === "closed" ) ||
                ( options.servers[ip] === "error" )
            ) {
                continue;
            }

            if ( options.debug ) {
                var strstate = "";
                var statectr = 0;
                for ( statectr = 0; statectr < options.command_state[ip].length; statectr++ ) {
                    if ( strstate.length != 0 ) {
                        strstate += ", ";
                    }
                    strstate += "\n  " + statectr + ":" + options.command_state[ip][statectr];
                }
                console.error( "[%s] command_state = %s", ip, strstate );
            }

            var all_completed = true;
            var instance = 0;
            for ( inst = 0; inst < options.command_state[ip].length; inst++ ) {
                var state = options.command_state[ip][inst];
                if (
                    ( state === "error" ) ||
                    ( state === "completed" )
                ) {
                    ;
                } else {
                    all_completed = false;
                }
            }

            if ( all_completed ) {
                options.conns[ip].end();
                options.servers[ip] = "closed";
            }
        }
    }

    for ( var ip in options.servers ) {
        if (
            ( options.servers[ip] === "error" ) ||
            ( options.servers[ip] === "closed" )
        ) {
            ;
        } else {
            all_done = false;
        }
    }

    if ( all_done ) {
        if ( options.debug ) {
            console.error( "- all completed" );
        }
        process.exit( 0 );
    }

    var all_awaiting_cmd = true;
    var any_closed = false;
    var any_awaiting = false;

    for ( var ip in options.servers ) {
        if (
            ( options.servers[ip] === "awaiting_cmd" )
        ) {
            any_awaiting = true;
        } else {
            all_awaiting_cmd = false;
            if (
                ( options.servers[ip] === "closed" ) ||
                ( options.servers[ip] === "error" )
            ) {
                // connection closed before "awaiting_command"
                any_closed = true;
            }
        }
    }

    if ( all_awaiting_cmd ) {
        options["command_state"] = { };
        for ( var ip in options.servers ) {
            options.servers[ip] = "executing";

            if ( options.command != null ) {
                var conn = options.conns[ip];
                if ( ! conn ) {
                    console.error( "Programmer error, conn should exist" );
                    process.exit( 98 );
                }

                // execute command number of times
                var cmdidx;
                options.command_state[ip] = new Array( options.command_count );
                for ( cmdidx = 0; cmdidx < options.command_count; cmdidx++ ) {
                    options.command_state[ip][cmdidx] = "initial";
                }
                var timeoffset = 0;
                for ( cmdidx = 0; cmdidx < options.command_count; cmdidx++ ) {
                    setTimeout(
                        function (conn, options, ip, cmdidx) {
                            do_exec( conn, options, ip, cmdidx );
                        },
                        timeoffset,
                        conn, options, ip, cmdidx
                    );
                    timeoffset += options.command_spread;
                }
            } else {
                options.servers[ip] = "closed";
                check_states( options );
            }
        }
    } else if ( any_closed && any_awaiting ) {
        console.error( "Error, connection has been prematurely closed" );
        process.exit( 2 );
    }
}

/***
  do_sftp() - perform file upload via established SSH connection
***/
function do_sftp( conn, options, ip, filesremaining, callback ) {
    if ( options.debug ) {
        console.error( "[%s] doing sftp %d", ip, filesremaining.length );
    }

    if ( filesremaining.length == 0 ) {
        callback();
        return;
    }

    var nextfile = filesremaining.shift();
    if ( options.debug ) {
        console.error( "[%s] next file is %s", ip, nextfile.source );
    }

    conn.sftp(
        function (err, sftp) {
            if ( err ) {
                console.error( "[%s] Error, problem starting SFTP: %s", ip, err );
                process.exit( 3 );
            }

            if ( options.debug ) {
                console.error( "[%s] SFTP started", ip );
            }

            // start upload
            var readStream = fs.createReadStream( nextfile.source );
            var writeStream = sftp.createWriteStream(
                nextfile.target,
                { flags: 'w', encoding: null, mode: 0666 }
            );

            writeStream.on(
                'close',
                function () {
                    if ( options.debug ) {
                        console.error( "[%s] end transfer of \"%s\"", ip, nextfile.source );
                    }
                    sftp.end();
                    do_sftp( conn, options, ip, filesremaining, callback );
                }
            );

            readStream.pipe( writeStream );
        }
    );
}

/***
  do_ready_action_awaiting_cmd() - change state to waiting for cmd and check state machine
***/
function do_ready_action_awaiting_cmd( conn, options, ip ) {
    options.servers[ip] = "awaiting_cmd";
    check_states( options );
}

/***
 do_ready_action_upload_files() - upload files if required
***/
function do_ready_action_upload_files( conn, options, ip ) {
    if ( options.files.length > 0 ) {
        var filesremaining = options.files.slice( 0 );

        do_sftp(
            conn, options, ip, filesremaining,
            function () {
                do_ready_action_awaiting_cmd( conn, options, ip );
            }
        );
    } else {
        // no files to transfer, change state
        do_ready_action_awaiting_cmd( conn, options, ip );
    }
}

/***
  connect_to_server() - set up a connection to a server
***/
function connect_to_server( options, ip ) {
    var conn = new ssh2();

    conn.setMaxListeners(0);

    conn.on(
        'connect',
        function () {
            if ( options.debug ) {
                console.error( "[%s] connect", ip );
            }
            options.servers[ip] = "connected";
        }
    );

    conn.on(
        'ready',
        function () {
            if ( options.debug ) {
                console.error( "[%s] ready", ip );
            }
            options.servers[ip] = "ready";
            options.conns[ip] = conn;

            do_ready_action_upload_files(
                conn, options, ip
            );
        }
    );

    conn.on(
        'error',
        function (err) {
            if ( options.debug ) {
                console.error( "[%s] error: %s", ip, err );
            }
            options.servers[ip] = "error";
            check_states( options );
        }
    );

    conn.on(
        'end',
        function () {
            if ( options.debug ) {
                console.error( "[%s] end", ip );
            }
            options.servers[ip] = "closed";
            check_states( options );
        }
    );

    conn.on(
        'close',
        function (had_error) {
            if ( options.debug ) {
                console.error( "[%s] end: %s", ip, had_error );
            }
            options.servers[ip] = "closed";
            check_states( options );
        }
    );

    var connectoptions = {
        "host": ip,
        "port": options.port,
        "username": options.user
    };

    if ( options.identity !== null ) {
        connectoptions["privateKey"] = fs.readFileSync( options.identity );
    } else if ( options.password !== null ) {
        connectoptions["password"] = options.password;
    } else {
        console.error( "No authentication option for SSH provided" );
        process.exit( 1 );
    }

    conn.connect( connectoptions );
}

/***
  connect_to_servers() - connect to all the servers
***/
function connect_to_servers( options ) {
    var idx;
    for ( var ip in options.servers ) {
        connect_to_server( options, ip );
    }
}

/***
  main() - process command line, then connect to servers
***/
function main() {
    var options = process_command_line();

    // establish ssh connections to all servers, wait until this is achieved
    if ( options.debug ) {
        console.error( "options = %s", util.inspect( options ) );
    }

    // timeout to blow up if connections haven't been established
    if ( options.timeout ) {
        setTimeout(
            function () {
                die_if_not_connected( options.servers );
            },
            options.timeout * 1000
        );
    }

    // establish connection to servers
    connect_to_servers( options );
}

main();

Example Usage

One could have a local jar file they wish to run remotely on four servers. And they want to run 20 instances of this on each server but with the instances started spread out over 60 seconds. In total this means that 80 instances of the program will be started spread out over 60 seconds.

The jar file exists in ~/myjar.jar on the local server and before executing this jar file it must be uploaded to /tmp/ on the remote servers. Command execution won’t begin until the file has been successfully transferred to all the remote servers.

/usr/local/node/bin/node sshlauncher.js \
  -s test_server1.my.net \
  -s test_server2.my.net \
  -s test_server3.my.net \
  -s test_server4.my.net \
  -u testuser \
  -w testpassword \
  -x 15 \
  -g 60 \
  -n 20 \
  -f ~/myjar.jar -t /tmp/myjar.jar \
  -c 'java -jar /tmp/myjar.jar'

Debugging mode can be enabled with a -d flag so you can see when connections take place and file transfers, etc.

You may wish to provide a text comma-separated value file so that you can provide unique or different parameters to your executed command instances (e.g. for testing login processes to a site).

If you wanted to launch 4 instances of a program with unique usernames you could have a file named ~/keywords.txt containing:

user1,password1
user2,password2
user3,password3
user4,password4

and a command line of:

/usr/local/node/bin/node sshlauncher.js \
  -s test_server1.my.net \
  -s test_server2.my.net \
  -s test_server3.my.net \
  -s test_server4.my.net \
  -u testuser \
  -w testpassword \
  -x 15 \
  -g 60 \
  -n 20 \
  -f ~/myjar.jar -t /tmp/myjar.jar \
  -c 'java -jar /tmp/myjar.jar ${1} ${2}' \
  -k ~/keywords.txt

If your CSV (comma-separated value) keyword file is long enough (say, a file full of dictionary words) you can provide the -r flag to start reading from a random line in the file (e.g. when trying to test a cached server and you want to feed it unique words to circumvent caching).

Node.JS String Length Undefined

This issue arose using node.js version v0.10.3.

Here’s some example code:

function writeStringLength( str ) {
  console.log( "str = \"" + str + "\"" );
  console.log( "str.length is " + str.length );

  var copy = str;
  console.log( "copy.length is " + copy.length );

  var copyObj = new String( str );
  console.log( "copyObj.length is " + copyObj.length );
}

var now = new Date();
writeStringLength( now.getFullYear() );

This outputs:

user@localhost:/tmp$ /usr/local/node/bin/node /tmp/brokenstring.js
str = "2013"
str.length is undefined
copy.length is undefined
copyObj.length is 4

So – why does JavaScript think the string doesn’t have a defined value for the length property? According to the Node.JS manual entry for the String object “JavaScript distinguishes between String objects and primitive string values”. Conversion of primitive strings to String objects should happen automatically but clearly in this case node.js is unable to determine that the function parameter is a string that needs “boxing” into a string object. By forcing a new String object to be created we can be sure the length property will return a useful value.

Note that the values returned by the Date() object functions causes this problem – perhaps because Node.JS doesn’t know for certain what kind of values these functions returns (could they be from an external library?). Whereas if you try and pass an ordinary string (e.g. “example”) to writeStringLength() then the parameter will, as expected, have a defined length property. Perhaps this is because node.js knows the parameter is an explicitly defined string in this example.

How to Upload a File over SSH Using Node.JS

The pure-JavaScript SSH2 Node.JS library available at https://github.com/mscdex/ssh2 allows a variety of SSH protocol client operations to be performed within a Node.JS script.

Specifically I wanted to know how to upload a file. Essentially, after creating a SSH connection to a server, it is a matter of opening a read stream from the local file system and opening a write stream to the SSH connection and then piping the two together.

The example code may provide illustration:

var fs = require('fs');
var ssh2 = require('/usr/local/node/node_modules/ssh2');

var conn = new ssh2();

conn.on(
    'connect',
    function () {
        console.log( "- connected" );
    }
);

conn.on(
    'ready',
    function () {
        console.log( "- ready" );

        conn.sftp(
            function (err, sftp) {
                if ( err ) {
                    console.log( "Error, problem starting SFTP: %s", err );
                    process.exit( 2 );
                }

                console.log( "- SFTP started" );

                // upload file
                var readStream = fs.createReadStream( "/proc/meminfo" );
                var writeStream = sftp.createWriteStream( "/tmp/meminfo.txt" );

                // what to do when transfer finishes
                writeStream.on(
                    'close',
                    function () {
                        console.log( "- file transferred" );
                        sftp.end();
                        process.exit( 0 );
                    }
                );

                // initiate transfer of file
                readStream.pipe( writeStream );
            }
        );
    }
);

conn.on(
    'error',
    function (err) {
        console.log( "- connection error: %s", err );
        process.exit( 1 );
    }
);

conn.on(
    'end',
    function () {
        process.exit( 0 );
    }
);

conn.connect(
    {
        "host": "10.0.0.1",
        "port": 22,
        "username": "root",
        "privateKey": "/home/root/.ssh/id_root"
    }
);

Sharing Port 443 With HTTPS and SSH Using Node.JS

I want to use port 443 for both a HTTPS web server and SSH. This blog post gave the technical details – which is that SSH and HTTPS can be differentiated by the fact that a HTTPS client sends traffic immediately after connecting, whereas SSH clients wait for the SSH banner to be produced by the SSH server.

So I present, here, a TCP proxy that listens on the external IP (whatever your eth0 interface thinks its address is) and then intelligently switches traffic to localhost ports 443 or 22 (configurable) depending on whether it thinks it is HTTPS or SSH.

Note that, because this is a proxy, your SSH and HTTPS logs will not record the source address of who connected to your services – rather the localhost address (127.0.0.1 or ::1) will be shown. See the blog link above for a low-level tool that handles this situation properly.

// import required modules
var net = require('net');

// global variables (defaults)
var debugging = 0;
var internal_ssh_port = 22;
var internal_ssl_port = 443;
var external_port = 443;
var external_address = "0.0.0.0";
var internal_address = "127.0.0.1";
var is_ssh_timeout = 2500; // milliseconds to wait before assuming SSH
var socket_terminate_delay = 500; // milliseconds before destroying end'd sock

// generic function for destroying a socket
function destroySocket( socket, delay, comment ) {
  setTimeout(
    function () {
      try {
        socket.destroy();
      } catch ( err ) {
        if ( debugging ) {
          console.log( '  - error destroying socket "' + comment + '": ' + err );
        }
      }
    },
    delay
  );
}

// generic function for creating a proxied connection
function createSocketProxy( socketServer, type, address, port ) {
  if ( debugging ) {
    console.log( '  > opening ' + type + ' connection to ' + address + ':' + port );
  }

  // create socket and connect to back-end server
  var socketProxy = new net.Socket();
  socketProxy.connect(
    port,
    address,
    function () {
      if ( debugging ) {
        console.log( '  > proxied to ' + type );
      }
    }
  );

  // handle data from the server
  var bytesReceived = 0;
  socketProxy.on(
    'data',
    function ( chunk ) {
      if ( debugging ) {
        bytesReceived += chunk.length;
        console.log( '  < ' + chunk.length + ' ' + type + ' bytes (' + bytesReceived + ' total)' );
      }
      socketServer.write( chunk );
    }
  );

  // handle socket close by the server
  socketProxy.on(
    'end',
    function () {
      if ( debugging ) {
        console.log( '  < ' + type + ' close' );
      }
      var emptyPkt = '';
      socketServer.end( emptyPkt );
      // destroySocket( socketServer, socket_terminate_delay, 'server ' + type );
      socketProxy.end( emptyPkt );
      // destroySocket( socketProxy, socket_terminate_delay, 'proxy ' + type );
      socketProxy = undefined;
    }
  );

  // handle socket error from the server
  socketProxy.on(
    'error',
    function ( err ) {
      if ( debugging ) {
        console.log( '  < ' + type + ' error: ' + err );
      }
      socketServer.end();
      destroySocket( socketServer, socket_terminate_delay, 'server ' + type );
      if ( socketProxy ) {
        socketProxy.destroy(); // sometimes it is undef
      }
      socketProxy = undefined;
    }
  );

  // return the newly created socket
  return( socketProxy );
}

// function for handling connection from client
function socketCallback( socketServer ) {
  var clientSentTraffic = false;
  var socketProxy = undefined;

  if ( debugging ) {
    console.log( "- received connection from " + socketServer.remoteAddress + ":" + socketServer.remotePort );
  }

  // we want to set a timeout here
  setTimeout(
    function() {
      if ( ! clientSentTraffic ) {
        if ( debugging ) {
          console.log( "  > timeout, switching to SSH mode" );
        }
        socketProxy = createSocketProxy(
          socketServer,
          'SSH',
          internal_address,
          internal_ssh_port
        );
      } else {
        if ( debugging ) {
          console.log( "  > ignoring timeout" );
        }
      }
    },
    is_ssh_timeout
  );

  // handle data from client
  socketServer.on(
    'data',
    function( chunk ) {
      console.log( '  > ' + chunk.length + ' bytes' );

      // have we determined yet what kind of connection this is?
      if ( ! socketProxy ) {
        clientSentTraffic = true;

        socketProxy = createSocketProxy(
          socketServer,
          'SSL',
          internal_address,
          internal_ssl_port
        );
      }

      if ( socketProxy )
        socketProxy.write( chunk );
    }
  );

  // handle socket close by client
  socketServer.on(
    'end',
    function() {
      if ( debugging ) {
        console.log( '  > socket close' );
      }

      if ( socketProxy ) {
        socketProxy.end();
        //destroySocket( socketProxy, socket_terminate_delay, 'proxy' );
        socketProxy = undefined;
      }
      socketServer.end();
      //destroySocket( socketServer, socket_terminate_delay, 'server' );
    }
  );

  // handle socket error from client
  socketServer.on(
    'error',
    function( err ) {
      if ( debugging ) {
        console.log( '  > socket error: ' + err );
      }

      if ( socketProxy ) {
        socketProxy.end();
        destroySocket( socketProxy, socket_terminate_delay, 'proxy' );
        socketProxy = undefined;
      }
      if ( socketServer ) {
        socketServer.destroy();
      }
    }
  );
}

function main() {
  // check for any command line arguments
  if ( process.argv.length < 3 ) {
    console.log(
      'ssl/ssh switcher\n' +
      '\n' +
      'Arguments:\n' +
      '  -x <ip> - external interface IP address (default ' + external_address + ')\n' +
      '  -i <ip> - internal interface IP address (default ' + internal_address + ')\n' +
      '  -s <port> - internal SSH port number (default ' + internal_ssh_port + ')\n' +
      '  -w <port> - internal HTTPS port number (default ' + internal_ssl_port + ')\n' +
      '  -p <port> - external port to switch (default ' + external_port + ')\n' +
      '  -d - turn debugging on\n'
    );
    process.exit( 1 );
  }

  for ( var argn = 2; argn < process.argv.length; argn++ ) {
    if ( process.argv[argn] === '-p' ) {
      external_port = parseInt( process.argv[argn + 1] );
      argn++;
      continue;
    }

    if ( process.argv[argn] === '-s' ) {
      internal_ssh_port = parseInt( process.argv[argn + 1] );
      argn++;
      continue;
    }

    if ( process.argv[argn] === '-w' ) {
      internal_ssl_port = parseInt( process.argv[argn + 1] );
      argn++;
      continue;
    }

    if ( process.argv[argn] === '-x' ) {
      external_address = process.argv[argn + 1];
      argn++;
      continue;
    }

    if ( process.argv[argn] === '-i' ) {
      internal_address = process.argv[argn + 1];
      argn++;
      continue;
    }

    if ( process.argv[argn] === '-d' ) {
      debugging = 1;
      continue;
    }
  }

  if ( debugging ) {
    console.log( 'server listening on ' + external_address + ':' + external_port );
  }

  // start TCP server with custom callback function
  var server = net.createServer( socketCallback );
  server.on(
    'error',
    function ( err ) {
      console.log( 'Error with server: ' + err );
    }
  );
  server.listen( external_port, external_address );
}

main();

When run without any command-line arguments you will get the help text:

root@myserver:~# /usr/local/node/bin/node sslsshproxy.js
ssl/ssh switcher

Arguments:
  -x <ip> - external interface IP address (default 0.0.0.0)
  -i <ip> - internal interface IP address (default 127.0.0.1)
  -s <port> - internal SSH port number (default 22)
  -w <port> - internal HTTPS port number (default 443)
  -p <port> - external port to switch (default 443)
  -d - turn debugging on

At a minimum you should specify the -x option with the IP address of your interface. You will find this by running the following command:

root@myserver:~# ifconfig eth0
eth0      Link encap:Ethernet  HWaddr b8:dc:7f:89:2a:a0  
          inet addr:192.168.1.48  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: 2a02:ac8:1:3500::3/64 Scope:Global
          inet6 addr: fe80::bbdd:6fff:fe89:4c1a/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:4223010 errors:0 dropped:0 overruns:0 frame:0
          TX packets:1418451 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:4455158325 (4.4 GB)  TX bytes:466186918 (466.1 MB)
          Interrupt:16 Memory:da000000-da012800 

Then use this in the command:

setsid node sslsshproxy.js -x 192.168.1.48 2>/dev/null >/dev/null &

Node.JS HTTP and HTTPS Proxy

Writing a HTTP proxy using Node.JS is easy. There are plenty of examples on how to do this on the web. But things get trickier when you want to also proxy HTTPS requests. This is because http.createServer() doesn’t call the callback when a CONNECT request comes through. Instead you have to explicitly add a separate listener for the “connect” event and handle the socket communications directly. But this is very similar to handling a HTTP proxy request – you set up a socket connection to the target and then proxy communications between the client request socket and the proxied socket.

This proxy application accepts two command line arguments: -d switches on debugging (which you won’t need normally) and -p port selects which TCP port the proxy should listen on. For example:

user@myserver:~# /usr/local/node/bin/node webproxy.js -d -p 8080

Here is the source code:

var http = require('http');
var net = require('net');

var debugging = 0;

var regex_hostport = /^([^:]+)(:([0-9]+))?$/;

function getHostPortFromString( hostString, defaultPort ) {
  var host = hostString;
  var port = defaultPort;

  var result = regex_hostport.exec( hostString );
  if ( result != null ) {
    host = result[1];
    if ( result[2] != null ) {
      port = result[3];
    }
  }

  return( [ host, port ] );
}

// handle a HTTP proxy request
function httpUserRequest( userRequest, userResponse ) {
  if ( debugging ) {
    console.log( '  > request: %s', userRequest.url );
  }

  var httpVersion = userRequest['httpVersion'];
  var hostport = getHostPortFromString( userRequest.headers['host'], 80 );

  // have to extract the path from the requested URL
  var path = userRequest.url;
  result = /^[a-zA-Z]+:\/\/[^\/]+(\/.*)?$/.exec( userRequest.url );
  if ( result ) {
    if ( result[1].length > 0 ) {
      path = result[1];
    } else {
      path = "/";
    }
  }

  var options = {
    'host': hostport[0],
    'port': hostport[1],
    'method': userRequest.method,
    'path': path,
    'agent': userRequest.agent,
    'auth': userRequest.auth,
    'headers': userRequest.headers
  };

  if ( debugging ) {
    console.log( '  > options: %s', JSON.stringify( options, null, 2 ) );
  }

  var proxyRequest = http.request(
    options,
    function ( proxyResponse ) {
      if ( debugging ) {
        console.log( '  > request headers: %s', JSON.stringify( options['headers'], null, 2 ) );
      }

      if ( debugging ) {
        console.log( '  < response %d headers: %s', proxyResponse.statusCode, JSON.stringify( proxyResponse.headers, null, 2 ) );
      }

      userResponse.writeHead(
        proxyResponse.statusCode,
        proxyResponse.headers
      );

      proxyResponse.on(
        'data',
        function (chunk) {
          if ( debugging ) {
            console.log( '  < chunk = %d bytes', chunk.length );
          }
          userResponse.write( chunk );
        }
      );

      proxyResponse.on(
        'end',
        function () {
          if ( debugging ) {
            console.log( '  < END' );
          }
          userResponse.end();
        }
      );
    }
  );

  proxyRequest.on(
    'error',
    function ( error ) {
      userResponse.writeHead( 500 );
      userResponse.write(
        "<h1>500 Error</h1>\r\n" +
        "<p>Error was <pre>" + error + "</pre></p>\r\n" +
        "</body></html>\r\n"
      );
      userResponse.end();
    }
  );

  userRequest.addListener(
    'data',
    function (chunk) {
      if ( debugging ) {
        console.log( '  > chunk = %d bytes', chunk.length );
      }
      proxyRequest.write( chunk );
    }
  );

  userRequest.addListener(
    'end',
    function () {
      proxyRequest.end();
    }
  );
}

function main() {
  var port = 5555; // default port if none on command line

  // check for any command line arguments
  for ( var argn = 2; argn < process.argv.length; argn++ ) {
    if ( process.argv[argn] === '-p' ) {
      port = parseInt( process.argv[argn + 1] );
      argn++;
      continue;
    }

    if ( process.argv[argn] === '-d' ) {
      debugging = 1;
      continue;
    }
  }

  if ( debugging ) {
    console.log( 'server listening on port ' + port );
  }

  // start HTTP server with custom request handler callback function
  var server = http.createServer( httpUserRequest ).listen(port);

  // add handler for HTTPS (which issues a CONNECT to the proxy)
  server.addListener(
    'connect',
    function ( request, socketRequest, bodyhead ) {
      var url = request['url'];
      var httpVersion = request['httpVersion'];

      var hostport = getHostPortFromString( url, 443 );

      if ( debugging )
        console.log( '  = will connect to %s:%s', hostport[0], hostport[1] );

      // set up TCP connection
      var proxySocket = new net.Socket();
      proxySocket.connect(
        parseInt( hostport[1] ), hostport[0],
        function () {
          if ( debugging )
            console.log( '  < connected to %s/%s', hostport[0], hostport[1] );

          if ( debugging )
            console.log( '  > writing head of length %d', bodyhead.length );

          proxySocket.write( bodyhead );

          // tell the caller the connection was successfully established
          socketRequest.write( "HTTP/" + httpVersion + " 200 Connection established\r\n\r\n" );
        }
      );

      proxySocket.on(
        'data',
        function ( chunk ) {
          if ( debugging )
            console.log( '  < data length = %d', chunk.length );

          socketRequest.write( chunk );
        }
      );

      proxySocket.on(
        'end',
        function () {
          if ( debugging )
            console.log( '  < end' );

          socketRequest.end();
        }
      );

      socketRequest.on(
        'data',
        function ( chunk ) {
          if ( debugging )
            console.log( '  > data length = %d', chunk.length );

          proxySocket.write( chunk );
        }
      );

      socketRequest.on(
        'end',
        function () {
          if ( debugging )
            console.log( '  > end' );

          proxySocket.end();
        }
      );

      proxySocket.on(
        'error',
        function ( err ) {
          socketRequest.write( "HTTP/" + httpVersion + " 500 Connection error\r\n\r\n" );
          if ( debugging ) {
            console.log( '  < ERR: %s', err );
          }
          socketRequest.end();
        }
      );

      socketRequest.on(
        'error',
        function ( err ) {
          if ( debugging ) {
            console.log( '  > ERR: %s', err );
          }
          proxySocket.end();
        }
      );
    }
  ); // HTTPS connect listener
}

main();

Postscript

I wrote this proxy and have documented it because I couldn’t find any examples on the Internet for HTTPS proxies. I was trying to connect my web browser to my Node.JS HTTP proxy (which I’d duplicated from code I’d found elsewhere on the web) – but couldn’t see my callback being fired. After delving into the source of http.js that comes with Node.JS and looking at the documentation for the connect event later (it’s obvious when you think about it…) I eventually got myself a working HTTPS proxy (in the same script as the HTTP proxy on the same port).

Creating a Reverse Proxy For National Rail’s Mobile Website

The Nation Rail mobile website, http://m.nationalrail.co.uk/, thinks it is smarter than you. If you try and access this link from your desktop browser you will be redirected to the full website: http://www.nationalrail.co.uk/.

I wanted to create an iframe on a personal webpage so that I could keep up-to-date with the next departing train times. Let’s say you want to know what trains are departing London Waterloo and stopping at Surbiton. You could visit the full website live departures page (http://ojp.nationalrail.co.uk/service/ldbboard/dep/WAT/SUR/To) but the page has more than you might want in a smaller iframe. The mobile site view is what you’d prefer at http://m.nationalrail.co.uk/pj/ldbboard/dep/WAT/SUR/To using the following embedded code:

<style type="text/css">
iframe.infoframe {
  float: right;
}
</style>



<script type="text/javascript">
function updatetrainframe() {
  var myFrame = document.getElementById("idtrainframe");
  myFrame.src = "http://m.nationalrail.co.uk/pj/ldbboard/dep/PAD/RDG/To";
}

function startrepeat( callback, interval ) {
  callback();
  setInterval( callback, interval );
}

startrepeat( updatetrainframe, 1000 * 60 * 5 ); // update every 5 minutes starting now
</script>

Of course the problem is that the National Rail website hijacks your request to the mobile website and plonks you on the full desktop version of the website – which is not what you wanted (this is incredibly bad practice and somebody at National Rail should be fired for this: if someone goes to the effort to deliberately type “m.” instead of “www.” then it is clear they want the mobile version of the site).

Creating a Reverse Proxy Using Apache

Creating a reverse proxy is easy using the Apache web server. Create a new virtual site and add the following to the configuration:

ProxyPass / http://m.nationalrail.co.uk/
ProxyPassReverse / http://m.nationalrail.co.uk/
RequestHeader set User-Agent "Mozilla/5.0 (Linux; U; Android 2.2; nl-nl; Desire_A8181 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"

This will proxy-pass requests to your virtual site to the National Rail mobile website and substitute the user-agent for a mobile phone’s – fooling the National Rail website into serving the content you wanted all along.

Creating a Reverse Proxy Using Node.JS

Maybe you don’t have the convenience of running an Apache webserver – or perhaps you do but want to use a different method.

If you use node.js then the following script will proxy-pass requests to the National Rail mobile website:

var http = require('http');

var debugging = 0;

function deleteAnyCase( associativeArray, keyString ) {
    for ( var key in associativeArray ) {
        if ( key.toLowerCase() === keyString.toLowerCase() ) {
            delete associativeArray[ key ];
        }
    }
}

function httpUserRequest( userRequest, userResponse ) {
    if ( debugging )
        console.log( '  > request: ' + userRequest.url );

    // create options object required for proxy request we will make
    var options = {
        host: 'm.nationalrail.co.uk',
        port: 80,
    };
    options.path = userRequest.url;
    options.headers = userRequest.headers;
    options.method = userRequest.method;
    deleteAnyCase( options.headers, "user-agent" );

    // we must fool National Rail into thinking we are mobile device
    options.headers['user-agent'] = 'Mozilla/5.0 (Linux; U; Android 2.2; nl-nl; Desire_A8181 Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1'
    options.headers.host = options.host; // important: site name but no port
    delete options.headers.cookie; // stop National Rail tracking us

    if ( debugging )
        console.log( '  >   options = %j', options );

    // make request to national rail, and route data between user and target
    var proxyRequest = http.request(
        options,
        function (proxyResponse) {
            if ( debugging )
                console.log( '  < response' );

            if ( debugging )
                console.log( '      headers = %j', proxyResponse.headers );

            // protect again National Rail trying to redirect to new host
            var newHeaders = proxyResponse.headers;
            if ( "location" in newHeaders ) {
                newHeaders["location"] = newHeaders["location"].replace(
                    "http://m.nationalrail.co.uk", ""
                );
            }

            userResponse.writeHead(
                proxyResponse.statusCode,
                newHeaders
            );

            // route incoming data from National Rail to user response
            proxyResponse.on(
                'data',
                function (chunk) {
                    if ( debugging )
                        console.log( '  < response data' );

                    userResponse.write( chunk );
                }
            );

            proxyResponse.on(
                'end',
                function() {
                    if ( debugging )
                        console.log( '  < response end' );

                    userResponse.end();
                }
            );
        }
    );

    proxyRequest.on( 'error', function(error) {
        console.error( '  problem with request: ' + error.message );
    } );

    // route outgoing user request to National Rail
    userRequest.on( 'data', function(chunk) {
        if ( debugging )
            console.log( '  > request data' );

        proxyRequest.write( chunk );
    } );
    userRequest.on( 'end', function() {
        if ( debugging )
            console.log( '  > request end' );

        proxyRequest.end();
    } );
}

function main() {
    var port = 5555; // default port if none on command line

    // check for any command line arguments
    for ( var argn = 2; argn < process.argv.length; argn++ ) {
        if ( process.argv[argn] === '-p' ) {
            port = parseInt( process.argv[argn + 1] );
            argn++;
            continue;
        }

        if ( process.argv[argn] === '-d' ) {
            debugging = 1;
            continue;
        }
    }

    if ( debugging ) {
        console.log( 'server listening on port ' + port );
    }

    // start HTTP server with custom request handler callback function
    http.createServer( httpUserRequest ).listen(port);
}

main();

This script checks the command line for a port argument (and optionally a debugging flag) and will listen on that port and proxy any requests to the National Rail mobile website. By default it attaches to locahost:5555 – so you can run this script and fetch the departures from London Waterloo to Surbiton using the URL http://localhost:5555/pj/ldbboard/dep/WAT/SUR/To.