newspaint

Documenting Problems That Were Difficult To Find The Answer To

Category Archives: JavaScript

Getting Statistics from HG612 Revision 3B VDSL Modem Using Node

The HG612 Revision 3B VDSL modem commonly provided by BT OpenReach in the United Kingdom for fibre-to-the-cabinet (FTTC) connected households can be unlocked with alternative firmware.

For those modems that are unlocked they can be connected to on the modem’s default IP address of 192.168.1.1 with a username of admin, password of admin, using web (if enabled), telnet, and ssh.

Such a session entails getting into the shell, then running a Linux command to show the DSL statistics, like the following:

me@home:~$ ssh admin@192.168.1.1
admin@192.168.1.1's password: admin
PTY allocation request failed on channel 0

ATP>sh
sh


BusyBox v1.9.1 (2010-10-15 17:59:06 CST) built-in shell (ash)
Enter 'help' for a list of built-in commands.

# xdslcmd info --stats
xdslcmd: ADSL driver and PHY status
Status: Showtime
Retrain Reason: 0
Max:    Upstream rate = 30992 Kbps, Downstream rate = 90516 Kbps
Path:   0, Upstream rate = 20000 Kbps, Downstream rate = 79999 Kbps
...
# exit
exit

ATP>exit
exit
exit from configuration console.
[6408] Jan 03 23:45:49 exit after auth (admin): child failed
Connection to 192.168.1.1 closed.

I wanted a way of dumping this information on a regular basis automatically but it wasn’t as simple as merely executing a command from the Linux command line, e.g. ssh admin@192.168.1.1 “sh\nxdslcmd info –stats\nexit\nexit”.

So I wrote the following Node.JS script to open a ssh connection to the modem, and execute the appropriate instructions, and dump the output from the xdslcmd command.

'use strict';

/*

  You will need to run the following in the directory you place this script:
    npm install ssh2
    npm install bluebird

*/

var Promise = require("bluebird");
var ssh2 = require("ssh2");

var modem_ip = "192.168.1.1";
var modem_user = "admin";
var modem_pass = "admin";

var debugging = false;

function debug( msg ) {
  if ( debugging ) {
    console.log( msg );
  }
}

function connect() {
  return new Promise( function( resolve, reject ) {
    var conn = new ssh2();

    conn.setMaxListeners(0);

    conn.on(
      'connect',
      function () {
        debug( "ssh connect" );
      }
    );

    conn.on(
      'ready',
      function () {
        debug( "ssh ready" );

        resolve( conn );
        return;
      }
    );

    conn.on(
      'error',
      function (err) {
        debug( "error: " + err );

        reject();
        return;
      }
    );

    conn.on(
      'end',
      function () {
        debug( "end" );

        //reject();
      }
    );

    conn.on(
      'close',
      function (had_error) {
        debug( "close: " + had_error );

        reject();
        return;
      }
    );

    conn.connect(
      {
        "host": modem_ip,
        "port": 22,
        "username": modem_user,
        "password": modem_pass,
        "debug": function ( msg ) { debug( msg ); },
        "algorithms": {
          "kex": [
            'ecdh-sha2-nistp256',
            'ecdh-sha2-nistp384',
            'ecdh-sha2-nistp521',
            'diffie-hellman-group-exchange-sha256',
            'diffie-hellman-group14-sha1',
            'diffie-hellman-group1-sha1', /* required for modem */
          ],
          "cipher": [
            'aes128-ctr',
            'aes192-ctr',
            'aes256-ctr',
            'aes128-gcm',
            'aes128-gcm@openssh.com',
            'aes256-gcm',
            'aes256-gcm@openssh.com',
            '3des-cbc', /* required for modem */
          ],
        },
      }
    );

    debug( "Starting..." );
  } );
}

function interactive( conn ) {
  return new Promise( function( resolve, reject ) {
    debug( "Starting shell..." );
    conn.exec( '',
      function( err, stream ) {
        if ( err ) {
          reject( err );
          return;
        }

        var data = "";

        stream.on(
          'close',
          function( code, signal ) {
            debug( "close stream " + code + " " + signal );
            conn.end();
            resolve( data );
            return;
          }
        );

        /* use a state machine */
        var stage = "ATPprompt";

        stream.on(
          'data',
          function ( chunk ) {
            debug( "Chunk: " + chunk );
            data += chunk;

            /* determine state we are in and if expected prompt is seen */
            if ( ( stage == "ATPprompt" ) && /^ATP>/m.test( data ) ) {
              debug( "writing sh" );
              data = ""; /* clear buffer */
              stream.write( "sh\n" );
              stage = "shprompt";
              return;
            }

            if ( ( stage == "shprompt" ) && /^#/m.test( data ) ) {
              debug( "writing xdslcmd info --stats" );
              data = ""; /* clear buffer */
              stream.write( "xdslcmd info --stats\n" );
              stage = "shprompt2";
              return;
            }

            if ( ( stage == "shprompt2" ) && /^#/m.test( data ) ) {
              debug( "writing exit" );
              console.log( data );
              data = ""; /* clear buffer */
              stream.write( "exit\n" );
              stage = "ATPprompt2";
              return;
            }

            if ( ( stage == "ATPprompt2" ) && /^ATP>/m.test( data ) ) {
              debug( "writing exit" );
              data = ""; /* clear buffer */
              stream.write( "exit\n" );
              stage = "none";

              /* we now expect connection to close, so resolve when we do */
              conn.on(
                'close',
                function (had_error) {
                  resolve();
                  return;
                }
              );

              return;
            }
          }
        );
      }
    );
  } );
}

/* set out abort timer in case something goes wrong */
setTimeout(
  function () {
    console.error( "ERROR: Timeout" );
    process.exit( 1 );
  },
  5000 /* 5.000 seconds */
);

/* main routine as a chain of promises */
connect().then(
  function ( conn ) {
    return interactive( conn );
  }
).then(
  function () {
    debug( "Done." );
    process.exit();
  }
).catch(
  function ( err ) {
    debug( "Failed to connect." );
    process.exit();
  }
);

Which can be run simply as node xdslcmd.js.

Thread Pool Pattern for Node.JS using Q.js Promises

I wanted to emulate the Thread Pool Pattern in Node.JS using Q.js promises.

Of course Node.JS does not have multiple threads for the user to use. Which is why Q.js is used in the first place – to make concurrency programming easier using the event/callback model that Node.JS implements.

Given an array of functions which return promises I wanted a pattern function that provided a customisable n number of workers. These workers would run asynchronously processing one function from the list at a time, and when each returned promise resolved it would then fetch the next function from the list – until no functions were left to process. When the last worker finished processing it would then resolve the promise returned by the pattern function to the caller. Usually there would be many tasks and limited number of workers.

Why would I want to do this? Let’s say I have a hundred or thousand URLs I want to fetch. If I put them all into a list and called Q.allSettled then Node.JS would attempt to fetch them all at the same time – this could result in excessive delays, a peak in network traffic, or might be plain impolite on the target webserver. Instead it may be better to download a maximum of 4 web pages at any one time.

For simplicity’s sake here is an example. Let’s create a function that returns a promise:

// delayms() - returns a promise that resolves approximately n milliseconds into the future
//   - ms - milliseconds to wait before resolving promise
function delayms( ms ) {
  var deferred = Q.defer();

  setTimeout( function () { deferred.resolve( "Finished after " + ms + "ms" ); }, ms);
  return deferred.promise;
}

Now let’s create a list of functions that return a promise (we could call these functions “tasks”). It’s important that the array contains functions, not the result of a function that returns a promise (i.e. a promise).

For example:

// create array of functions to process
var tasklist = [ ];
for ( var item = 0; item < 12; item++ ) {
  tasklist.push(
    function () {
      console.log( "Going to wait 1 second..." );
      return delayms( 1000 ); // return promise as last statement
    }
  );
}

// let's throw in an error, too
tasklist.push(
  function () {
    throw new Error( "Deliberate error!" );
  }
);

Now we have an array of tasks to process. How about we process them all – but a maximum of 3 running at any one time? We would want to do something like:

var Q = require('./q.js'); // load Q.js library

Q.fcall(
  // call a function, this function resolves immediately
  function () {
    console.log( "Starting" );
  }
).then(
  // do the list of tasks, max concurrency of 3
  function () {
    // return the promise
    return workermodel( 3, tasks );
  }
).then(
  // output the list of results
  function ( results ) {
    console.log( "End: " + JSON.stringify( results, null, 2 ) );
  }
).fail(
  function ( reason ) {
    console.error( "Error: " + reason );
  }
).done();

Well all we need now is the workermodel() function to run through our list of tasks:

// workermodel() - returns a promise that resolves after task list processes with max concurrency
//   - workers - maximum concurrency or number of workers
//   - tasklist - array of functions that return a promise
var debug = true;
function workermodel( workers, tasklist ) {
  var taskidx = 0;
  var workersfree = 0;

  // if no tasks to perform return a completed promise with empty array
  if ( tasklist.length == 0 ) {
    return Q.fcall( function () { return []; } );
  }

  var deferred = Q.defer();

  // if less tasks than workers, limit workers to task size
  if ( tasklist.length < workers ) {
    workers = tasklist.length;
  }

  // results will go into this array
  var resultsarray = new Array( workers );

  var getNextTaskGenerator = function () { }; // place holder

  var startNextTask = function ( innerworkeridx, innertaskidx ) {
    if ( debug )
      console.error( new Date() + " task[" + innertaskidx + "/" + tasklist.length + "] assigned to worker[" + innerworkeridx + "]" );
    var nextTask = Q.fcall(
      function () {
        return tasklist[innertaskidx]();
      }
    );
    nextTask.then(
      function ( value ) {
        if ( debug )
          console.error( new Date() + " task[" + innertaskidx + "] resolved on worker[" + innerworkeridx + "]" );
        resultsarray[innertaskidx] = { state: "fulfilled", value: value };
        return getNextTaskGenerator( innerworkeridx )();
      },
      function ( reason ) {
        if ( debug )
          console.log( new Date() + " task[" + innertaskidx + "] rejected on worker[" + innerworkeridx + "]" );
        resultsarray[innertaskidx] = { state: "rejected", reason: reason };
        return getNextTaskGenerator( innerworkeridx )();
      }
    );

    return nextTask;
  };

  getNextTaskGenerator = function ( workeridx ) {
    return function () {
      if ( debug )
        console.error( new Date() + " getnext task[" + taskidx + "] for worker[" + workeridx + "]" );
      if ( taskidx < tasklist.length ) {
        var nextTask = startNextTask( workeridx, taskidx );
        taskidx++;
        return nextTask;
      } else {
        workersfree++;
        if ( workersfree == workers ) {
          if ( debug )
            console.error( new Date() + " workermodel RESOLVE" );
          deferred.resolve( resultsarray );
        } else {
          if ( debug )
            console.error( new Date() + " no more work but " + ( workers - workersfree ) + " workers busy" );
        }
      }
    };
  };

  // start workers
  for ( var workeridx = 0; workeridx < workers; workeridx++ ) {
    startNextTask( workeridx, taskidx );
    taskidx++;
  }

  if ( debug )
    console.error( new Date() + " RETURNING PROMISE" );
  return deferred.promise;
}

Like Q.allSettled() this function always resolves and when it does the value is a list of associative arrays, each row (related to the function in the tasklist provided) contains a state field of “fulfilled” or “rejected” and if “fulfilled” then the “value” field will contain the result. Otherwise the “reason” field will contain the error.

What is the output of this script (with debugging turned on)?

Starting
Sat Sep 12 2015 19:48:29 GMT+0100 (BST) task[0/13] assigned to worker[0]
Sat Sep 12 2015 19:48:29 GMT+0100 (BST) task[1/13] assigned to worker[1]
Sat Sep 12 2015 19:48:29 GMT+0100 (BST) task[2/13] assigned to worker[2]
Sat Sep 12 2015 19:48:29 GMT+0100 (BST) RETURNING PROMISE
Going to wait 1 second...
Going to wait 1 second...
Going to wait 1 second...
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[0] resolved on worker[0]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) getnext task[3] for worker[0]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[3/13] assigned to worker[0]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[1] resolved on worker[1]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) getnext task[4] for worker[1]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[4/13] assigned to worker[1]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[2] resolved on worker[2]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) getnext task[5] for worker[2]
Sat Sep 12 2015 19:48:30 GMT+0100 (BST) task[5/13] assigned to worker[2]
Going to wait 1 second...
Going to wait 1 second...
Going to wait 1 second...
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[3] resolved on worker[0]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) getnext task[6] for worker[0]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[6/13] assigned to worker[0]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[4] resolved on worker[1]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) getnext task[7] for worker[1]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[7/13] assigned to worker[1]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[5] resolved on worker[2]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) getnext task[8] for worker[2]
Sat Sep 12 2015 19:48:31 GMT+0100 (BST) task[8/13] assigned to worker[2]
Going to wait 1 second...
Going to wait 1 second...
Going to wait 1 second...
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[6] resolved on worker[0]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) getnext task[9] for worker[0]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[9/13] assigned to worker[0]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[7] resolved on worker[1]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) getnext task[10] for worker[1]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[10/13] assigned to worker[1]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[8] resolved on worker[2]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) getnext task[11] for worker[2]
Sat Sep 12 2015 19:48:32 GMT+0100 (BST) task[11/13] assigned to worker[2]
Going to wait 1 second...
Going to wait 1 second...
Going to wait 1 second...
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) task[9] resolved on worker[0]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) getnext task[12] for worker[0]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) task[12/13] assigned to worker[0]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) task[10] resolved on worker[1]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) getnext task[13] for worker[1]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) no more work but 2 workers busy
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) task[11] resolved on worker[2]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) getnext task[13] for worker[2]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) no more work but 1 workers busy
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) task[12] rejected on worker[0]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) getnext task[13] for worker[0]
Sat Sep 12 2015 19:48:33 GMT+0100 (BST) workermodel RESOLVE
End: [
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "fulfilled",
    "value": "Finished after 1000ms"
  },
  {
    "state": "rejected",
    "reason": {}
  }
]

All of the above is my own work. It is free for others to use.

See Also

  • bluebird Javascript promises library has a concurrency option which would appear to be similar in functionality

How To Get Cookies From Node.JS HTTP Response

So you make a HTTP request using the http library and you want the cookies returned by the remote webserver (maybe for a later request, a session cookie for example)?

The approach I ended up taking in Node.JS was to inspect the response.headers[“set-cookie”] field which is set to an array of cookies if they have been given in the server response.

e.g.

var http = require( "http" );
var url = require( "url" );

var urlstring = "http://www.voa.com/";
var parsedurl = url.parse( urlstring );
var options = {
  hostname: parsedurl.hostname,
  port: ( parsedurl.port || 80 ), // 80 by default
  method: 'GET',
  path: parsedurl.path,
  headers: { },
};

var request = http.request(
  options,
  function ( response ) {
    // display returned cookies in header
    var setcookie = response.headers["set-cookie"];
    if ( setcookie ) {
      setcookie.forEach(
        function ( cookiestr ) {
          console.log( "COOKIE:" + cookiestr );
        }
      );
    }

    var data = "";
    response.on(
      "data",
      function ( chunk ) { data += chunk; }
    );

    response.on(
      "end",
      function () {
        console.log( "STATUS:" + response.statusCode );
        console.log( "  DATA:" + data );
      }
    );
  }
);

request.on(
  "error",
  function( err ) {
    console.error( "ERROR:" + err );
  }
);

request.end(); // let request know it is finished sending

This will output cookie information like:

COOKIE:JSESSIONID=7bcc3a128; Path=/

If you want to send cookies with your request add a header to the options you pass the request, e.g.:

var cookies = [
  "JSESSIONID=c2aa6fa21",
  "site_ip=10.18.32.12",
];

options.headers["Cookie"] = cookies.join( "; " );

Writing Rotated Text on a JavaScript Canvas

This is a tutorial on rotating the canvas to produce text in directions other than horizontal. I wrote this because I wanted to write vertical text on a canvas and didn’t know how.

The Trick

You can only write text horizontally on a canvas. But we can rotate a canvas that we write on – then draw our horizontal text on the rotated canvas. When we’re finished we restore the canvas (restore it to its original orientation).

This can be a bit hard to imagine at first so I’ll work through a practical example. It is actually pretty easy once you know the trick.

Tutorial

Skeleton and Blank Canvas

Let’s start with a blank canvas with a border so we can see how big the canvas is.

<html>
  <head>
    <script src="example.js"></script>
  </head>
  <body onload="init();">
    <canvas id="demoCanvas" width="400" height="240">
      Please enable JavaScript
    </canvas>
  </body>
</html>
"use strict";

function init() {
  var canvas = document.getElementById( "demoCanvas" );
	
  var context = canvas.getContext( '2d' );
	
  // draw a box around the canvas
  context.beginPath(); // always start a new line with beginPath
  context.lineWidth = 3;
  context.moveTo( 0, 0 ); // start position
  context.lineTo( canvas.width - 1, 0 );
  context.lineTo( canvas.width - 1, canvas.height - 1 );
  context.lineTo( 0, canvas.height - 1 );
  context.lineTo( 0, 0 );
  context.stroke(); // actually draw the line
}

This results in the blank canvas image:

Blank Canvas

Blank Canvas

Drawing Vertical Text Going Down

If we want to write text going down beginning at the upper left corner (co-ordinate [0,0]) then we have to virtually rotate the paper we are drawing on by holding onto the left-hand corner and rotating it 90 degrees (that’s pi divided by 2 radians) anti-clockwise.

So if we start with the following canvas – and we grab the origin (0, 0) with our finger and thumb:

Grab the original canvas at the top left hand corner

Grab the original canvas at the top left hand corner

… then rotate the canvas 90 degrees anti-clockwise at this point:

Rotate the image at the origin 90 degrees

Rotate the image at the origin 90 degrees

… and once rotated draw on the rotated canvas horizontally at (0, 0). Note that text is always drawn above the selected point.

Draw at the origin horizontally on the rotated canvas

Draw at the origin horizontally on the rotated canvas

When you restore the canvas to the original orientation using the restore() function the text will appear to go down from the top-left hand corner.

So let’s add the following code to our JavaScript init() function:

// start by saving the current context (current orientation, origin)
context.save();

// when we rotate we will be pinching the
// top-left hand corner with our thumb and finger
context.translate( 0, 0 );

// now rotate the canvas anti-clockwise by 90 degrees
// holding onto the translate point
context.rotate( Math.PI / 2 );

// specify the font and colour of the text
context.font = "16px serif";
context.fillStyle = "#ff0000"; // red

// set alignment of text at writing point (left-align)
context.textAlign = "left";

// write the text
context.fillText( "left-aligned 90 deg", 0, 0 );

// now restore the canvas flipping it back to its original orientation
context.restore();

This produces the following on the canvas:

Canvas with vertical text down from top-left corner

Canvas with vertical text down from top-left corner

Drawing Vertical Text Going Up and Right-Aligned

What about text going up that lines up against the top-right?

This time we’ll grab the top-right hand corner of the canvas and rotate it -90 degrees (or 270 degrees) anti-clockwise.

Rotate the canvas holding the top-right hand corner

Rotate the canvas holding the top-right hand corner

Bear in mind that any drawing done is relative from the translate point which in this case is top-right of the original canvas (or bottom-right of the rotated canvas). So when we draw text we’ll again draw to (0, 0) but right-align it so that the text extends to the left of the bottom-right corner of the rotated canvas.

Draw right-aligned text at the origin of the translated canvas

Draw right-aligned text at the origin of the translated canvas

Now for some code:

// save orientation again
context.save();

// hold top-right hand corner when rotating
context.translate( canvas.width - 1, 0 );

// rotate 270 degrees
context.rotate( 3 * Math.PI / 2 );

context.font = "16px serif";
context.fillStyle = "#0000ff"; // blue
context.textAlign = "right";

// draw relative to translate point
context.fillText( "right-aligned 270 deg", 0, 0 );

context.restore();

Now we have a canvas that looks like the following:

Canvas with text going up aligned to the top right

Canvas with text going up aligned to the top right

Text 45 Degrees Down and Centred

So you should have a pretty good handle on this now. Let’s have some fun with 45 degree text sloping downwards but at the centre.

We’ll obviously rotate the canvas from the centre.

Rotate 45 degrees anti-clockwise from the centre

Rotate 45 degrees anti-clockwise from the centre

Then we can draw at the translate point but with centred text:

Draw text at translate point but centre-aligned

Draw text at translate point but centre-aligned

The code to do this:

context.save();
context.translate( canvas.width / 2, canvas.height / 2 );
context.rotate( Math.PI / 4 );
context.font = "16px serif";
context.fillStyle = "#00df00"; // green
context.textAlign = "center";
context.fillText( "center-aligned 45 deg", 0, 0 );
context.restore();

The canvas now looks like the following:

Canvas with 45-degree centred text

Canvas with 45-degree centred text

Node.JS Utility To Launch Processes Remotely Using ssh2 Library

This Node.JS utility was written with the sole purpose of running many processes on remote servers and collating the output (stdout and stderr) line by line such that no two lines would get mixed.

It uses the ssh2 library available for Node.JS.

Note that this opens a single ssh connection to each target server and multiplexes the file transfers and command executions through channels within this single ssh connection. By default OpenSSH allowed 10 simultaneous channels in a ssh connection but versions 5.1 and subsequent introduced a MaxSessions parameter which allows this to be greatly increased (I’ve had it in the thousands).

// node.js SSH process launcher and line logger

var fs = require('fs');
var util = require('util');
var ssh2 = require('/usr/local/node/node_modules/ssh2');

/***
  help_and_exit() - print out help for command line, then exit
***/
function help_and_exit() {
    var helpstr = (
        "Usage:\n" +
        "    node launcher.js -s <ip> [-s <ip> ...] \\\n" +
        "        {-i <identity_key> or -w <password>}\\\n" +
        "        [-u <user{root}>] \\\n" +
        "        [-p <port{22}>] \\\n" +
        "        [-c <command_line>] \\\n" +
        "        [-n <num_of_commands>] \\\n" +
        "        [-g <num_of_seconds>] \\\n" +
        "        [-f <file> -t <target_file>] [-f <file> -t <target_file> ...] \\\n" +
        "        [-x <connect_timeout_seconds>] \\\n" +
        "        [-k <keyword_file>] \\\n" +
        "        [-r] \\\n" +
        "        [-d]\n" +
        "\n" +
        "Notes:\n" +
        "  -g <sec> - spread start of n commands over this many seconds\n" +
        "  -f <file> -t <target_file> - uploads specified in source/dest pairs\n" +
        "  -k <file> - keyword file consists of CSV lines, one line per command\n" +
        "  -c - string may use \${1} \${2} ... \${99} to replace from fields in keyword file\n" +
        "       \${time} - timestamp in milliseconds\n" +
        "       \${index} - unique number per command executed\n" +
        "  -r - randomise starting line from keyword file\n" +
        "\n"
    );

    console.log( "%s", helpstr );
    process.exit( 1 );
}

/***
  process_command_line() - handle command line arguments
***/
function process_command_line() {
    if ( process.argv.length < 3 ) {
        help_and_exit();
    }

    var servers = new Array();
    var identity_file = null;
    var user = "root";
    var password = null;
    var port = 22;
    var command = null;
    var command_count = 1;
    var command_spread = 1; // milliseconds to wait between launches
    var command_spread_period = null;
    var files = new Array();
    var debugging = false;
    var timeout_connect = null;
    var keyword_file = null;
    var keyword_line_randomise = false;

    var idx;
    for ( idx = 2; idx < process.argv.length; idx++ ) {
        if ( process.argv[idx] === "-s" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -s option" );
                help_and_exit();
            }
            servers.push( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-i" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -i option" );
                help_and_exit();
            }
            if ( ! fs.existsSync( process.argv[idx] ) ) {
                console.error( "Error, identity file \"%s\" does not exist", process.argv[idx] );
                help_and_exit();
            }
            identity_file = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-u" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -u option" );
                help_and_exit();
            }
            user = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-w" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -w option" );
                help_and_exit();
            }
            password = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-p" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -p option" );
                help_and_exit();
            }
            if (
                ( parseInt( process.argv[idx] ) < 1 ) ||
                ( parseInt( process.argv[idx] ) > 65535 )
            ) {
                console.error( "Invalid port number supplied to -p option" );
                help_and_exit();
            }
            port = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-c" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -c option" );
                help_and_exit();
            }
            command = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-n" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -n option" );
                help_and_exit();
            }
            if ( parseInt( process.argv[idx] ) < 1 ) {
                console.error( "Invalid command count supplied to -n option" );
                help_and_exit();
            }
            command_count = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-g" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -g option" );
                help_and_exit();
            }
            if ( parseInt( process.argv[idx] ) < 1 ) {
                console.error( "Invalid number of seconds supplied to -g option" );
                help_and_exit();
            }
            command_spread_period = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-f" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -f option" );
                help_and_exit();
            }
            if ( ! fs.existsSync( process.argv[idx] ) ) {
                console.error( "Error, source file \"%s\" does not exist", process.argv[idx] );
                help_and_exit();
            }
            var source_file = process.argv[idx];

            idx++;
            if ( process.argv[idx] !== "-t" ) {
                console.error( "Error, must provide -t option following -f option" );
                help_and_exit();
            }

            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -t option" );
                help_and_exit();
            }
            var target_file = process.argv[idx];

            var collection = { "source": source_file, "target": target_file };
            files.push( collection );

            continue;
        }

        if ( process.argv[idx] === "-d" ) {
            debugging = true;
            continue;
        }

        if ( process.argv[idx] === "-x" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -x option" );
                help_and_exit();
            }
            if ( parseInt( process.argv[idx] ) < 1 ) {
                console.error( "Invalid timeout supplied to -x option" );
                help_and_exit();
            }
            timeout_connect = parseInt( process.argv[idx] );
            continue;
        }

        if ( process.argv[idx] === "-k" ) {
            idx++;
            if ( idx >= process.argv.length ) {
                console.error( "Error, must provide argument to -k option" );
                help_and_exit();
            }
            if ( ! fs.existsSync( process.argv[idx] ) ) {
                console.error( "Error, keyword file \"%s\" does not exist", process.argv[idx] );
                help_and_exit();
            }
            keyword_file = process.argv[idx];
            continue;
        }

        if ( process.argv[idx] === "-r" ) {
            keyword_line_randomise = true;
            continue;
        }
    }

    if ( command_spread_period !== null ) {
        command_spread = parseInt( ( command_spread_period * 1000 ) / command_count );
    }

    if ( servers.length < 1 ) {
        console.error( "Must specify a minimum of one server" );
        help_and_exit();
    }

    if ( ( identity_file === null ) && ( password === null ) ) {
        console.error( "Must specify an identity file or login password" );
        help_and_exit();
    }

    var serverstates = { };
    for ( idx = 0; idx < servers.length; idx++ ) {
        serverstates[ servers[idx] ] = "initial";
    }

    var keyword_lines = null;
    if ( keyword_file ) {
        keyword_lines = [ ];

        // load whole file into memory (simplest)
        var keyword_buffer = fs.readFileSync( keyword_file, { encoding: 'utf8' } );
        // split into array of lines and words
        var regexpLine = new RegExp( "([^\r\n]+)[\r\n]+", "g" );
        while ( 1 ) {
            var mymatch = regexpLine.exec( keyword_buffer );
            if ( mymatch === null )
                break;

            var line = mymatch[1];

            if ( debugging ) {
                console.error( "  - processing line \"%s\"", line );
            }

            // process the line of text
            var words = [ ];
            while ( 1 ) {
                var next = line.indexOf( "," );
                if ( next < 0 ) {
                    words.push( line );
                    break;
                }

                words.push( line.substring( 0, next ) );
                line = line.substring( next + 1 );
            }

            keyword_lines.push( words );
        }
    }

    var keyword_line_number = 0;
    if ( keyword_line_randomise ) {
        keyword_line_number = Math.floor(Math.random() * keyword_lines.length);
    }

    var options = {
        "servers": serverstates,
        "identity": identity_file,
        "user": user,
        "password": password,
        "port": port,
        "command": command,
        "command_count": command_count,
        "command_spread": command_spread,
        "files": files,
        "debug": debugging,
        "timeout": timeout_connect,
        "keyword_file": keyword_file,
        "keyword_lines": keyword_lines,
        "keyword_line_number": keyword_line_number,
        "conns": { }
    };

    return( options );
}

/***
  log_server_state() - print out the states of all the servers
***/
function log_server_state( serverlist ) {
    var idx;
    for ( var key in serverlist ) {
        console.error( "    %s: %s", key, serverlist[key] );
    }
}

/***
  die_if_not_connected() - scan serverlist and die if any not connected
***/
function die_if_not_connected( serverlist ) {
    var idx;
    var connected = true;
    var disconnected_list = new Array();
    for ( var key in serverlist ) {
        if (
            ( serverlist[key] === "initial" ) ||
            ( serverlist[key] === "connected" ) ||
            ( serverlist[key] === "error" )
        ) {
            connected = false;
            disconnected_list.push( key );
        }
    }

    if ( connected )
        return;

    console.error( "ERROR: timeout, server%s failed to connect", ( disconnected_list.length === 1 ? "" : "s" ) );

    log_server_state( serverlist );

    process.exit( 2 );
}

/***
  interpret_command() - replace variables in command string with literals
***/
function interpret_command( options, index ) {
    var cmd = options.command;

    if ( options.keyword_lines ) {
        if ( options.keyword_line_number >= options.keyword_lines.length )
            options.keyword_line_number = 0;

        var words = options.keyword_lines[options.keyword_line_number];

        options.keyword_line_number++;

        var find_substitution = new RegExp( "[$][{]([0-9]+)[}]" );
        while ( 1 ) {
            var result = find_substitution.exec( cmd );
            if ( ! result )
                break;

            var start = result.index;
            var end = start + result[0].length;

            var wanted_idx = result[1] - 1;
            var substitution;
            if ( wanted_idx >= words.length ) {
                substitution = "";
            } else {
                substitution = words[wanted_idx];
            }

            cmd = cmd.substring( 0, start ) + substitution + cmd.substring( end );
        }

        var find_time = new RegExp( "[$][{](time)[}]" );
        while ( 1 ) {
            var result = find_time.exec( cmd );
            if ( ! result )
                break;

            var start = result.index;
            var end = start + result[0].length;

            var substitution = (new Date()).getTime();

            cmd = cmd.substring( 0, start ) + substitution + cmd.substring( end );
        }

        var find_index = new RegExp( "[$][{](index)[}]" );
        while ( 1 ) {
            var result = find_index.exec( cmd );
            if ( ! result )
                break;

            var start = result.index;
            var end = start + result[0].length;

            var substitution = index;

            cmd = cmd.substring( 0, start ) + substitution + cmd.substring( end );
        }
    }

    return( cmd );
}

/***
  ip2num() - convert dotted IP address to integer
***/
function ip2num( ip ) {
    var elements = ip.split( '.' );
    var result = (
        parseInt( elements[0] ) * 256 * 256 * 256 +
        parseInt( elements[1] ) * 256 * 256 +
        parseInt( elements[2] ) * 256 +
        parseInt( elements[3] )
    );

    return( result );
}

/***
   getconnidx() - get index of this connection in array of connections
***/
function getconnidx( options, conn ) {
    var idx = 0;
    for ( var tmpip in options.conns ) {
        if ( conn == options.conns[tmpip] ) {
            return idx;
        }
        idx++;
    }

    return( undefined );
}

/***
  do_exec() - execute command via established SSH connection
***/
function do_exec( conn, options, ip, instance_num ) {
    var uniqueindex = getconnidx( options, conn ) * options.command_count + instance_num;

    var cmdline = interpret_command( options, uniqueindex );

    if ( options.debug ) {
        console.error( "  - command line is \"%s\"", cmdline );
    }

    conn.exec(
        cmdline,
        function ( err, stream ) {
            if ( err ) {
                console.error( "[%s]/%d Error executing \"%s\": %s", ip, instance_num, cmdline, err );
                options.servers[ip] = "error";
                options.command_state[ip][instance_num] = "error";
                check_states( options );
                return;
            } else {
                options.command_state[ip][instance_num] = "exec";
            }

            var buffer_stdout = "";
            var buffer_stderr = "";

            var regexpLine = new RegExp( "([^\r\n]+)[\r\n]+" );

            stream.on(
                'data',
                function (data, extended) {
                    if ( extended === 'stderr' ) {
                        buffer_stderr += data;
                    } else {
                        buffer_stdout += data;
                    }

                    if ( options.debug ) {
                        console.error( "[%s]/%d Data received = %d", ip, instance_num, data.length );
                    }

                    while ( 1 ) {
                        var mymatch = regexpLine.exec( buffer_stderr );
                        if ( mymatch == null )
                            break;

                        var start = mymatch.index;
                        var end = start + mymatch[0].length;
                        var line = mymatch[1];

                        buffer_stderr = buffer_stderr.substring( end );

                        console.log( "[%s]/%d stderr: %s", ip, instance_num, line );
                    }

                    while ( 1 ) {
                        var mymatch = regexpLine.exec( buffer_stdout );
                        if ( mymatch == null )
                            break;

                        var start = mymatch.index;
                        var end = start + mymatch[0].length;
                        var line = mymatch[1];

                        buffer_stdout = buffer_stdout.substring( end );

                        console.log( "[%s]/%d stdout: %s", ip, instance_num, line );
                    }
                }
            );

            stream.on(
                'end',
                function () {
                    if ( options.debug ) {
                        console.error( "[%s]/%d stream end", ip, instance_num );
                    }
                    options.command_state[ip][instance_num] = "completed";
                    check_states( options );
                }
            );

            stream.on(
                'close',
                function () {
                    if ( options.debug ) {
                        console.error( "[%s]/%d stream closed", ip, instance_num );
                    }
                }
            );
            stream.on(
                'exit',
                function (retval, signalName, didCoreDump, description) {
                    if ( retval != null ) {
                        if ( options.debug ) {
                            console.error( "[%s]/%d stream exit = %d", ip, instance_num, retval );
                        }
                        stream.end();
                    } else {
                        if ( options.debug ) {
                            console.error( "[%s]/%d stream exit signal %s%s [%s]", ip, instance_num, signalName, didCoreDump ? " (dumped core)" : "", description );
                        }
                    }
                }
            );
        }
    );
}

/***
  check_states() - state machine engine, checking states, calling next action when ready
***/
function check_states( options ) {
    var all_done = true;

    if ( options.debug ) {
        for ( var sip in options.servers ) {
            console.error( "  %s: %s", sip, options.servers[sip] );
        }
    }

    if ( options.command_state ) {
        for ( var ip in options.command_state ) {
            if (
                ( options.servers[ip] === "closed" ) ||
                ( options.servers[ip] === "error" )
            ) {
                continue;
            }

            if ( options.debug ) {
                var strstate = "";
                var statectr = 0;
                for ( statectr = 0; statectr < options.command_state[ip].length; statectr++ ) {
                    if ( strstate.length != 0 ) {
                        strstate += ", ";
                    }
                    strstate += "\n  " + statectr + ":" + options.command_state[ip][statectr];
                }
                console.error( "[%s] command_state = %s", ip, strstate );
            }

            var all_completed = true;
            var instance = 0;
            for ( inst = 0; inst < options.command_state[ip].length; inst++ ) {
                var state = options.command_state[ip][inst];
                if (
                    ( state === "error" ) ||
                    ( state === "completed" )
                ) {
                    ;
                } else {
                    all_completed = false;
                }
            }

            if ( all_completed ) {
                options.conns[ip].end();
                options.servers[ip] = "closed";
            }
        }
    }

    for ( var ip in options.servers ) {
        if (
            ( options.servers[ip] === "error" ) ||
            ( options.servers[ip] === "closed" )
        ) {
            ;
        } else {
            all_done = false;
        }
    }

    if ( all_done ) {
        if ( options.debug ) {
            console.error( "- all completed" );
        }
        process.exit( 0 );
    }

    var all_awaiting_cmd = true;
    var any_closed = false;
    var any_awaiting = false;

    for ( var ip in options.servers ) {
        if (
            ( options.servers[ip] === "awaiting_cmd" )
        ) {
            any_awaiting = true;
        } else {
            all_awaiting_cmd = false;
            if (
                ( options.servers[ip] === "closed" ) ||
                ( options.servers[ip] === "error" )
            ) {
                // connection closed before "awaiting_command"
                any_closed = true;
            }
        }
    }

    if ( all_awaiting_cmd ) {
        options["command_state"] = { };
        for ( var ip in options.servers ) {
            options.servers[ip] = "executing";

            if ( options.command != null ) {
                var conn = options.conns[ip];
                if ( ! conn ) {
                    console.error( "Programmer error, conn should exist" );
                    process.exit( 98 );
                }

                // execute command number of times
                var cmdidx;
                options.command_state[ip] = new Array( options.command_count );
                for ( cmdidx = 0; cmdidx < options.command_count; cmdidx++ ) {
                    options.command_state[ip][cmdidx] = "initial";
                }
                var timeoffset = 0;
                for ( cmdidx = 0; cmdidx < options.command_count; cmdidx++ ) {
                    setTimeout(
                        function (conn, options, ip, cmdidx) {
                            do_exec( conn, options, ip, cmdidx );
                        },
                        timeoffset,
                        conn, options, ip, cmdidx
                    );
                    timeoffset += options.command_spread;
                }
            } else {
                options.servers[ip] = "closed";
                check_states( options );
            }
        }
    } else if ( any_closed && any_awaiting ) {
        console.error( "Error, connection has been prematurely closed" );
        process.exit( 2 );
    }
}

/***
  do_sftp() - perform file upload via established SSH connection
***/
function do_sftp( conn, options, ip, filesremaining, callback ) {
    if ( options.debug ) {
        console.error( "[%s] doing sftp %d", ip, filesremaining.length );
    }

    if ( filesremaining.length == 0 ) {
        callback();
        return;
    }

    var nextfile = filesremaining.shift();
    if ( options.debug ) {
        console.error( "[%s] next file is %s", ip, nextfile.source );
    }

    conn.sftp(
        function (err, sftp) {
            if ( err ) {
                console.error( "[%s] Error, problem starting SFTP: %s", ip, err );
                process.exit( 3 );
            }

            if ( options.debug ) {
                console.error( "[%s] SFTP started", ip );
            }

            // start upload
            var readStream = fs.createReadStream( nextfile.source );
            var writeStream = sftp.createWriteStream(
                nextfile.target,
                { flags: 'w', encoding: null, mode: 0666 }
            );

            writeStream.on(
                'close',
                function () {
                    if ( options.debug ) {
                        console.error( "[%s] end transfer of \"%s\"", ip, nextfile.source );
                    }
                    sftp.end();
                    do_sftp( conn, options, ip, filesremaining, callback );
                }
            );

            readStream.pipe( writeStream );
        }
    );
}

/***
  do_ready_action_awaiting_cmd() - change state to waiting for cmd and check state machine
***/
function do_ready_action_awaiting_cmd( conn, options, ip ) {
    options.servers[ip] = "awaiting_cmd";
    check_states( options );
}

/***
 do_ready_action_upload_files() - upload files if required
***/
function do_ready_action_upload_files( conn, options, ip ) {
    if ( options.files.length > 0 ) {
        var filesremaining = options.files.slice( 0 );

        do_sftp(
            conn, options, ip, filesremaining,
            function () {
                do_ready_action_awaiting_cmd( conn, options, ip );
            }
        );
    } else {
        // no files to transfer, change state
        do_ready_action_awaiting_cmd( conn, options, ip );
    }
}

/***
  connect_to_server() - set up a connection to a server
***/
function connect_to_server( options, ip ) {
    var conn = new ssh2();

    conn.setMaxListeners(0);

    conn.on(
        'connect',
        function () {
            if ( options.debug ) {
                console.error( "[%s] connect", ip );
            }
            options.servers[ip] = "connected";
        }
    );

    conn.on(
        'ready',
        function () {
            if ( options.debug ) {
                console.error( "[%s] ready", ip );
            }
            options.servers[ip] = "ready";
            options.conns[ip] = conn;

            do_ready_action_upload_files(
                conn, options, ip
            );
        }
    );

    conn.on(
        'error',
        function (err) {
            if ( options.debug ) {
                console.error( "[%s] error: %s", ip, err );
            }
            options.servers[ip] = "error";
            check_states( options );
        }
    );

    conn.on(
        'end',
        function () {
            if ( options.debug ) {
                console.error( "[%s] end", ip );
            }
            options.servers[ip] = "closed";
            check_states( options );
        }
    );

    conn.on(
        'close',
        function (had_error) {
            if ( options.debug ) {
                console.error( "[%s] end: %s", ip, had_error );
            }
            options.servers[ip] = "closed";
            check_states( options );
        }
    );

    var connectoptions = {
        "host": ip,
        "port": options.port,
        "username": options.user
    };

    if ( options.identity !== null ) {
        connectoptions["privateKey"] = fs.readFileSync( options.identity );
    } else if ( options.password !== null ) {
        connectoptions["password"] = options.password;
    } else {
        console.error( "No authentication option for SSH provided" );
        process.exit( 1 );
    }

    conn.connect( connectoptions );
}

/***
  connect_to_servers() - connect to all the servers
***/
function connect_to_servers( options ) {
    var idx;
    for ( var ip in options.servers ) {
        connect_to_server( options, ip );
    }
}

/***
  main() - process command line, then connect to servers
***/
function main() {
    var options = process_command_line();

    // establish ssh connections to all servers, wait until this is achieved
    if ( options.debug ) {
        console.error( "options = %s", util.inspect( options ) );
    }

    // timeout to blow up if connections haven't been established
    if ( options.timeout ) {
        setTimeout(
            function () {
                die_if_not_connected( options.servers );
            },
            options.timeout * 1000
        );
    }

    // establish connection to servers
    connect_to_servers( options );
}

main();

Example Usage

One could have a local jar file they wish to run remotely on four servers. And they want to run 20 instances of this on each server but with the instances started spread out over 60 seconds. In total this means that 80 instances of the program will be started spread out over 60 seconds.

The jar file exists in ~/myjar.jar on the local server and before executing this jar file it must be uploaded to /tmp/ on the remote servers. Command execution won’t begin until the file has been successfully transferred to all the remote servers.

/usr/local/node/bin/node sshlauncher.js \
  -s test_server1.my.net \
  -s test_server2.my.net \
  -s test_server3.my.net \
  -s test_server4.my.net \
  -u testuser \
  -w testpassword \
  -x 15 \
  -g 60 \
  -n 20 \
  -f ~/myjar.jar -t /tmp/myjar.jar \
  -c 'java -jar /tmp/myjar.jar'

Debugging mode can be enabled with a -d flag so you can see when connections take place and file transfers, etc.

You may wish to provide a text comma-separated value file so that you can provide unique or different parameters to your executed command instances (e.g. for testing login processes to a site).

If you wanted to launch 4 instances of a program with unique usernames you could have a file named ~/keywords.txt containing:

user1,password1
user2,password2
user3,password3
user4,password4

and a command line of:

/usr/local/node/bin/node sshlauncher.js \
  -s test_server1.my.net \
  -s test_server2.my.net \
  -s test_server3.my.net \
  -s test_server4.my.net \
  -u testuser \
  -w testpassword \
  -x 15 \
  -g 60 \
  -n 20 \
  -f ~/myjar.jar -t /tmp/myjar.jar \
  -c 'java -jar /tmp/myjar.jar ${1} ${2}' \
  -k ~/keywords.txt

If your CSV (comma-separated value) keyword file is long enough (say, a file full of dictionary words) you can provide the -r flag to start reading from a random line in the file (e.g. when trying to test a cached server and you want to feed it unique words to circumvent caching).

PhantomJS window.setTimeout Appears to be Ignored

This could equally apply to a Node.JS program as much as a PhantomJS program.

You’ve got a JavaScript program that has many setTimeout() or window.setTimeout() calls. Yet your program appears to be failing for no apparent reason. Even after scrutinising your program – perhaps running it through esvalidate and concluding that there are no syntax errors – your script just appears to terminate without any reason. And those terminations happen before an expected setTimeout() callback is due to trigger.

You wouldn’t be alone if you suffered this problem. Quite a few people have contributed to a Github PhantomJS issue on this topic. Many assuming their call to setTimeout() failed.

Here is an example script that appears to terminate without reason before the final window.setTimeout() call:

function step3() {
    console.log( "+step3()" );
    window.setTimeout(
        function () {
            console.log( "-exiting phantomJS" );
            phantom.exit(1);
        }, 1000
    );
}

function step2() {
    console.log( "+step2()" );
    window.setTimeout(
        function () {
            step3();
            phantom.exit(1);
        }, 1000
    );
}

function step1() {
    console.log( "+step1()" );
    window.setTimeout(
        function () {
            step2();
        }, 1000
    );
}

step1();

When run this script outputs:

user@host:~$ phantomjs /tmp/test.js
+step1()
+step2()
+step3()

This script appeared to terminate without displaying the final “-exiting phantomJS” log message. So what went wrong? After all, step3() was executed as expected.

The problem is that an unwanted call to phantom.exit() was made after the call to step3() in step2(). What actually happens is that step3() is called, and the setTimeout() is made, but then the function returns back and executes the call to phantom.exit() – which terminates the script before the final setTimeout() callback has a chance to trigger.

How Can I Avoid This Silent Killer?

You can protect yourself from having this problem in the future. Never, never call phantom.exit() directly from your script. Instead add the following function and call this instead whenever you want to terminate your script:

function quit( reason, value ) {
    console.log( "QUIT: " + reason );
    phantom.exit( value );
}

Why do this? If your program aborts you want to know WHY. If you provide a unique message at every point in which your program can exit then you can quickly identify when a mistaken quit() call has interrupted your expected setTimeout() callback.

Here’s the fixed program:

function quit( reason, value ) {
    console.log( "QUIT: " + reason );
    phantom.exit( value );
}

function step3() {
    console.log( "+step3()" );
    window.setTimeout(
        function () {
            console.log( "-exiting phantomJS" );
            quit( "finished", 0 );
        }, 1000
    );
}

function step2() {
    console.log( "+step2()" );
    window.setTimeout(
        function () {
            step3();
            // phantom.exit(1); WAS CAUSING PROBLEM
        }, 1000
    );
}

function step1() {
    console.log( "+step1()" );
    window.setTimeout(
        function () {
            step2();
        }, 1000
    );
}

step1();

This outputs:

user@host:~$ phantomjs /tmp/test.js
+step1()
+step2()
+step3()
-exiting phantomJS
QUIT: finished

JavaScript Function to Call Functions In Sequence

This post is targeted towards PhantomJS where you may want to simulate a user entering data into several fields using the keyboard but pausing between each field entry.

The following function takes a list of functions and executes them with a fixed delay between each event. It takes two parameters:

  • events – array of functions to call sequentially
  • delay – delay, in milliseconds, to pause before calling each event
function space_out_events( events, delay, index ) {
    index = ( typeof( index ) !== 'undefined' ) ? index : 0;

    window.setTimeout(
        function () {
            events[index]();
            if ( index < ( events.length - 1 ) ) {
                space_out_events( events, delay, index + 1 );
            }
        }, delay
    );
}

Example Usage

You have a form on a page to fill out. You want to simulate all the information entered by keyboard with a pause of half a second (500ms), then tab key, then pause another 500ms, before filling out the next field.

    space_out_events(
        [
            function(){ page.sendEvent( 'keypress', config.email ); },
            function(){ page.sendEvent( 'keypress', page.event.key.Tab ); },
            function(){ page.sendEvent( 'keypress', "Testing123" ); },
            function(){ page.sendEvent( 'keypress', page.event.key.Tab ); },
            function(){ page.sendEvent( 'keypress', "Testing123" ); },
            function(){ page.sendEvent( 'keypress', page.event.key.Tab ); },
            // ignore title
            function(){ page.sendEvent( 'keypress', page.event.key.Tab ); },
            function(){ page.sendEvent( 'keypress', "John" ); },
            function(){ page.sendEvent( 'keypress', page.event.key.Tab ); },
            function(){ page.sendEvent( 'keypress', "Smith" );},
            function(){ page.sendEvent( 'keypress', page.event.key.Tab ); },
            // ignore mobile number
            function(){ page.sendEvent( 'keypress', page.event.key.Tab ); },
            // ignore day of birth
            function(){ page.sendEvent( 'keypress', page.event.key.Tab ); },
            // ignore month of birth
            function(){ page.sendEvent( 'keypress', page.event.key.Tab ); },
            // ignore year of birth
            function(){ page.sendEvent( 'keypress', page.event.key.Tab ); },
            // ignore screen name
            function(){ page.sendEvent( 'keypress', page.event.key.Tab ); },
            function(){ if ( config.debug ) { page.render( "filled_form.png" ); } },
            function(){ do_click_next_step( page ); }
        ], 500
    );

The technique could also be used with Node.JS.

Node.JS String Length Undefined

This issue arose using node.js version v0.10.3.

Here’s some example code:

function writeStringLength( str ) {
  console.log( "str = \"" + str + "\"" );
  console.log( "str.length is " + str.length );

  var copy = str;
  console.log( "copy.length is " + copy.length );

  var copyObj = new String( str );
  console.log( "copyObj.length is " + copyObj.length );
}

var now = new Date();
writeStringLength( now.getFullYear() );

This outputs:

user@localhost:/tmp$ /usr/local/node/bin/node /tmp/brokenstring.js
str = "2013"
str.length is undefined
copy.length is undefined
copyObj.length is 4

So – why does JavaScript think the string doesn’t have a defined value for the length property? According to the Node.JS manual entry for the String object “JavaScript distinguishes between String objects and primitive string values”. Conversion of primitive strings to String objects should happen automatically but clearly in this case node.js is unable to determine that the function parameter is a string that needs “boxing” into a string object. By forcing a new String object to be created we can be sure the length property will return a useful value.

Note that the values returned by the Date() object functions causes this problem – perhaps because Node.JS doesn’t know for certain what kind of values these functions returns (could they be from an external library?). Whereas if you try and pass an ordinary string (e.g. “example”) to writeStringLength() then the parameter will, as expected, have a defined length property. Perhaps this is because node.js knows the parameter is an explicitly defined string in this example.

Sampling Pixels on Web Page Using PhantomJS

Concerned your website might be hacked and defaced? How do you protect against this? One way might be to get a “snapshot” of your webpage as an image and then test pixels to see if they match the colours you expect at particular static locations. Not perfect but you’ll know quickly if someone wasn’t subtle when replacing the front-page content.

The technique I propose is this:

  • load the web page
  • render to a base-64 string
  • load “about:blank”
  • dynamically add an image to the DOM
  • assign the base-64 string to the image
  • dynamically add a canvas to the DOM
  • draw the image to the canvas’ context
  • sample points from the canvas’ context

First things first: load the web page and render it to a base 64 string:

var system = require('system');
var page = require('webpage').create();
page.onResourceError = function(resourceError) {
    page.reason = resourceError.errorString;
    page.reason_url = resourceError.url;
};

page.viewportSize = {
    'width': 1000,
    'height': 768
};

page.onConsoleMessage = function(msg) {
    system.stderr.writeLine('console: ' + msg);
};

page.open(
    'http://www.google.com/',
    function (status) {
        if ( status !== 'success' ) {
            console.log(
                "Error opening url \"" + page.reason_url
                + "\": " + page.reason
            );
            phantom.exit( 1 );
            return;
        }

        // take snapshot after 2 seconds to allow page AJAX to run
        window.setTimeout(
            function () {
                var imgBase64 = page.renderBase64();
                check_image( imgBase64 ); // function defined below
            },
            2 * 1000 // give page 2 seconds to execute before render
        );
    }
);

Now we have our base-64 string. Before proceeding I recommend that the page gets cleared to a blank page (so that the tested page’s JavaScript can’t wreak havoc on our image test function):

page.open(
    'about:blank',
    function ( status ) {
        check_image( imgBase64 ); // function defined below
    }
);

The pixel checking code must be done in a page.evaluate() function call because it will be getting the page to run JavaScript:

function check_image_js( imgString ) {
    // returns hex colour in ttrrggbb format (tt is opacity)
    function get_color( context, x, y ) {
        // return 2-character hex string (0-255)
        function decToHex( dd ) {
            var hex = Number(dd).toString(16);
            hex = "00".substr( 0, 2 - hex.length ) + hex;
            return hex;
        }

        var imgd = context.getImageData( x, y, 1, 1 ).data;
        var colorstr = (
            decToHex( imgd[3] ) + decToHex( imgd[0] ) +
            decToHex( imgd[1] ) + decToHex( imgd[2] )
        );

        return colorstr;
    }

    // list of pixels to check
    var pixels = [
        { x: 272, y: 135, col: 'ff000000', desc: 'Black in title' },
        { x: 97, y: 9, col: 'ffffffff', desc: 'White in button' }
    ];

    // create canvas
    var canvas = document.createElement( 'canvas' );
    canvas.width = 1000;
    canvas.height = 768;
    var context = canvas.getContext( '2d' );
    
    // load image
    var img = new Image();
    img.onload = function () { context.drawImage( img, 0, 0 ); };
    img.src = "data:image/png;base64," + imgString;

    // give time for image to load
    window.setTimeout(
        function () {
            var problemDetected = false;

            console.log( "- doing pixels" );

            var idx;
            for ( idx = 0; idx < pixels.length; idx++ ) {
                var row = pixels[idx];

                var actual = get_color( context, row.x, row.y );
                if ( actual !== row.col ) {
                    problemDetected = true;
                    console.log( "ERROR mismatch: expected \"" + row.col + "\", got \"" + actual + "\" (ttrrggbb) for \"" + row.desc + "\" at (" + row.x + ", " + row.y + ")" );
                } else {
                    console.log( "  - matched colour \"" + actual + "\" (ttrrggbb) for \"" + row.desc + "\" at (" + row.x + ", " + row.y + ")" );
                }
            }
        }, 500
    );
}

// call check_image_js in page.evaluate sandbox
function check_image( imgString ) {
    page.evaluate(
        function ( callback, string ) {
            callback( string );
        },
        check_image_js,
        imgString
    );
}

Getting To The Bottom Of Why A PhantomJS Page Load Fails

For this post I’m using PhantomJS version 1.9.

Quite frustratingly I occasionally have a call to page.open() where my callback receives a status of “fail”. This isn’t very helpful as it doesn’t describe what went wrong. Was it a SSL handshake problem (using the --ignore-ssl-errors=true command line argument may solve such problems)? Something else?

Unfortunately the PhantomJS API, at present, doesn’t appear to have an ability to determine the reason for the failure of the page to load. But there are a number of callbacks we can hook into to generate a lot of debugging messages to allow us to determine the reason for the failure.

Simplified Reason Tracking

Just before calling page.open() add the following code (after creating the page variable):

    page.onResourceError = function(resourceError) {
        page.reason = resourceError.errorString;
        page.reason_url = resourceError.url;
    };

Now you can print out the reason for a problem in your page.open() callback, e.g.:

var page = require('webpage').create();

page.onResourceError = function(resourceError) {
    page.reason = resourceError.errorString;
    page.reason_url = resourceError.url;
};

page.open(
    "http://www.nosuchdomain/",
    function (status) {
        if ( status !== 'success' ) {
            console.log(
                "Error opening url \"" + page.reason_url
                + "\": " + page.reason
            );
            phantom.exit( 1 );
        } else {
            console.log( "Successful page open!" );
            phantom.exit( 0 );
        }
    }
);

This script outputs the following:

Error opening url "http://www.nosuchdomain/": Host www.nosuchdomain not found

Detailed Logging

Just before calling page.open() add the following code (after creating the page variable):

    page.onResourceRequested = function (request) {
        system.stderr.writeLine('= onResourceRequested()');
        system.stderr.writeLine('  request: ' + JSON.stringify(request, undefined, 4));
    };

    page.onResourceReceived = function(response) {
        system.stderr.writeLine('= onResourceReceived()' );
        system.stderr.writeLine('  id: ' + response.id + ', stage: "' + response.stage + '", response: ' + JSON.stringify(response));
    };

    page.onLoadStarted = function() {
        system.stderr.writeLine('= onLoadStarted()');
        var currentUrl = page.evaluate(function() {
            return window.location.href;
        });
        system.stderr.writeLine('  leaving url: ' + currentUrl);
    };

    page.onLoadFinished = function(status) {
        system.stderr.writeLine('= onLoadFinished()');
        system.stderr.writeLine('  status: ' + status);
    };

    page.onNavigationRequested = function(url, type, willNavigate, main) {
        system.stderr.writeLine('= onNavigationRequested');
        system.stderr.writeLine('  destination_url: ' + url);
        system.stderr.writeLine('  type (cause): ' + type);
        system.stderr.writeLine('  will navigate: ' + willNavigate);
        system.stderr.writeLine('  from page\'s main frame: ' + main);
    };

    page.onResourceError = function(resourceError) {
        system.stderr.writeLine('= onResourceError()');
        system.stderr.writeLine('  - unable to load url: "' + resourceError.url + '"');
        system.stderr.writeLine('  - error code: ' + resourceError.errorCode + ', description: ' + resourceError.errorString );
    };

    page.onError = function(msg, trace) {
        system.stderr.writeLine('= onError()');
        var msgStack = ['  ERROR: ' + msg];
        if (trace) {
            msgStack.push('  TRACE:');
            trace.forEach(function(t) {
                msgStack.push('    -> ' + t.file + ': ' + t.line + (t.function ? ' (in function "' + t.function + '")' : ''));
            });
        }
        system.stderr.writeLine(msgStack.join('\n'));
    };

It is important that before this block gets called after the page and system variables are defined, e.g.:

var system = require('system');
var page = require('webpage').create();