/*******************************************************************/
/***                                                             ***/
/***   Tokenizer.js - JavaScript String Tokenizer Function       ***/
/***                                                             ***/
/***   Version   : 0.2                                           ***/
/***   Date      : 01.05.2005                                    ***/
/***   Copyright : 2005 Adrian Zentner                           ***/
/***   Website   : http://www.adrian.zentner.name/               ***/
/***                                                             ***/
/***   This library is free software. It can be freely used as   ***/
/***   long as this this copyright notice is not removed.        ***/
/***                                                             ***/
/*******************************************************************/

String.prototype.tokenize = tokenize;

function tokenize()
  {
     var input             = "";
     var separator         = " ";
     var trim              = "";
     var ignoreEmptyTokens = true;

     try {
       String(this.toLowerCase());
     }
     catch(e) {
       window.alert("Tokenizer Usage: string myTokens[] = myString.tokenize(string separator, string trim, boolean ignoreEmptyTokens);");
       return;
     }

     if(typeof(this) != "undefined")
       {
          input = String(this);
       }

     if(typeof(tokenize.arguments[0]) != "undefined")
       {
          separator = String(tokenize.arguments[0]);
       }

     if(typeof(tokenize.arguments[1]) != "undefined")
       {
          trim = String(tokenize.arguments[1]);
       }

     if(typeof(tokenize.arguments[2]) != "undefined")
       {
          if(!tokenize.arguments[2])
            ignoreEmptyTokens = false;
       }

     var array = input.split(separator);

     if(trim)
       for(var i=0; i<array.length; i++)
         {
           while(array[i].slice(0, trim.length) == trim)
             array[i] = array[i].slice(trim.length);
           while(array[i].slice(array[i].length-trim.length) == trim)
             array[i] = array[i].slice(0, array[i].length-trim.length);
         }

     var token = new Array();
     if(ignoreEmptyTokens)
       {
          for(var i=0; i<array.length; i++)
            if(array[i] != "")
              token.push(array[i]);
       }
     else
       {
          token = array;
       }

     return token;
  }