/*
 * CSpider.js
 * $Id: CSpider.js,v 1.4 2004/07/12 10:17:58 bclary Exp $
 */

/* ***** BEGIN LICENSE BLOCK *****
 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Original Code is Netscape code.
 *
 * The Initial Developer of the Original Code is
 * Netscape Corporation.
 * Portions created by the Initial Developer are Copyright (C) 2003
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s): Bob Clary <bclary@netscape.com>
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 2 or later (the "GPL"), or
 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 * in which case the provisions of the GPL or the LGPL are applicable instead
 * of those above. If you wish to allow use of your version of this file only
 * under the terms of either the GPL or the LGPL, and not to allow others to
 * use your version of this file under the terms of the MPL, indicate your
 * decision by deleting the provisions above and replace them with the notice
 * and other provisions required by the GPL or the LGPL. If you do not delete
 * the provisions above, a recipient may use your version of this file under
 * the terms of any one of the MPL, the GPL or the LGPL.
 *
 * ***** END LICENSE BLOCK ***** */

/*
  CSpider

  aUrl     url of the first page to visit
  aDepth   depth in terms of pages to visit
  aPageLoader WDocumentLoader to load pages
  aRestrictUrl restrict spider to follow links that are prefixed by aUrl
*/

var gCSpiderOnLoadCallbackDelay = 20;
var gSecurityMessage = 'CSpider requires security privileges to operate.\nPlease see Help for more details.';

function CSpider(/* String */ aUrl, 
                 /* String */ aDomain,
                 /* Boolean */ aRestrictUrl,
                 /* Number */ aDepth, 
                 /* WDocumentLoader */ aPageLoader,
                 /* Seconds */ aOnLoadTimeoutInterval)
{

  if (aUrl.indexOf('http://') == -1 && aUrl.indexOf('https://') == -1)
  {
    aUrl = 'http://' + aUrl; 
  }

  this.mUrl = aUrl;

  if (aDomain)
  {
    this.mDomain = aDomain;
  }
  else
  {
    // remove extraneous leading parts of the url
    // to allow restricted urls to stay in the domain

    if (aUrl.indexOf('http://') != -1)
    {
      this.mDomain = aUrl.substr('http://'.length);
    }
    else 
    {
      this.mDomain = aUrl.substr('https://'.length);
    }
    
    if (this.mDomain.indexOf('www.') != -1)
    {
      this.mDomain = this.mDomain.substr('www.'.length);
    }
  }

  this.mRestrictUrl = aRestrictUrl;
  this.mDepth  = aDepth;
  this.mPageLoader = aPageLoader;
  this.mCallWrapperOnLoadPage = null;
  this.mCallWrapperOnLoadPageTimeout = null;
  this.mCallWrapperLoadPage = null;
  this.mCallWrapperPause = null;
  this.mOnLoadTimeoutInterval = (aOnLoadTimeoutInterval || 60) * 1000;
  this.init(aUrl);

}

CSpider.handlePageLoad = function(/* CFormData */ loaderFormData)
{
  var spiderid = loaderFormData.getValue('spiderid');
  var callwrapper = CCallWrapper.mPendingCalls[spiderid];
  // if call wrapper is not defined, the page load was cancelled
  // and the WDocumentLoader is calling us
  if (callwrapper) 
  {
    callwrapper.execute();
  }
}

CSpider.prototype.init = function(aUrl)
{
  this.mPagesVisited  = [];
  this.mPagesPending  = [{ mDepth: 0, mUrl: aUrl }];
  this.mPageHash = {};
  this.mPageHash[aUrl] = true;
  this.mState = 'initialized';
};

CSpider.prototype.run =
function()
{
  if (this.mDepth > -1)
  {
    this.init(this.mUrl);
    this.mState = 'running';
    if (!this.mOnStart())
    {
      this.mState = 'initialized';
      return;
    }
    this.loadPage();
  }
};

CSpider.prototype.restart =
function()
{
  if (this.mState == 'paused' || this.mState == 'timeout')
  {
    this.mState = 'running';
    if(!this.mOnRestart())
    {
      this.mState = 'paused';
      return;
    }
    this.loadPage();
  }
};

CSpider.prototype.pause =
function()
{
  this.mState = 'pausing';

  if (this.mCallWrapperOnLoadPage || 
      this.mCallWrapperOnLoadPageTimeout)
  {
    this.mCallWrapperPause = new CCallWrapper(this, gCSpiderOnLoadCallbackDelay, 'pause');
    CCallWrapper.asyncExecute(this.mCallWrapperPause);
  }
  else
  {
    this.mCallWrapperPause = null;
    this.mState = 'paused';
    this.mOnPause();
  }
};

CSpider.prototype.stop =
function ()
{
  this.mState = 'stopped';

  if (this.mCallWrapperOnLoadPage && this.mCallWrapperOnLoadPage.mTimerId)
  {
    this.mCallWrapperOnLoadPage.cancel();
    this.mCallWrapperOnLoadPage = null;
  }
  
  if (this.mCallWrapperOnLoadPageTimeout && this.mCallWrapperOnLoadPageTimeout.mTimerId)
  {
    this.mCallWrapperOnLoadPageTimeout.cancel();
    this.mCallWrapperOnLoadPageTimeout = null;
  }
  
  if (this.mCallWrapperLoadPage && this.mCallWrapperLoadPage.mTimerId)
  {
    this.mCallWrapperLoadPage.cancel();
    this.mCallWrapperLoadPage = null;
  }
  
  this.mOnStop();
};

CSpider.prototype.mOnStart = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnBeforePage = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnAfterPage = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnPageTimeout = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnStop = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnPause = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.mOnRestart = 
function() 
{ 
  // override this
  return true;
};

CSpider.prototype.addPage =
function(href)
{
  if (!href)
  {
    return;
  }

  if (href.indexOf('http') != 0)
  {
    return;
  }

  if (this.mRestrictUrl && href.indexOf(this.mDomain) == -1)
  {
    return;
  }

  href = href.toLowerCase();

  // only spider http protocols
  if (href.search(/^http:/) == -1 && href.search(/^https:/) == -1)
  {
    return;
  }

  // don't load bad extensions
  if (
      href.search(/\.ads$/) != -1 || 
      href.search(/\.adp$/) != -1 || 
      href.search(/\.app$/) != -1 || 
      href.search(/\.bas$/) != -1 || 
      href.search(/\.bat$/) != -1 || 
      href.search(/\.bin$/) != -1 || 
      href.search(/\.chm$/) != -1 ||
      href.search(/\.cmd$/) != -1 ||
      href.search(/\.com$/) != -1 ||
      href.search(/\.cpl$/) != -1 ||
      href.search(/\.crt$/) != -1 ||
      href.search(/\.csh$/) != -1 ||
      href.search(/\.dmg$/) != -1 ||
      href.search(/\.exe$/) != -1 || 
      href.search(/\.fxp$/) != -1 || 
      href.search(/\.hlp$/) != -1 || 
      href.search(/\.hta$/) != -1 ||
      href.search(/\.inf$/) != -1 ||
      href.search(/\.ins$/) != -1 ||
      href.search(/\.isp$/) != -1 ||
      href.search(/\.js$/)  != -1 ||
      href.search(/\.jse$/) != -1 ||
      href.search(/\.gz$/)  != -1 || 
      href.search(/\.ksh$/) != -1 ||
      href.search(/\.lnk$/) != -1 || 
      href.search(/\.mda$/) != -1 ||
      href.search(/\.mdb$/) != -1 ||
      href.search(/\.mde$/) != -1 ||
      href.search(/\.mdt$/) != -1 ||
      href.search(/\.mdw$/) != -1 ||
      href.search(/\.mdz$/) != -1 ||
      href.search(/\.msc$/) != -1 ||
      href.search(/\.msi$/) != -1 ||
      href.search(/\.msp$/) != -1 ||
      href.search(/\.mst$/) != -1 ||
      href.search(/\.ops$/) != -1 ||
      href.search(/\.pcd$/) != -1 ||
      href.search(/\.pif$/) != -1 || 
      href.search(/\.prf$/) != -1 || 
      href.search(/\.prg$/) != -1 || 
      href.search(/\.reg$/) != -1 ||
      href.search(/\.scf$/) != -1 || 
      href.search(/\.scr$/) != -1 || 
      href.search(/\.sct$/) != -1 || 
      href.search(/\.shb$/) != -1 || 
      href.search(/\.shs$/) != -1 || 
      href.search(/\.url$/) != -1 || 
      href.search(/\.vb$/)  != -1 || 
      href.search(/\.vbe$/) != -1 || 
      href.search(/\.vbs$/) != -1 || 
      href.search(/\.wsc$/) != -1 ||
      href.search(/\.wsf$/) != -1 ||
      href.search(/\.wsh$/) != -1 ||
      href.search(/\.tar$/) != -1 || 
      href.search(/\.tgz$/) != -1 || 
      href.search(/\.zip$/) != -1 
      )
  {
    return;
  }

  var hashIndex = href.indexOf('#');
  if (hashIndex != -1)
  {
    href = href.substr(0, hashIndex);
  }

  if (typeof(this.mPageHash[href]) != 'undefined' && this.mPageHash[href])
  {
    return;
  }

  if (this.mCurrentUrl.mDepth + 1 <= this.mDepth)
  {
    this.mPageHash[href] = true;
    this.mPagesVisited.push(href);
    this.mPagesPending.push( { mDepth: this.mCurrentUrl.mDepth + 1, mUrl: href } );
  }
};

CSpider.prototype.loadPage = 
function ()
{
  if (navigator.product == 'Gecko')
  {
    var e;
    try
    {
      netscape.security.PrivilegeManager.enablePrivilege('UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite');
    }
    catch(e)
    {
      alert(gSecurityMessage);
      this.stop();
      return;
    }
  }

  if (this.mState != 'running')
  {
    this.mCallWrapperLoadPage.cancel();
    this.mCallWrapperLoadPage = null;
    return;
  }

  this.mCurrentUrl = this.mPagesPending.pop();
  
  if (!this.mCurrentUrl)
  {
    this.stop();
    return;
  }

  if (this.mCurrentUrl.mDepth > this.mDepth)
  {
    this.mCallWrapperLoadPage = new CCallWrapper(this, gCSpiderOnLoadCallbackDelay, 'loadPage');
    CCallWrapper.asyncExecute(this.mCallWrapperLoadPage);
    return;
  }

  if (!this.mOnBeforePage())
  {
    this.mState = 'paused';
    return;
  }

  this.mCallWrapperOnLoadPage = new CCallWrapper(this, gCSpiderOnLoadCallbackDelay, 'onLoadPage');
  var loaderUrl = '?callbackfunction=CSpider.handlePageLoad&amp;spiderid=' + this.mCallWrapperOnLoadPage.mId + '&amp;file=' + escape(this.mCurrentUrl.mUrl);
  this.mPageLoader.location.search = loaderUrl;

  this.mCallWrapperOnLoadPageTimeout = new CCallWrapper(this, this.mOnLoadTimeoutInterval, 'onLoadPageTimeout');
  CCallWrapper.asyncExecute(this.mCallWrapperOnLoadPageTimeout);
};

CSpider.prototype.onLoadPageTimeout = 
function ()
{
  if (this.mCallWrapperOnLoadPage)
  {
    this.mCallWrapperOnLoadPage.cancel();
    this.mCallWrapperOnLoadPage = null;
  }

  this.mCallWrapperOnLoadPageTimeout = null;

  if (!this.mOnPageTimeout())
  {
    this.mState = 'timeout';
    return;
  }

  // this.loadPage();

  this.mCallWrapperLoadPage = new CCallWrapper(this, gCSpiderOnLoadCallbackDelay, 'loadPage');
  CCallWrapper.asyncExecute(this.mCallWrapperLoadPage);
};

CSpider.prototype.onLoadPage = 
function ()
{
  this.mCallWrapperOnLoadPage = null;

  if (this.mCallWrapperOnLoadPageTimeout)
  {
    this.mCallWrapperOnLoadPageTimeout.cancel();
    this.mCallWrapperOnLoadPageTimeout = null;
  }

  if (this.mState == 'stopped')
  {
    return;
  }

  if (navigator.product == 'Gecko')
  {
    var e;
    try
    {
      netscape.security.PrivilegeManager.enablePrivilege('UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite');
    }
    catch(e)
    {
      alert(gSecurityMessage);
      this.stop();
      return;
    }
  }

  this.mDocument = this.mPageLoader.frames[0].document;

  var i;
  var links  = this.mDocument.links;
  if (links)
  {
    var length = links.length;
    var href;

    for (i = 0; i < length; ++i)
    {
      href = links[i].href;
      if (href)
      {
        this.addPage(href);
      }
    }

    links = this.mDocument.getElementsByTagName('frame');
    length = links.length;

    for (i = 0; i < length; ++i)
    {
      href = links[i].src;
      if (href)
      {
        this.addPage(href);
      }
    }

    links = this.mDocument.getElementsByTagName('iframe');
    length = links.length;

    for (i = 0; i < length; ++i)
    {
      href = links[i].src;
      if (href)
      {
        this.addPage(href);
      }
    }
  }

  if (!this.mOnAfterPage())
  {
    this.mState = 'paused';
    return;
  }

  //this.loadPage();

  this.mCallWrapperLoadPage = new CCallWrapper(this, gCSpiderOnLoadCallbackDelay, 'loadPage');
  CCallWrapper.asyncExecute(this.mCallWrapperLoadPage);
};