
/*-------------------------------------------------------------*/
/*
 Note : currently, this spider does not support :
    - cookies
    - NETWOX_HTMLTAG_ELMTATTR_URIFORMAT_LISTSPACED (<object archive=...>)
    - NETWOX_HTMLTAG_ELMTATTR_URIFORMAT_LISTCOMMA (<applet archive=...>)
    - IMG ISMAP (<a href="t"><img src=u ismap></A>)
    - POST form with its variables
    - Javascript/VBscript (to search new urls)
*/

/*-------------------------------------------------------------*/
typedef enum {
  NETWOX_WEBSPIDER_LINKCONV_NONE = 0,
  NETWOX_WEBSPIDER_LINKCONV_WEB = 1
} netwox_webspider_linkconv;

/*-------------------------------------------------------------*/
typedef enum {
  NETWOX_WEBSPIDER_LOGLEVEL_NONE = 0,
  NETWOX_WEBSPIDER_LOGLEVEL_NORMAL = 1,
  NETWOX_WEBSPIDER_LOGLEVEL_DOT = 2,
  NETWOX_WEBSPIDER_LOGLEVEL_THREAD = 3,
  NETWOX_WEBSPIDER_LOGLEVEL_SCALE = 4
} netwox_webspider_loglevel;

/*-------------------------------------------------------------*/
typedef struct {
  netwib_buf localrootdir;
  netwox_httpclictx httpclictx;
  netwib_ring *plisturltodownload;
  netwib_hash *plistallurl;
  netwib_ring *pliststarturl;
  netwib_uint32 maxrecursion;
  netwib_uint32 maxthreads;
  netwib_uint32 maxsegmentlen;
  netwib_uint32 maxtotallen;
  netwib_uint32 millisleep;
  netwib_bool aclurlset;
  netwox_rules *paclurl;
  netwib_bool getimage;
  netwib_bool getmultimedia;
  netwib_bool getcode;
  netwox_webspider_linkconv linkconv;
  netwox_webspider_loglevel loglevel;
  netwib_uint32 totalurl;
} netwox_webspider;

/*-------------------------------------------------------------*/
netwib_err netwox_webspider_init(netwox_webspider *pwebspider);
netwib_err netwox_webspider_close(netwox_webspider *pwebspider);

/*-------------------------------------------------------------*/
netwib_err netwox_webspider_add(netwox_webspider *pwebspider,
                                netwib_constbuf *purl);
netwib_err netwox_webspider_add_canon(netwox_webspider *pwebspider,
                                      netwib_constbuf *purl);

/*-------------------------------------------------------------*/
netwib_err netwox_webspider_run(netwox_webspider *pwebspider);

/*-------------------------------------------------------------*/
netwib_err netwox_webspider_relink(netwox_webspider *pwebspider);

/*-------------------------------------------------------------*/
netwib_err netwox_webspider_index(netwox_webspider *pwebspider);

