/******************************************************************************
libprozilla - a download accelerator library
Copyright (C) 2001 Kalum Somaratna
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
******************************************************************************/
/* Main include-file. */
/* $Id$ */
#ifndef PROZILLA_H
#define PROZILLA_H
#include <stdio.h>
#include <pthread.h>
#include <sys/time.h>
#include "netrc.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef enum {
/*Connect establishment related values */
NOCONERROR, HOSTERR, CONSOCKERR, CONERROR,
CONREFUSED, ACCEPTERR, ACCEPTOK, BINDERR, BINDOK, LISTENERR, LISTENOK,
SERVERCLOSECONERR, CONPORTERR,
/* URL handling related value */
URLOK, URLHTTP, URLFTP, URLFILE, URLUNKNOWN, URLBADPORT,
URLBADHOST, URLBADPATTERN,
/* HTTP related values */
NEWLOCATION, HOK, HEOF, HERR, HAUTHREQ, HAUTHFAIL,
HTTPNSFOD,
/*FTP related value */
FTPOK, FTPLOGINC, FTPLOGREFUSED, FTPPORTERR,
FTPNSFOD, FTPRETROK, FTPUNKNOWNTYPE, FTPUNKNOWNCMD, FTPSIZEFAIL,
FTPERR, FTPRESTFAIL, FTPACCDENIED,
FTPPWDERR,
FTPINVPASV, FTPNOPASV, FTPCONREFUSED, FTPCWDFAIL, FTPPWDFAIL,
FTPSERVCLOSEDATLOGIN,
/*FTP parsing related values */
FTPPARSEOK, FTPPARSENOTEXIST, FTPPARSEFAIL,
/*Error values that can happen due to failed proxie request */
GATEWAYTIMEOUT, SERVICEUNAVAIL, BADGATEWAY, INTERNALSERVERR,
UNKNOWNREQ,
/*File handling related return values. */
FOPENERR, FWRITEERR,
RETROK,
/*Download related retrn values. */
DLINPROGRESS, DLABORTED, DLDONE, CANTRESUME, DLLOCALFATAL,
DLREMOTEFATAL,
/*FTPSEARCH+ping related return values. */
FTPSINPROGRESS, MASSPINGINPROGRESS, FTPSFAIL, MASSPINGDONE,
/*PING related values */
PINGOK, PINGTIMEOUT,
/*Misc Value */
RETRFINISHED, READERR,
PROXERR, WRITEERR,
FILEISDIR,
MIRINFOK, MIRPARSEOK, MIRPARSEFAIL, FILEGETOK,
/*Related to file joining */
JOININPROGRESS, JOINDONE, JOINERR
} uerr_t;
#define FTP_BUFFER_SIZE 2048
#define HTTP_BUFFER_SIZE 2048
#define MAX_MSG_SIZE 1024
#define DEFAULT_FTP_PORT 21
#define DEFAULT_HTTP_PORT 80
/* This is used when no password is found or specified. */
#define DEFAULT_FTP_USER "anonymous"
#define DEFAULT_FTP_PASSWD "billg@hotmail.com"
/* The D/L ed fragments will be saved to files with this extension.
* E.g.: gnu.jpg.prz0, gnu.jpg.prz1 etc... */
#define DEFAULT_FILE_EXT ".prz"
/* The extension for the log file created. */
#define DEFAULT_LOG_EXT ".log"
#define DEFAULT_USER_AGENT "Prozilla"
typedef char longstring[1024];
/* Callback message function. */
typedef void (*message_proc)(const char *msg, void *cb_data);
/* Structure containing info on a URL. */
typedef struct _urlinfo {
char *url; /* Unchanged URL. */
uerr_t proto; /* URL protocol. */
char *host; /* Extracted hostname. */
unsigned short port;
char ftp_type;
char *path, *dir, *file; /* Path, dir and file (properly decoded). */
char *user, *passwd; /* For FTP. */
char *referer; /* The source from which the request
URI was obtained. */
} urlinfo;
typedef enum {
USERatSITE,
USERatPROXYUSERatSITE,
USERatSITE_PROXYUSER,
PROXYUSERatSITE,
LOGINthenUSERatSITE,
OPENSITE,
SITESITE,
HTTPPROXY,
FTPGATE,
WINGATE
} proxy_type;
typedef enum {
IDLE = 0,
CONNECTING,
LOGGININ,
DOWNLOADING,
COMPLETED,
LOGINFAIL,
CONREJECT,
REMOTEFATAL,
LOCALFATAL,
TIMEDOUT,
MAXTRYS
} dl_status;
typedef enum {
NOT_FOUND,
REGULAR_FILE,
DIRECTORY,
SYMBOLIC_LINK,
UNKNOWN
} file_type_t;
typedef struct {
off_t len; /* Received length. */
off_t contlen; /* Expected length. */
int res; /* The result of last read. */
/* -1 = Accept ranges not found.
0 = Accepts range is none.
1 = Accepts ranges. */
int accept_ranges;
char *newloc; /* New location (redirection). */
char *remote_time; /* Remote time-stamp string. */
char *error; /* Textual HTTP error. */
int statcode; /* Status code. */
} http_stat_t;
typedef struct {
urlinfo proxy_url;
char *username;
char *passwd;
proxy_type type;
boolean use_proxy;
} proxy_info;
typedef struct libprozinfo {
int argc;
char **argv;
boolean debug_mode;
/* For netrc. */
netrc_entry *netrc_list;
boolean use_netrc;
char *home_dir;
char *ftp_default_user;
char *ftp_default_passwd;
char *dl_dir;
char *output_dir;
char *log_dir;
boolean ftp_use_pasv;
proxy_info *ftp_proxy;
proxy_info *http_proxy;
boolean http_no_cache;
/* the default timeout for all the connection types (ctrl, data etc) */
struct timeval conn_timeout;
struct timeval conn_retry_delay;
int max_attempts;
long max_bps_per_dl;
} libprozinfo;
extern libprozinfo libprozrtinfo;
typedef struct response_line {
char *line;
struct response_line *next;
} response_line;
typedef struct connection_t {
/* struct which contains the parsed url info. It includes the remote file,
path,protocol etc. */
urlinfo u;
/* The error status of the connection. */
uerr_t err;
/* Proxy specific info. */
proxy_info *ftp_proxy;
proxy_info *http_proxy;
/* Netrc. */
boolean use_netrc;
/* FTP specific info. */
boolean ftp_use_pasv;
struct timeval xfer_timeout;
struct timeval conn_timeout;
struct timeval ctrl_timeout;
unsigned char pasv_addr[6];
int ctrl_sock;
int data_sock;
int listen_sock;
/* Additional info about what this URL is. */
/* FIXME Should this be in the url_info struct? */
file_type_t file_type;
/* The lines that the server returned. */
response_line *serv_ret_lines;
/* Does the server support resume? */
boolean resume_support;
/* The file name to save the data to locally. */
char *localfile;
/* Pointer to file that we will be saving the data to locally. */
FILE *fp;
/* Tells whether to open the file for appending or for writing etc.
Used for adding resume support. */
char *file_mode;
/* FIXME Add an enum here to say whether run mode is resume or normal etc.
and remove the file mode. */
off_t remote_startpos;
off_t orig_remote_startpos;
off_t remote_endpos;
off_t remote_bytes_received;
off_t main_file_size;
/* The permanent base offset from the beginning of the file, put in
anticipation of making the threads download to a single file. */
off_t local_base_offset;
/* Indicates the startpos of the localfile. It is always 0 in normal mode
and can be any positive value in resume mode. */
off_t local_startpos;
/* The start position at the beginning of the download. */
off_t orig_local_startpos;
/* long bytes_xferred; */
dl_status status;
char *szBuffer;
/* Tells the download thread whether to abort the download or not. */
boolean abort;
/* Information about the connection's start and end time. */
struct timeval time_begin;
struct timeval time_end;
/* Info about whether to retry the thread or not. */
boolean retry;
/* The number of attempts to try to complete a connection. 0 means unlimited
connection attempts. */
int max_attempts;
/* The number of attempts that this connection has been tried */
int attempts;
/* The time when to try to restart the connection. */
struct timeval retry_delay;
/* Each connection will acquire this mutex before changing state. */
pthread_mutex_t *status_change_mutex;
/*This will be broadcast when the connection starts connecting */
pthread_cond_t connecting_cond;
/* User agent for HTTP. */
char *user_agent;
http_stat_t hs;
message_proc msg_proc;
/*additional callback data whcih is specified by the user
that is passed to msg_proc */
void *cb_data;
/* Indicates whether a conenction is running or not */
int running;
/*Mutex used to lock acesss to data in this struct
that is accesed/written by other threads */
pthread_mutex_t access_mutex;
/*This indicates that we should use the pragma no-cache directive for http proxies */
boolean http_no_cache;
/*The rate of this connection whcih is calcualted */
long rate_bps;
/*We limit the connections speed to this */
long max_allowed_bps;
} connection_t;
typedef enum {
UNTESTED = 0, RESPONSEOK, NORESPONSE, ERROR
} ftp_mirror_stat_t;
typedef enum {
LYCOS, FILESEARCH_RU
} ftpsearch_server_type_t;
typedef struct {
char *path;
boolean valid;
} mirror_path_t;
typedef struct ftp_mirror {
char *server_name;
mirror_path_t *paths;
char *file_name;
char *full_name;
char *file_size;
struct timeval tv;
int milli_secs;
int num_paths;
ftp_mirror_stat_t status;
int copied;
boolean resume_supported;
int max_simul_connections;
} ftp_mirror_t;
typedef struct {
off_t file_size;
char *file_name;
connection_t *connection;
ftpsearch_server_type_t server_type;
ftp_mirror_t *mirrors;
int num_mirrors;
uerr_t err;
boolean info_running;
boolean mass_ping_running;
pthread_mutex_t access_mutex;
pthread_t info_thread;
pthread_t mass_ping_thread;
int max_simul_pings;
struct timeval ping_timeout;
urlinfo *requested_url;
} ftps_request_t;
typedef struct download_t {
urlinfo u;
char *dl_dir;
char *log_dir;
char *output_dir;
connection_t **pconnections;
pthread_t *threads;
pthread_mutex_t status_change_mutex;
int num_connections;
/* Optional will be called back with info about download. */
message_proc msg_proc;
/*additional callback data which is specified by the user
that is passed to msg_proc */
void *cb_data;
off_t main_file_size;
boolean resume_mode;
struct timeval start_time;
/*Does this DL support resume? */
boolean resume_support;
/*This contains the building status, 1 = building,0 build finished, -1 error occured */
int building;
/*This is the percentage of the file that is been built currently */
float file_build_percentage;
int max_simul_connections;
/*Mutex used to lock acesss to data in this struct
that is accesed/written by other threads */
pthread_mutex_t access_mutex;
/* The message that is returned when the file build process is finished
*/
char *file_build_msg;
/*Max mps for this download */
long max_allowed_bps;
ftps_request_t *ftps_info;
boolean using_ftpsearch;
pthread_t join_thread;
} download_t;
typedef struct {
/* the number of connections that this download was started with */
int num_connections;
/*I have added these newly */
int version; /*The version of this logfile */
/*Indicates whether we have got info about the files size, final URL etc */
boolean got_info;
off_t file_size;
int url_len; /*The length in bytes of the url text stred in the log file */
char *url;
int reserved[30];
} logfile;
/*Structs for logfile handling */
typedef struct {
char *host;
int port;
struct timeval timeout;
struct timeval ping_time;
int sock;
uerr_t err;
} ping_t;
/*Functions for URL parsing */
uerr_t proz_parse_url(const char *url, urlinfo * u, boolean strict);
urlinfo *proz_copy_url(urlinfo * u);
void proz_free_url(urlinfo * u, boolean complete);
/*Functions that set values which will apply for all conenctions. */
int proz_init(int argc, char **argv);
void proz_shutdown(void);
void proz_die(const char *message, ...);
void proz_set_http_proxy(proxy_info * proxy);
void proz_set_ftp_proxy(proxy_info * proxy);
void proz_use_ftp_proxy(boolean use);
void proz_use_http_proxy(boolean use);
void proz_set_connection_timeout(struct timeval *timeout);
void proz_set_connection_retry_delay(struct timeval *delay);
void proz_set_download_dir(char *dir);
void proz_set_logfile_dir(char *dir);
void proz_set_output_dir(char *dir);
char *proz_get_libprozilla_version();
/*Functions for loggind debug messages */
void proz_debug(const char *format, ...);
void proz_debug_delete_log();
/*Functions which are for handling a single connection. */
connection_t * proz_connection_init(urlinfo * url, pthread_mutex_t * mutex);
void proz_connection_set_url(connection_t * connection, urlinfo *url);
char *proz_connection_get_status_string(connection_t * connection);
off_t proz_connection_get_total_bytes_got(connection_t * connection);
void proz_get_url_info_loop(connection_t * connection, pthread_t *thread);
off_t proz_connection_get_total_remote_bytes_got(connection_t *
connection);
void proz_connection_set_msg_proc(connection_t * connection,
message_proc msg_proc, void *cb_data);
void proz_connection_free_connection(connection_t * connection,
boolean complete);
dl_status proz_connection_get_status(connection_t * connection);
boolean proz_connection_running(connection_t * connection);
/*Functions which are for handling a download */
download_t *proz_download_init(urlinfo * u);
int proz_download_setup_connections_no_ftpsearch(download_t * download,
connection_t *
connection,
int req_connections);
void proz_download_start_downloads(download_t * download,
boolean resume);
int proz_download_load_resume_info(download_t * download);
void proz_download_stop_downloads(download_t * download);
boolean proz_download_all_dls_status(download_t * download,
dl_status status);
boolean proz_download_all_dls_err(download_t * download, uerr_t err);
boolean proz_download_all_dls_filensfod(download_t * download);
boolean proz_download_all_dls_ftpcwdfail(download_t * download);
off_t proz_download_get_total_bytes_got(download_t * download);
uerr_t proz_download_handle_threads(download_t * download);
int proz_download_prev_download_exists(download_t * download);
float proz_download_get_average_speed(download_t * download);
int proz_download_delete_dl_file(download_t * download);
void proz_download_wait_till_all_end(download_t * download);
off_t proz_download_get_total_remote_bytes_got(download_t * download);
void proz_download_set_msg_proc(download_t * download,
message_proc msg_proc, void *cb_data);
off_t proz_download_get_est_time_left(download_t * download);
void proz_download_free_download(download_t * download,
boolean complete);
int proz_download_target_exist(download_t * download);
int proz_download_delete_target(download_t * download);
/* Functions related to handling the logfile created for a download */
int proz_log_read_logfile(logfile * lf, download_t * download,
boolean load_con_info);
int proz_log_delete_logfile(download_t * download);
int proz_log_logfile_exists(download_t * download);
char *proz_strerror(uerr_t error);
/*Ftpsearch releated */
ftps_request_t * proz_ftps_request_init(
urlinfo * requested_url, off_t file_size,
char *ftps_loc,
ftpsearch_server_type_t server_type,
int num_req_mirrors);
void proz_get_complete_mirror_list(ftps_request_t * request);
boolean proz_request_info_running(ftps_request_t * request);
boolean proz_request_mass_ping_running(ftps_request_t * request);
void proz_mass_ping(ftps_request_t * request);
void proz_sort_mirror_list(ftp_mirror_t * mirrors, int num_servers);
void proz_cancel_mirror_list_request(ftps_request_t * request);
int proz_download_setup_connections_ftpsearch(download_t * download,
connection_t * connection,
ftps_request_t * request,
int req_connections);
void proz_cancel_mass_ping(ftps_request_t * request);
/*Misc functions */
int proz_timeval_subtract(struct timeval *result, struct timeval *x,
struct timeval *y);
/*Funcs related to joining the downloaded file portions */
void proz_download_join_downloads(download_t * download);
uerr_t proz_download_get_join_status(download_t *download);
float proz_download_get_file_build_percentage(download_t *download);
void proz_download_cancel_joining_thread(download_t * download);
void proz_download_wait_till_end_joining_thread(download_t * download);
#ifdef __cplusplus
}
#endif
#endif /* PROZILLA_H */