package page.config;

import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;

import org.wikiwebserver.core.ConfigManager;
import org.wikiwebserver.core.Privilege;
import org.wikiwebserver.core.WareHouse;
import org.wikiwebserver.core.WikiMap;
import org.wikiwebserver.handler.http.HTTPException;
import org.wikiwebserver.handler.http.HTTPHeaders;
import org.wikiwebserver.handler.http.HTTPRequest;
import org.wikiwebserver.handler.http.HTTPResponse;
import org.wikiwebserver.handler.http.interfaces.*;
import org.wikiwebserver.util.IPToCountry;

import page.tools.entity.Browser;
import page.tools.entity.Comment;
import page.tools.entity.NodeData;
import page.tools.entity.Payment;
import page.tools.entity.ProtectedStorable;
import page.tools.entity.User;
import page.tools.stats.Status;
/**
 * Logs lots of information about an HTTP connection.
 * The information is stored within the WareHouse as logs and indexed maps.
 * 
 * @author Michael Gardiner
 *
 */
public class SiteMonitor implements HTTPMonitor, Runnable {
    
    public static final int MAXIMUM_STATS_STORE_SIZE = 1000;

    // Period of time between browser or user requests needed to be
    // logged as a separate visit.
    public static final long VISIT_GAP = 30*60*1000;
    
    // Maintains active requests and responses
    private static Map<HTTPRequest, HTTPResponse> activeRequests = 
    	new HashMap<HTTPRequest, HTTPResponse>();
    

    // How often to put transfer data into persistent store
    private static long TRANSFER_DATA_UPDATE_PERIOD = 500;
    
    // If this number of requests are waiting to be processed
    // force the thread adding additional requests to yield CPU.
    // This is important because connections could saturate CPU
    // and cause these lists to fill heap space.
    private static int ENCOURAGE_DATA_PROCESSSING = 1000;
    
    // If no connections occur in this period then stop the data processing thread.
    // A new class loader could result in multiple instances of site monitor, so
    // we should automatically stop idle instances.
    private static final long IDLE_PERIOD_BEFORE_STOPPING =
        2 * ConfigManager.getLong("service-test-period");
    
    private static long lastConnectionTime;
    private static final AtomicLong bytesReadAcc;
    private static final AtomicLong bytesWrittenAcc;    
    private static long lastTransferDataUpdateTime;
    private static List<String> ipsPendingProcessing;
    // Maintains requests and responses that need to be analysed
    private static Map<HTTPRequest, HTTPResponse> requestsPendingProcessing;
    private static Object dataProcessingTrigger = new Object();
    private static Thread dataProcessingThread;
    private static SiteMonitor backgroundProcessor;
    static {
        lastConnectionTime = System.currentTimeMillis();
        bytesReadAcc = new AtomicLong(0);
        bytesWrittenAcc = new AtomicLong(0);    
        
    	ipsPendingProcessing = new LinkedList<String>();
    	requestsPendingProcessing = new HashMap<HTTPRequest, HTTPResponse>();

    }
    
    public static void startProcessingThread() {
        if (backgroundProcessor == null) {
            backgroundProcessor = new SiteMonitor();
        }
        dataProcessingThread = new Thread(backgroundProcessor, "SiteMonitor Data Processor");
        dataProcessingThread.start();
    }
    
    public void run() {
        
        long time = System.currentTimeMillis();        

        // This site monitor should stop operating when a new class loader is phased in
    	while (time < lastConnectionTime + IDLE_PERIOD_BEFORE_STOPPING) {
    		
    		synchronized (dataProcessingTrigger) {
	    		try {
	    		    dataProcessingTrigger.wait(TRANSFER_DATA_UPDATE_PERIOD);
	    		} catch (InterruptedException ignored) { } 
    		}

    		if (time > lastTransferDataUpdateTime + TRANSFER_DATA_UPDATE_PERIOD) {	    		
	    		
	    		long bytesRead = bytesReadAcc.getAndSet(0);    	
	    		long bytesWritten = bytesWrittenAcc.getAndSet(0);  
	    		
	        	int[][] times = getCounterTimes();   	    		
	    		incrementCounter("dataRead", bytesRead);
	        	updateBytesReadCounter("Second", times[0][0], bytesRead);
		        updateBytesReadCounter("Minute", times[0][1], bytesRead);
		        updateBytesReadCounter("Hour",   times[0][2], bytesRead);   
		        updateBytesReadCounter("Day",    times[0][3], bytesRead);        
		        updateBytesReadCounter("Month",  times[0][4], bytesRead);
		        
	    		incrementCounter("dataWritten", bytesWritten);
	        	updateBytesWrittenCounter("Second", times[0][0], bytesWritten);
	        	updateBytesWrittenCounter("Minute", times[0][1], bytesWritten);
	        	updateBytesWrittenCounter("Hour",   times[0][2], bytesWritten);   
	        	updateBytesWrittenCounter("Day",    times[0][3], bytesWritten);        
	        	updateBytesWrittenCounter("Month",  times[0][4], bytesWritten);
	        	
	        	clearNextCounters(times[1]);
	        	updateRates();
	        	
	        	lastTransferDataUpdateTime = time;
    		}
	        

    		List<String> ips = ipsPendingProcessing;
    		synchronized (ips) {
    			if (ips.size() > 0) {
		    		ipsPendingProcessing = new LinkedList<String>();	    			
		    		Iterator<String> i = ips.iterator();
		    		while (i.hasNext()) {
		    			String ip = i.next();
		    			processIPData(ip);
		    		}
    			}
    		}

        	Map<HTTPRequest, HTTPResponse> requests = requestsPendingProcessing;
        	synchronized (requests) {
	        	if (requests.size() > 0) {
		        	requestsPendingProcessing = new HashMap<HTTPRequest, HTTPResponse>();		        		
		    		Iterator<Map.Entry<HTTPRequest, HTTPResponse>> i = requests.entrySet().iterator();
		    		while (i.hasNext()) {
		    			Map.Entry<HTTPRequest, HTTPResponse> entry = i.next();
		    			
		    	        List<String> details = getRequestDetails(entry.getKey(), entry.getValue());
		    	        WareHouse.logData("requests", requestHeadings, details); 
		    	        
		    	        List<String> gl = new LinkedList<String>();
		    	        gl.add(details.get(8));
                                gl.add(details.get(9));
		    	        gl.add(details.get(10));
		    	        gl.add(details.get(13));
		    	        gl.add(details.get(16));
		    	        gl.add(details.get(19));
		    	        gl.add(details.get(20));
		    	        WareHouse.logData("requests_gltail", requestGlTailHeadings, gl);
		    	        
		    			processRequestData(entry.getKey(), entry.getValue());		    	        
		    		}
	        	}
        	}
        	
        	time = System.currentTimeMillis();
    	}
    	
        System.out.println("Warning: " + dataProcessingThread.getName() + " stopped.");    	
	} 	
    
    
    
    public void incrementNumBytesRead(long bytes) {
    	bytesReadAcc.addAndGet(bytes);
    } 
    
    public void incrementNumBytesWritten(long bytesTransferred) {
    	bytesWrittenAcc.addAndGet(bytesTransferred);
    }         
    
    public void logConnection(String sourceAddress) {
        synchronized (ipsPendingProcessing) {
    		ipsPendingProcessing.add(sourceAddress);
    	}       
        
        lastConnectionTime = System.currentTimeMillis();        
        if (dataProcessingThread == null || !dataProcessingThread.isAlive()) {
            startProcessingThread();
        }        
    }
    
    public void requestStarted(HTTPRequest request, HTTPResponse response) {
    	synchronized (activeRequests) {
    		activeRequests.put(request, response);
    	}  	
    }
    
    public void responseComplete(HTTPRequest request, HTTPResponse response) {
    	synchronized (activeRequests) {
    		activeRequests.remove(request);
    	}    	
    	synchronized (requestsPendingProcessing) {
    		if (requestsPendingProcessing.size() < ENCOURAGE_DATA_PROCESSSING ) {
    			// Add to the queue of things to do
    			requestsPendingProcessing.put(request, response);
    		}
    		else {
    			System.out.println("Warning: " + dataProcessingThread.getName() + " is saturated.");
    			// Current thread should do the work
    		    logRequest(request, response);    			
    			processRequestData(request, response);
    		}
    	}
		synchronized (dataProcessingTrigger) {
		    dataProcessingTrigger.notify();
		}    	
    }
    
    private void processIPData(String sourceAddress) {
    	
        incrementCounter("numConnections", 1);
        incrementCounter("ConnectionAddresses", sourceAddress, 1);   
        String country = IPToCountry.getCountryName(sourceAddress);
        if (country != null) {
            incrementCounter("ConnectionCountries", country, 1); 
        }
    }    
    
    private void processRequestData(HTTPRequest request, HTTPResponse response) {
        
        try {
        	String sourceAddress = request.getSourceAddress();
            String browserID = request.getHeaders().getRequestCookies().get("browserID");
            String userID = request.getHeaders().getRequestCookies().get("userID");
            String nodeID = request.getHeaders().getFirst("X-Node-ID");
            
            long requestTime = request.getStartTime();
            
            // Requests mapped to browsers      
            if (browserID != null) {
                Browser browser = Browser.getBrowserById(browserID);
                if (browser != null) {
                    storeEntityDetails(browser, sourceAddress, request, response);
                    if (userID != null) {
                        // A browser may be used by multiple users
                        browser.put(userID, requestTime, "UserIDs");   
                    }
                    
                    putStatistic("BrowserAccessTimes", browserID, requestTime); 
                    incrementCounter("BrowserRequests", browserID, 1);
                    Number visits = (Number) browser.get("numVisits");
                    putStatistic("BrowserVisits", browserID, visits);
                    Number downloaded = (Number) browser.get("bytesDownloaded");
                    putStatistic("BrowserBytesDownloaded", browserID, downloaded);
                }
            }
            
            // Requests mapped to users
            if (userID != null) {
                User user = User.getUserById(userID);
                if (user != null) {
                    
                    storeEntityDetails(user, sourceAddress, request, response);
                    if (browserID != null) {
                        // A user may sign in using multiple browsers
                        user.put(browserID, requestTime, "BrowserIDs");  
                    }
                    
                    putStatistic("UserAccessTimes", userID, requestTime); 
                    incrementCounter("UserRequests", userID, 1);
                    Number visits = (Number) user.get("numVisits");
                    putStatistic("UserVisits", userID, visits);
                    Number downloaded = (Number) user.get("bytesDownloaded");
                    putStatistic("UserBytesDownloaded", userID, downloaded);     
                }
            }
            
            // Requests mapped to nodes      
            if (nodeID != null) {
                NodeData nodeData = NodeData.getNodeDataById(nodeID);
                if (nodeData != null) {
                    storeEntityDetails(nodeData, sourceAddress, request, response);
                    if (userID != null) {
                        // A browser may be used by multiple users
                        nodeData.put(userID, requestTime, "UserIDs");   
                    }
                    
                    putStatistic("NodeAccessTimes", nodeID, requestTime); 
                    incrementCounter("NodeRequests", nodeID, 1);
                    Number visits = (Number) nodeData.get("numVisits");
                    putStatistic("NodeVisits", nodeID, visits);
                    Number downloaded = (Number) nodeData.get("bytesDownloaded");
                    putStatistic("NodeBytesDownloaded", nodeID, downloaded);
                    Number uploaded = (Number) nodeData.get("bytesUploaded");
                    putStatistic("NodeBytesUploaded", nodeID, uploaded);                    
                }
            }            
    
            String referer = request.getHeaders().getFirst("Referer");
            if (referer != null && referer.trim().length() > 0) {
                
                boolean isExternal = false;
                String host = request.getHeaders().getFirst("Host");
                if (host != null) {
                    int idx = referer.indexOf("://");
                    if (idx > -1) {
                        int edx = referer.indexOf("/", idx+3);
                        if (edx > -1) {
                            String refererHost = referer.substring(idx+3, edx);
                            if (!host.equals(refererHost)) {
                                isExternal = true;
                            }
                        }
                    }
                }
                
                if (isExternal) {

                    incrementCounter("Referers", referer, 1); 
                    int idx = referer.indexOf("q=");
                    if (idx > 0) {
                        int end = referer.indexOf('&', idx);
                        if (end == -1) referer = referer.substring(idx+2);
                        else referer = referer.substring(idx+2, end);
                        try {
                            String query = URLDecoder.decode(referer, "utf8");
                            if (!query.startsWith("cache:")) {
                                incrementCounter("Keywords", query, 1);
                            }
                        } catch (Exception e) {
                            // bad encoding
                        }
                    } 
     
                }
            }
            incrementCounter("numRequests", 1);    
            incrementCounter("RequestedUrls", request.getUrl(), 1);    
            incrementCounter("bytesRead", request.getNumBytesRead());
            if (response.getHeaders().getFirst("Age") != null) {
                incrementCounter("numCacheHits", 1);  
            }
            else incrementCounter("numCacheMisses", 1);  
            
            String responseInfo = response.getCode() + " " + response.getInfo();
            incrementCounter("ResponseInfo", responseInfo, 1);            
            incrementCounter("bytesWritten", response.getNumBytesWritten());
            long responseTime = response.getFinishTime() - request.getStartTime();
            incrementCounter("processingTime", responseTime); 
            
            double procTime = getLong("processingTimeRate");
            long reqSec = getLong("numRequestsRate");
            double avgResponse = procTime / reqSec;
            putStatistic("averageResponse", avgResponse);
            
            long responseSize = response.getNumBytesWritten();
            recalculateAverage("averageResponseSize", responseSize);        
         
            recalculateAverage("ResponseTimes", request.getUrl(), responseTime);
            recalculateAverage("ResponseSizes", request.getUrl(), responseSize);
            incrementCounter("TotalResponseSizes", request.getUrl(), responseSize);
            
            Object obj = response.getData();
            if (obj instanceof HTTPResponder) {
                String name = obj.getClass().getName();
                incrementCounter("RequestedResponders", name, 1);
                recalculateAverage("ResponderResponseTimes", name, responseTime);
                recalculateAverage("ResponderResponseSizes", name, responseSize);     
                incrementCounter("ResponderTotalResponseSizes", name, responseSize);
            }
            
            updateResponseTimes(responseTime);
        }
        catch (Throwable t) {
            System.err.println("An error occured while logging:");
            t.printStackTrace();
        }
        
        synchronized (activeRequests) {
        	activeRequests.remove(request);
        }
    }
    
    private static void storeEntityDetails(ProtectedStorable entity, String sourceAddress, HTTPRequest request, HTTPResponse response) {

        entity.incrementValue("numRequests", 1);
        entity.incrementValue("bytesUploaded", request.getNumBytesRead());        
        entity.incrementValue("bytesDownloaded", response.getNumBytesWritten());
        
        Number lastRequestTime = (Number) entity.get("lastRequestTime");
        Long longTime = new Long(System.currentTimeMillis());
        if (lastRequestTime == null) {
            // First request (after setting cookie)
            entity.incrementValue("numVisits", 1); // set to 1
            entity.put("firstRequestTime", longTime);
            entity.put("entrance", request.getHeaders().getRequestCookies().get("entrance"));
            entity.put(longTime.toString(), request.getUri(), "VisitTimes");
            entity.put("referer", request.getHeaders().getRequestCookies().get("referer"));
        } else {
            if (lastRequestTime.longValue() + VISIT_GAP < System.currentTimeMillis()) {
                entity.incrementValue("numVisits", 1);
                entity.put(longTime.toString(), request.getUri(), "VisitTimes");
            }            
        }
        

        entity.incrementValue(sourceAddress, 1, "RequestAddresses");
        
        entity.put("lastRequestTime", longTime);
        entity.put("lastRequestAddress", sourceAddress);
        entity.put("lastRequestAddressForwardedFor", request.getHeaders().getFirst("X-Forwarded-For"));
        entity.put("lastRequestCountry", IPToCountry.getCountryName(sourceAddress));
        entity.put("userAgent", request.getHeaders().getFirst("User-Agent"));   
        
    }
    
    private static void updateResponseTimes(long responseTime) {
    	int[][] times = getCounterTimes();
        updateResponseTimeCounter("Second", times[0][0], responseTime);
        updateResponseTimeCounter("Minute", times[0][1], responseTime);
        updateResponseTimeCounter("Hour",   times[0][2], responseTime);   
        updateResponseTimeCounter("Day",    times[0][3], responseTime);        
        updateResponseTimeCounter("Month",  times[0][4], responseTime);
    }    
    
    
    private static void clearNextCounters(int[] times) {
        clearNextCounters("Second", times[0]);
        clearNextCounters("Minute", times[1]);
        clearNextCounters("Hour",   times[2]);   
        clearNextCounters("Day",    times[3]);        
        clearNextCounters("Month",  times[4]);
    }        
     
    

    
    private static int[][] getCounterTimes() {
    	
    	Calendar cal = Calendar.getInstance();
    	
    	int[][] times = new int[2][5];
    	
    	times[0][0] = cal.get(Calendar.SECOND);
    	times[0][1] = cal.get(Calendar.MINUTE);
    	times[0][2] = cal.get(Calendar.HOUR_OF_DAY);
    	times[0][3] = cal.get(Calendar.DATE);
    	times[0][4] = cal.get(Calendar.MONTH) + 1;

    	cal.roll(Calendar.SECOND, 1);
    	times[1][0] = cal.get(Calendar.SECOND);
    	cal.roll(Calendar.SECOND, -1);
    	
    	cal.roll(Calendar.MINUTE, 1);
    	times[1][1] = cal.get(Calendar.MINUTE);
    	cal.roll(Calendar.MINUTE, -1);
    	
    	cal.roll(Calendar.HOUR_OF_DAY, 1);
    	times[1][2] = cal.get(Calendar.HOUR_OF_DAY);
    	cal.roll(Calendar.HOUR_OF_DAY, -1);
    	
    	cal.roll(Calendar.DATE, 1);    	
    	times[1][3] = cal.get(Calendar.DATE);
    	cal.roll(Calendar.DATE, -1);
    	
    	cal.roll(Calendar.MONTH, 1);    	
    	times[1][4] = cal.get(Calendar.MONTH) + 1;
    	cal.roll(Calendar.MONTH, -1);  	
    	
    	return times;
    }

    
    private static void updateResponseTimeCounter(String label, int idx, long responseTime) {
        String key = String.valueOf(idx);
        getStatsStore(label + "RequestTotal").incrementLongValue(key, 1);    
        getStatsStore(label + "AverageResponseTime").recalculateAverage(key, responseTime);
    } 
    
    private static void clearNextCounters(String label, int clear) {
        String clearKey = String.valueOf(clear);   
        getStatsStore(label + "RequestTotal").remove(clearKey);
        
        getStatsStore(label + "AverageResponseTime").remove(clearKey);
        getStatsStore(label + "AverageResponseTime").remove(clearKey + "$count");
        getStatsStore(label + "AverageResponseTime").remove(clearKey + "$total");    
        
        getStatsStore(label + "BytesReadTotal").remove(clearKey);    
        getStatsStore(label + "BytesWrittenTotal").remove(clearKey);    
    }
    
    private static void updateBytesReadCounter(String label, int idx, long bytes) {
        String key = String.valueOf(idx);    	    
        getStatsStore(label + "BytesReadTotal").incrementLongValue(key, bytes);           
    }    
    
    private static void updateBytesWrittenCounter(String label, int idx, long bytes) {
        String key = String.valueOf(idx);    	    
        getStatsStore(label + "BytesWrittenTotal").incrementLongValue(key, bytes);         
    }     
    
    
    private static WikiMap getStats() {
        WikiMap stats = (WikiMap) WareHouse.getWikiMap().get("Statistics");
        if (stats == null) {
            stats = WareHouse.initWikiMap("Statistics");
        }
        return stats;
    }  
    
    private static WikiMap getStatsStore(String store) {
        WikiMap stats = (WikiMap) getStats().get(store);
        if (stats == null) {
        	stats = WareHouse.initWikiMap(getStats(), MAXIMUM_STATS_STORE_SIZE, null, store);
        }
        return stats;
    }      
    
    public static Object getStatistic(String key) {
        return getStats().get(key);
    }    
    
    public static Object getStatistic(String store, String key) {
        return getStatsStore(store).get(key);
    }  
    
    public static long getStatisticAsLong(String store, String key) {
        Object obj = getStatistic(store, key);
        if (obj == null) return 0;
        else if (obj instanceof Number) return ((Number)obj).longValue();    
        return 0;
    }    
    
    public static Map<HTTPRequest, HTTPResponse> getActiveRequestResponseMap() {
        
        // Don't allow just anyone access active requests and responses!
        String[] excludes = { WareHouse.class.getName(), SiteMonitor.class.getName() };
    	String caller = WareHouse.getCallingClassName(excludes);
    	if (!caller.equals(Status.class.getName())) {
    	    throw new SecurityException(caller + " cannot access active data.");
    	}
    	
    	Map<HTTPRequest, HTTPResponse> snapShot = new HashMap<HTTPRequest, HTTPResponse>();
    	synchronized (activeRequests) {
    		snapShot.putAll(activeRequests);
    	}
    	return snapShot;
    }    
    
    private static void putStatistic(String key, Object value) {
        getStats().put(key, value);
    }     
    
    private static void putStatistic(String store, String key, Object value) {
        getStatsStore(store).put(key, value);
    }
      
    private static long incrementCounter(String key, long amount) {
        return getStats().incrementLongValue(key, amount);
    }     
    
    private static long incrementCounter(String store, String key, long amount) {
         return getStatsStore(store).incrementLongValue(key, amount);
    }
    
    private static double recalculateAverage(String key, long amount) {
        return getStats().recalculateAverage(key, amount);
    }     
    
    private static double recalculateAverage(String store, String key, long amount) {
        return getStatsStore(store).recalculateAverage(key, amount);
    }          
      
    
    public static void updateRates() {
        updateRate("dataRead", 8);
        updateRate("dataWritten", 8); 
        updateRate("numRequests"); 
        updateRate("numConnections"); 
        updateRate("processingTime");
    }
    
    private static long getLong(String key) {
        Number l = (Number) getStats().get(key);
        if (l != null) return l.longValue();
        return 0;
    }    
    
    private static void setLong(String key, long value) {
        putStatistic(key, new Long(value));
    }
    
    private static void updateRate(String name) {
        updateRate(name, 1);
    }
    
    private static void updateRate(String name, int multiplyer) {
        long time = System.currentTimeMillis();
        long statTime = getLong(name + "StatTime");
        long period = time - statTime;
        long amount = getLong(name) - getLong(name + "StatValue");
        long rate = (long) ((double) amount*1000 / period + 0.5);
        if (time > statTime + RATE_RESET_PERIOD) {
            long backValue = getLong(name) - rate;
            setLong(name + "StatValue", backValue);
            setLong(name + "StatTime", time-1000);
        }           
        setLong(name + "Rate", rate*multiplyer);
        if (rate*multiplyer > getLong(name + "MaxRate")) {
            setLong(name + "MaxRate", rate*multiplyer);
        }        
    }    
    
    private static List<String> getRequestDetails(HTTPRequest request, HTTPResponse response) {
    	
    	List<String> details = new ArrayList<String>(24);
               
    	details.add(HTTPHeaders.formatDate(request.getStartTime()));
    	details.add(request.getThreadId());
    	details.add(request.getRequestID());
    	details.add(request.getHeaders().getRequestCookies().get("browserID"));
        
        String userID = request.getHeaders().getRequestCookies().get("userID");  
        
        details.add(userID);
    	details.add(request.getHeaders().getFirst("X-Node-ID"));
    	
        String userPrivileges = null;        
        if (userID != null) {
            try {
                User u  = User.getUser(request);
                if (u != null) {
                    userPrivileges = u.getPrivilege().getLabel();
                }
            } catch (SecurityException se) {
                userPrivileges = Privilege.GUEST.getLabel();
            }
        }
        
        details.add(userPrivileges);
        
        details.add(request.getSourceAddress());
        details.add(request.getMethod());
        details.add(request.getUri());
        details.add(request.getHeaders().getFirst("Host"));
        details.add(request.getHeaders().getFirst("User-Agent"));
        details.add(request.getHeaders().getFirst("Referer"));
        

        details.add(String.valueOf(response.getCode()));
        
        details.add( response.getInfo());
        Object responseData = response.getData();
        String className = null;
        if (responseData != null) {
            className = responseData.getClass().getName();
        }
        details.add(className);
        details.add(response.getHeaders().getFirst("Content-Type"));
        details.add(response.getHeaders().getFirst("Content-Encoding"));
        details.add(response.getHeaders().getFirst("Content-Length"));
        details.add(String.valueOf(response.getNumBytesWritten()));
        details.add(String.valueOf(response.getFinishTime() - request.getStartTime()));
        
        String exceptionMessage = null;
        String exceptionClass = null;
        String exceptionStack = null;
        
        Throwable exception = response.getException();
        if (exception != null) {
	        List<String> exceptionDetails = getExceptionDetails(null, request, exception);
	        exceptionMessage = exceptionDetails.get(5); // exception message
	        exceptionClass = exceptionDetails.get(6); // exception class name
	        exceptionStack = exceptionDetails.get(7); // stack trace     
        }
        
        details.add(exceptionMessage);
        details.add(exceptionClass);
        details.add(exceptionStack);
        
        return details;
    }  
    
    private static List<String> getExceptionDetails(String errorDetail, HTTPRequest request, Throwable exception) {
    	
        
        List<String> details = new ArrayList<String>(8);
        
        details.add(HTTPHeaders.formatDate(System.currentTimeMillis()));
       
        String threadId = null;
        String requestID = null;        
        try {
            threadId = request.getThreadId();            
            requestID = request.getRequestID();
        } catch (NullPointerException ex) { /* unavailable */ }
        
        details.add(threadId);        
        details.add(requestID);
        details.add(errorDetail);
        

        String code = null;
        String message = null;
        String className = null;
        String stack = null;

        if (exception != null) {
            code = "500";
            if (exception instanceof HTTPException) {
            	code = String.valueOf(((HTTPException)exception).getCode());
            }
            
            message = exception.getMessage();
            className = exception.getClass().getName();
            
            StringBuilder billy = new StringBuilder();
            
            billy.append("<b>STACK:</b> " + exception.getClass().getName() + ": " + 
                    WareHouse.escapeHTMLEntities(exception.getMessage()) + "<br>" +
                    WareHouse.formatStackTrace(exception.getStackTrace(), true, false));
             
             int causeCount = 0;
             Throwable cause = exception.getCause();
             while (cause != null && causeCount < 10) {
            	 billy.append("<b>CAUSE:</b> " + cause.getClass().getName() + ": " + 
                        WareHouse.escapeHTMLEntities(cause.getMessage()) + "<br>" +
                        WareHouse.formatStackTrace(cause.getStackTrace(), true, false));
                 causeCount++;
                 cause = cause.getCause();
             }
             
             stack = billy.toString();
        }

        details.add(code);
        details.add(message);
        details.add(className);
        details.add(stack);
        
        return details;      
    }    
    
    public void logRequest(HTTPRequest request, HTTPResponse response) {
    	// Requests are now logged by completeResponse();
    }  
    
    public static void logRegistration(User u) {
        long time = System.currentTimeMillis();
        putStatistic("UserRegistrationTimes", u.getId(), time); 
    }  
    
    public static void logNewNode(NodeData n) {
        long time = System.currentTimeMillis();
        putStatistic("NewNodeTimes", n.getId(), time); 
    }     
    
    public static void logPayment(Payment p) {
        long time = System.currentTimeMillis();
        putStatistic("PaymentTimes", p.getId(), time); 
    }  
    
    public static void logComment(Comment c) {
        long time = System.currentTimeMillis();
        putStatistic("CommentTimes", c.getId(), time); 
    }    
    
    private static final List<String> exceptionLogHeadings = Arrays.asList(new String[] {
        "Time","Thread ID","Request ID","Error Detail","Code","Message","Exception Class Name","Stack Trace"
    });       
    
    private static final List<String> requestHeadings = Arrays.asList(new String[] {
        "Time","Thread ID","Request ID","Browser ID","User ID","Node ID","User Privileges","IP Address","Method","URI","Host","User Agent","Referer",
        "Code","Info","Class Name","Content Type","Content Encoding","Content Length","Bytes Written","Execution Time",
        "Exception message","Exception Class Name","Stack Trace"
    });  
    
    private static final List<String> requestGlTailHeadings = Arrays.asList(new String[] {
            "Method","URI","Host","Code","Content Type","Bytes Written","Execution Time",
            // 8, 9, 10,   13,     16,            19, 20
        });   
        
    private static final long RATE_RESET_PERIOD = 5000;

    public void logException(Throwable ex, HTTPRequest request) {
        List<String> details = getExceptionDetails(ex.getMessage(), request, ex);
        WareHouse.logData("exceptions", exceptionLogHeadings, details);  
    }
}