#!/usr/bin/env python

# debpartial-mirror - partial debian mirror package tool
# (c) 2004 Otavio Salvador <otavio@debian.org>, Henrique Vilela <jacare@ucpel.tche.br>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
# $Id$

import getopt
import gzip
import os
import re
import sys
import time
import urllib

sys.path.append("/usr/share/debpartial-mirror")

import Config
import LocalPackages
import Package
import RemotePackages

class Mirror:
    def __init__(self, cfg_file, getFiles, getPackages, Simulate = False):
        try:
            self.__cfg = Config.Config(cfg_file)

            if not Simulate:
                self.__getPackagesIndex()
        except Exception, msg:
            print msg
            sys.exit(1)

        if getFiles:
            for filename in self.__cfg.getFiles():
                self.__getFile(self.__cfg.getServer() + filename, self.__cfg.getLocalDirectory() + filename, True, Simulate)

        if getPackages:
            rules = {}
            excludes = {}
            for d in self.__cfg.getDists():
                for s in self.__cfg.getSections(d):
                    for f in self.__getPackagesURI(d, s, self.__cfg.getLocalDirectory()):
                        rules[f] = self.__cfg.getFilters(d, s)
                        excludes[f] = self.__cfg.getExcludes(d)

                        if self.__cfg.getIncludeTask(d):
                            for a in self.__cfg.getArchs():
                                try:
                                    for i in os.popen('cpp -I /usr/share/debian-cd/tasks -I ' + os.path.dirname(self.__cfg.getIncludeTask(d)) + ' -DARCH_' + a + ' ' + self.__cfg.getIncludeTask(d) + '| grep -v \'^$\|^#.*\'', 'r').readlines():
                                        rules[f].append({'section':'*', 'priority':'*', 'package':i[:-1]})
                                except Exception, msg:
                                    print msg
                                    sys.exit(1)

                        if self.__cfg.getExcludeTask(d):
                            for a in self.__cfg.getArchs():
                                try:
                                    for i in os.popen('cpp -I /usr/share/debian-cd/tasks -I ' + os.path.dirname(self.__cfg.getExcludeTask(d)) + ' -DARCH_' + a + ' ' + self.__cfg.getExcludeTask(d) + '| grep -v \'^$\|^#.*\'', 'r').readlines():
                                        excludes[f].append(i[:-1])
                                except Exception, msg:
                                    print msg
                                    sys.exit(1)

            self.__selected = RemotePackages.RemotePackages(rules,
                                                            excludes,
                                                            self.__cfg.getSuggests(),
                                                            self.__cfg.getRecomends(),
                                                            self.__cfg.getProvides()).getSelectedPackages()
            self.__local = LocalPackages.LocalPackages(self.__cfg.getLocalDirectory() + 'pool/').getLocalPackages()


            remove = []
            for local_pkg in self.__local.itervalues():
                for local_version in local_pkg:
                    if not self.__checkRemotePackage(local_version):
                        remove.append(local_version)

            c = 0
            for i in remove:
                c=c+1
                print "\n", c, " of ", len(remove)
                self.__removePath(i.getFilename(), Simulate);

            get = []
            for filename in self.__selected:
                for selected_version in self.__selected[filename].itervalues():
                    if not self.__checkLocalPackage(selected_version):
                        get.append(selected_version)
            c = 0
            for i in get:
                c=c+1
                print c, " of ", len(get)
                self.__getFile(self.__cfg.getServer() + i.getFilename(), self.__cfg.getLocalDirectory() + i.getFilename(), Simulate)

    def __checkRemotePackage(self, local):
        for filename in self.__selected:
            if self.__selected[filename].has_key(local.getName()):
                if self.__selected[filename][local.getName()].getMD5Sum() == local.getMD5Sum():
                    return True
        return False

    def __checkLocalPackage(self, remote):
        for local_pkg in self.__local.itervalues():
            if self.__local.has_key(remote.getName()):
                for local_version in local_pkg:
                    if local_version.getMD5Sum() == remote.getMD5Sum():
                        return True
        return False

    def __getPackagesURI(self, dist, section, where, append=""):
        files = []

        for arch in self.__cfg.getArchs():
            files.append(where + 'dists/' + dist + '/' + section + '/binary-' + arch + '/Packages' + append)

        return files
    
    def __getPackagesIndex(self):
        for dist in self.__cfg.getDists().keys():
            self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/Release",
                           self.__cfg.getLocalDirectory() + "dists/" + dist + "/Release",
                           True);
            
            self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/Release.gpg",
                           self.__cfg.getLocalDirectory() + "dists/" + dist + "/Release.gpg",
                           True);
            
            for section in self.__cfg.getSections(dist).keys():
                for arch in self.__cfg.getArchs():
                    self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/" + section + "/binary-" + arch + "/Release",
                                   self.__cfg.getLocalDirectory() + "dists/" + dist + "/" + section + "/binary-" + arch + "/Release",
                                   True);

                    self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/Contents-" + arch + ".gz",
                                   self.__cfg.getLocalDirectory() + "dists/" + dist + "/Contents-" + arch + ".gz",
                                   True);

                for package in self.__getPackagesURI(dist, section, '', '.gz'):
                    origin =  self.__cfg.getServer() + package
                    destiny = self.__cfg.getLocalDirectory() + package
                    if self.__getFile(origin, destiny, True):
                        self.__uncompressFile(destiny)

    def __removePath(self, filename, Simulate=False):
        if os.path.isdir(filename):
            files = os.listdir(filename)
            if not len(files):
                print "Removing " + filename
                if not Simulate:
                    os.rmdir(filename)
                    matches = re.compile('(.*)/.*').findall(filename);
                    if matches:
                        self.__removePath(matches[0])
        else:
            print "Removing " + filename
            if not Simulate:
                os.unlink(filename)
                self.__removePath(os.path.dirname(filename))

    def __getFile(self, origin, destiny, checkNew = False, Simulate = False):
        matches = re.compile('(.*)\((.*)\)', re.IGNORECASE | re.MULTILINE ).findall(origin)
        if matches:
            origin=matches[0][0]

        expression=""
        matches = re.compile('(.*)\((.*)\)', re.IGNORECASE | re.MULTILINE ).findall(destiny)
        if matches:
            destiny=matches[0][0]
            expression=matches[0][1]
        
        self.__createPath(destiny)

        existSize = -1
        if os.path.isfile(destiny):
            existSize = os.path.getsize(destiny)

        webPage = urllib.urlopen(origin)

        #If the file exists, but we already have the whole thing, don't download again
        if checkNew and webPage.headers.has_key('content-length') and int(webPage.headers['content-length']) == existSize:
            if webPage.headers.has_key('Last-Modified') and int(time.mktime(time.strptime(webPage.headers['Last-Modified'], "%a, %d %b %Y %H:%M:%S %Z"))) < (os.path.getmtime(destiny) + time.altzone):
                print 'Match:', destiny
                webPage.close()
                return False

        if not Simulate:
            # Get the file
            content = ""
            if origin[-1] == '/':
                directory = True
            else:
                directory = False

            if not directory and webPage.headers.has_key('Content-Length'):
                content_length = int(webPage.headers['content-length'])

                numBytes = 0
                line = "Getting " + origin + "\n"
                sys.stdout.write(line)
                while numBytes < content_length:
                    percent_line = "... %d%%" % (numBytes * 100 / content_length)
                    sys.stdout.write(percent_line + '\r')
                    sys.stdout.flush()

                    data = webPage.read(8 * 1024)
                    if not data:
                        break
                    content += data
                    numBytes = numBytes + len(data)

                sys.stdout.write("\rDone " + line)
            else:
                if directory:
                    content = webPage.read()
                else:
                    print "Warning: " + origin + " does not exist."
                    webPage.close()
                    return False;

            webPage.close()

            if directory:
                matches = re.compile('<a href="([a-zA-Z0-9_][a-zA-Z0-9\-\./_]+)">', re.IGNORECASE | re.MULTILINE ).findall(content)
                if matches:
                    for filename in matches:
                        if re.match(expression, filename):
                            self.__getFile(origin + filename, destiny + filename, True, Simulate)
                        else:
                            print "Filed ", filename
            else:
                try:
                    outputFile = open(destiny, "wb")
                    outputFile.write(content)
                    outputFile.close()
                except IOError, msg:
                    print msg
                    sys.exit(1)

        return True

    def __uncompressFile(self, filename):
        print 'Uncompressing', filename
        # uncompress the file
        compressedFile = gzip.GzipFile(filename, 'rb')
        try:
            outputFile = open(filename.split('.gz')[0], "wb")
        except IOError, msg:
            print msg
            sys.exit(1)

        numBytes = 0
        while True:
            data = compressedFile.read(64 * 1024)
            if not data:
                break
            outputFile.write(data)
            numBytes = numBytes + len(data)

        compressedFile.close()
        outputFile.close()
        return True
        
    def __createPath(self, path):
        dir_name = os.path.dirname(path)
        if not os.path.exists(dir_name):
            print "Making directory " + dir_name + "..."
            os.makedirs(dir_name)

# Main function, no further comment needed. :)
def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'hvs:c:', ["help",
                                                            "version",
                                                            "skip-packages",
                                                            "skip-files",
                                                            "simulate",
                                                            "configfile="])
    except getopt.GetoptError:
        print "ERROR"
        usage()

    configurefile = '/etc/debpartial-mirror.conf'

    getFiles = True
    getPackages = True
    Simulate = False

    for o, a in opts:
        if o in ("-h", "--help"):
            usage()
            sys.exit()
            
        if o in ("-v", "--version"):
            version()
            sys.exit()
            
        if o in ("-c", "--configfile"):
            if a == '':
                usage()
            configurefile = a

        if o == "-s" or o in ("--skip-files",
                              "--skip-packages",
                              "--simulate"):
            if o == "--skip-files" or a == "f":
                getFiles = False

            if o == "--skip-packages" or a == "p":
                getPackages = False
                
            if o == "--simulate" or a == "":
                Simulate = True

    version()
    mirror = Mirror(configurefile, getFiles, getPackages, Simulate)

def version():
    print "debpartial-mirror 0.2.11 - Partial mirroring tool for Debian - Sat, 30 Jul 2005 20:27:01 -0300"
    print "(c) 2004 Otavio Salvador <otavio@debian.org>, Henrique Vilela <jacare@ucpel.tche.br>"
    print "This program is free software and was released under the terms of the GNU General Public License"
    print

def usage():
    print "Usage: debpartial-mirror [OPTION]"
    print
    print "-h --help\t\t\tDisplay this help end exit"
    print "-c<file> --configfile=<file>\tSelect a config file"
    print "-v --version\t\t\tShow the version"
    print
    sys.exit(2)

    
# Main
if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        print "exiting due to user interrupt."
