diff --git a/TODO b/TODO index 64f5917cb..8d680d660 100644 --- a/TODO +++ b/TODO @@ -1,8 +1,6 @@ TODO list: - include license data in the database (*) - implement configuration files snapshot tool (*) - - try dump+bzip2 - - lzma compression for the database? - migrate server code to ServerInterface [] write a tool that helps keeping packages updated (also supporting injected ones) [] complete reagent spm interface diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index db584bece..03c6e5622 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -1023,15 +1023,17 @@ def uploadDatabase(uris): dbfilec = eval(cmethod[0])(etpConst['etpdatabasedir'] + "/" + etpConst[cmethod[2]], "wb") dbpath = etpConst['etpdatabasefilepath'] - if etpConst['metadata-compression']: - dbpath = os.path.basename(etpConst['etpdatabasefilepath']) - dbpath = os.path.join(etpConst['packagestmpdir'],dbpath) - if not os.path.isdir(etpConst['packagestmpdir']): - os.makedirs(etpConst['packagestmpdir']) - shutil.copy2(etpConst['etpdatabasefilepath'],dbpath) - dbconn = Entropy.databaseTools.openGenericDatabase(dbpath, indexing = False, OutputInterface = Entropy) - dbconn.doContentCompression() - dbconn.closeDB() + + # dump the schema to a file + schemafilename = etpConst['etpdatabasedir'] + "/" + etpConst[cmethod[3]] + schemafilename_digest = etpConst['etpdatabasedir'] + "/" + etpConst[cmethod[4]] + schemafile = eval(cmethod[0])(schemafilename, "w") + dbconn = Entropy.openGenericDatabase(dbpath, xcache = False, indexing_override = False) + dbconn.doDatabaseExport(schemafile) + schemafile.close() + dbconn.closeDB() + del dbconn + schema_hexdigest = Entropy.entropyTools.md5sum(schemafilename) # compress the database file first dbfile = open(dbpath,"rb") @@ -1042,6 +1044,13 @@ def uploadDatabase(uris): dbfilec.close() del dbcont + # uploading schema file + rc = ftp.uploadFile(schemafilename) + if (rc == True): + print_info(green(" * ")+red("Upload of ")+bold(etpConst[cmethod[3]])+red(" completed.")) + else: + print_warning(brown(" * ")+red("Cannot properly upload to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check.")) + # uploading database file rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst[cmethod[2]]) if (rc == True): @@ -1051,6 +1060,7 @@ def uploadDatabase(uris): # remove the compressed file os.remove(etpConst['etpdatabasedir'] + "/" + etpConst[cmethod[2]]) + os.remove(schemafilename) # generate digest hexdigest = Entropy.entropyTools.md5sum(dbpath) @@ -1059,6 +1069,20 @@ def uploadDatabase(uris): f.flush() f.close() + # schema digest + f = open(schemafilename_digest,"w") + f.write(schema_hexdigest+" "+etpConst[cmethod[3]]+"\n") + f.flush() + f.close() + + # upload schema digest + print_info(green(" * ")+red("Uploading file ")+bold(etpConst[cmethod[4]])+red(" ..."), back = True) + rc = ftp.uploadFile(schemafilename_digest,True) + if (rc == True): + print_info(green(" * ")+red("Upload of ")+bold(schemafilename_digest)+red(" completed.")) + else: + print_warning(brown(" * ")+red("Cannot properly upload to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check.")) + # upload digest print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'])+red(" ..."), back = True) rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'],True) @@ -1158,10 +1182,6 @@ def downloadDatabase(uri): else: print_warning(brown(" * ")+red("Cannot properly download from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check.")) - dbconn = Entropy.databaseTools.openGenericDatabase(etpConst['etpdatabasefilepath'], indexing = False, OutputInterface = Entropy) - dbconn.doContentExtraction() - dbconn.closeDB() - # download RSS if etpConst['rss-feed']: diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 7472dc74d..f368d8ee0 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -3167,114 +3167,60 @@ class etpDatabase: header = " * ! * ! * ! * " ) - def doContentExtraction(self): - - if not self.doesTableExist("packed_data"): - return - - dbconn.cursor.execute('select data from packed_data where idpack = 1') - data = dbconn.cursor.fetchone() - - if data == None: - return - - import bz2 - # write to disk - path = entropyTools.getRandomTempFile() - f = open(path,"wb") - f.write(data) - f.flush() - f.close() - - # read and fill back - compressed_file = bz2.BZ2File(path,"r") - count = 0 - self.cursor.execute('DELETE FROM content') - line = tuple(compressed_file.readline().strip().split("\t")) - while 1: - count += 1 - if count%100 == 0: - self.updateProgress( - red("Unpacking database TOC ")+"["+blue(str(count))+"]", - importance = 0, - type = "info", - header = "\t", - back = True - ) - self.cursor.execute('INSERT INTO content VALUES (?,?,?)', line ) - line = tuple(compressed_file.readline().strip().split("\t")) - if len(line) < 3: - break - compressed_file.close() + def doDatabaseImport(self, schema): + schemafile = open(schema, 'r') + self.cursor.executescript(schemafile.read()) + schemafile.close() self.commitChanges() - self.cursor.execute('DELETE FROM packed_data WHERE idpack = 1;') - self.cursor.execute('VACUUM') - os.remove(path) - def doContentCompression(self): + def doDatabaseExport(self, dumpfile): - import bz2 - path = entropyTools.getRandomTempFile() - compressed_file = bz2.BZ2File(path,"w") + dumpfile.write("BEGIN TRANSACTION;\n") + self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type=='table'") + for name, type, sql in self.cursor.fetchall(): - ### Compress data - self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") - self.cursor.execute('SELECT * FROM content') - row = self.cursor.fetchone() - count = 0 - while row: - count += 1 - if count%100 == 0: - self.updateProgress( - red("Packing database TOC ")+"["+blue(str(count))+"]", - importance = 0, - type = "info", - back = True - ) - try: - for item in row: - if type(item) is int: - compressed_file.write(str(item)+"\t") - else: - compressed_file.write(str(item.encode('raw_unicode_escape'))+"\t") - compressed_file.write("\n") - except UnicodeEncodeError: - self.updateProgress( - red("Error on item: %s" % (item,)), - importance = 0, - type = "info", - back = True - ) - raise - row = self.cursor.fetchone() + self.updateProgress( + red("Exporting database table ")+"["+blue(str(name))+"]", + importance = 0, + type = "info", + back = True + ) - compressed_file.close() + if name == "sqlite_sequence": + dumpfile.write("DELETE FROM sqlite_sequence;\n") + elif name == "sqlite_stat1": + dumpfile.write("ANALYZE sqlite_master;\n") + elif name.startswith("sqlite_"): + continue + else: + dumpfile.write("%s;\n" % sql) + self.cursor.execute("PRAGMA table_info('%s')" % name) + cols = [str(r[1]) for r in self.cursor.fetchall()] + q = "SELECT 'INSERT INTO \"%(tbl_name)s\" VALUES(" + q += ", ".join(["'||quote(" + c + ")||'" for c in cols]) + q += ")' FROM '%(tbl_name)s'" + self.cursor.execute(q % {'tbl_name': name}) + con.text_factory = lambda x: unicode(x, "raw_unicode_escape") + for row in self.cursor: + dumpfile.write("%s;\n" % str(row[0].encode('raw_unicode_escape'))) + + self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type!='table' AND type!='meta'") + for name, type, sql in self.cursor.fetchall(): + dumpfile.write("%s;\n" % sql) + + dumpfile.write("COMMIT;\n") + try: + dumpfile.flush() + except: + pass self.updateProgress( - red("Flusing database TOC into database..."), + red("Database Export completed."), importance = 0, type = "info" ) + # remember to close the file - # now write to table - if not self.doesTableExist("packed_data"): - self.CreatePackedDataTable() - - self.cursor.execute("DELETE FROM packed_data WHERE idpack = 1") - f = open(path,"rb") - f.seek(0,2) - size = f.tell() - f.seek(0) - data = buffer(f.read(size)) - f.flush() - f.close() - self.cursor.execute('INSERT INTO packed_data VALUES (?,?)', (1,data,)) - - ### Delete from db - self.cursor.execute('DELETE FROM CONTENT') - self.cursor.execute('VACUUM') - self.commitChanges() - os.remove(path) # FIXME: this is only compatible with SQLITE def doesTableExist(self, table): diff --git a/libraries/entropy.py b/libraries/entropy.py index 1c43cdaa8..c6f9ec37f 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -262,13 +262,17 @@ class EquoInterface(TextInterface): conn.clientUpdatePackagesData() return conn - def openGenericDatabase(self, dbfile, dbname = None, xcache = None, readOnly = False): + def openGenericDatabase(self, dbfile, dbname = None, xcache = None, readOnly = False, indexing_override = None): if xcache == None: xcache = self.xcache + if indexing_override != None: + indexing = indexing_override + else: + indexing = self.indexing dbconn = self.databaseTools.openGenericDatabase(dbfile, dbname = dbname, xcache = xcache, - indexing = self.indexing, + indexing = indexing, readOnly = readOnly ) return dbconn @@ -3663,6 +3667,7 @@ class RepoInterface: self.noEquoCheck = noEquoCheck self.alreadyUpdated = 0 self.notAvailable = 0 + self.dbformat_eapi = 2 # check if I am root if (not self.Entropy.entropyTools.isRoot()): @@ -3704,7 +3709,7 @@ class RepoInterface: def __construct_paths(self, item, repo, cmethod): - if item not in ("db","rev","ck", "lock","mask"): + if item not in ("db","rev","ck","lock","mask","dbdump","dbdumpck"): raise exceptionTools.InvalidData("InvalidData: supported db, rev, ck, lock") if item == "db": @@ -3712,12 +3717,22 @@ class RepoInterface: raise exceptionTools.InvalidData("InvalidData: for db, cmethod can't be None") url = etpRepositories[repo]['database'] + "/" + etpConst[cmethod[2]] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst[cmethod[2]] + elif item == "dbdump": + if cmethod == None: + raise exceptionTools.InvalidData("InvalidData: for db, cmethod can't be None") + url = etpRepositories[repo]['database'] + "/" + etpConst[cmethod[3]] + filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst[cmethod[3]] elif item == "rev": url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaserevisionfile'] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaserevisionfile'] elif item == "ck": url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasehashfile'] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasehashfile'] + elif item == "dbdumpck": + if cmethod == None: + raise exceptionTools.InvalidData("InvalidData: for db, cmethod can't be None") + url = etpRepositories[repo]['database'] + "/" + etpConst[cmethod[4]] + filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst[cmethod[4]] elif item == "mask": url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasemaskfile'] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasemaskfile'] @@ -3727,16 +3742,27 @@ class RepoInterface: return url, filepath - def __remove_repository_files(self, repo, dbfilenameid): + def __remove_repository_files(self, repo, cmethod): + dbfilenameid = cmethod[2] self.__validate_repository_id(repo) - if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']): - os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']) - if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid]): - os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid]) - if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']): - os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']) + if self.dbformat_eapi == 1: + if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']): + os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']) + if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid]): + os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid]) + if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']): + os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']) + elif self.dbformat_eapi == 2: + if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+cmethod[4]): + os.remove(etpRepositories[repo]['dbpath']+"/"+cmethod[4]) + if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[3]]): + os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[3]]) + if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']): + os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']) + else: + raise exceptionTools.InvalidData('self.dbformat_eapi must be in (1,2)') def __unpack_downloaded_database(self, repo, cmethod): @@ -3745,18 +3771,32 @@ class RepoInterface: path = eval("self.Entropy.entropyTools."+cmethod[1])(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]]) return path - def __verify_database_checksum(self, repo): + def __verify_database_checksum(self, repo, cmethod = None): self.__validate_repository_id(repo) - try: - f = open(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r") - md5hash = f.readline().strip() - md5hash = md5hash.split()[0] - f.close() - except: - return -1 - rc = self.Entropy.entropyTools.compareMd5(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasefile'],md5hash) + if self.dbformat_eapi == 1: + dbfile = etpConst['etpdatabasefile'] + try: + f = open(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r") + md5hash = f.readline().strip() + md5hash = md5hash.split()[0] + f.close() + except: + return -1 + elif self.dbformat_eapi == 2: + dbfile = cmethod[3] + try: + f = open(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[4]],"r") + md5hash = f.readline().strip() + md5hash = md5hash.split()[0] + f.close() + except: + return -1 + else: + raise exceptionTools.InvalidData('self.dbformat_eapi must be in (1,2)') + + rc = self.Entropy.entropyTools.compareMd5(etpRepositories[repo]['dbpath']+"/"+dbfile,md5hash) return rc # @returns -1 if the file is not available @@ -3885,7 +3925,10 @@ class RepoInterface: header = "\t" ) - down_status = self.download_item("db", repo, cmethod) + down_status = self.download_item("dbdump", repo, cmethod) + if not down_status: # fallback to old db + self.dbformat_eapi = 1 + down_status = self.download_item("db", repo, cmethod) if not down_status: self.Entropy.updateProgress( bold("Attention: ") + red("database does not exist online."), importance = 1, @@ -3899,22 +3942,31 @@ class RepoInterface: # database is going to be updated self.dbupdated = True - # unpack database - self.Entropy.updateProgress( red("Unpacking database to ") + darkgreen(etpConst['etpdatabasefile'])+red(" ..."), - importance = 0, - type = "info", - header = "\t" - ) - # unpack database - self.__unpack_downloaded_database(repo, cmethod) + if self.dbformat_eapi == 1: + # unpack database + self.Entropy.updateProgress( red("Unpacking database to ") + darkgreen(etpConst['etpdatabasefile'])+red(" ..."), + importance = 0, + type = "info", + header = "\t" + ) + # unpack database + self.__unpack_downloaded_database(repo, cmethod) + + + + hashfile = etpConst['etpdatabasehashfile'] + downitem = 'ck' + if self.dbformat_eapi == 2: # EAPI = 2 + hashfile = etpConst[cmethod[4]] + downitem = 'dbdumpck' # download checksum - self.Entropy.updateProgress( red("Downloading checksum ") + darkgreen(etpConst['etpdatabasehashfile'])+red(" ..."), + self.Entropy.updateProgress( red("Downloading checksum ") + darkgreen(hashfile)+red(" ..."), importance = 0, type = "info", header = "\t" ) - down_status = self.download_item("ck", repo) + down_status = self.download_item(downitem, repo, cmethod) if not down_status: self.Entropy.updateProgress( red("Cannot fetch checksum. Cannot verify database integrity !"), importance = 1, @@ -3922,14 +3974,17 @@ class RepoInterface: header = "\t" ) else: + dbfilename = etpConst['etpdatabasefile'] + if self.dbformat_eapi == 2: + dbfilename = etpConst[cmethod[3]] # verify checksum - self.Entropy.updateProgress( red("Checking downloaded database ") + darkgreen(etpConst['etpdatabasefile'])+red(" ..."), + self.Entropy.updateProgress( red("Checking downloaded database ") + darkgreen(dbfilename)+red(" ..."), importance = 0, back = True, type = "info", header = "\t" ) - db_status = self.__verify_database_checksum(repo) + db_status = self.__verify_database_checksum(repo, cmethod) if db_status == -1: self.Entropy.updateProgress( red("Cannot open digest. Cannot verify database integrity !"), importance = 1, @@ -3954,11 +4009,29 @@ class RepoInterface: header = "\t" ) # delete all - self.__remove_repository_files(repo, cmethod[2]) + self.__remove_repository_files(repo, cmethod) self.syncErrors = True self.Entropy.cycleDone() continue + if self.dbformat_eapi == 2: + # load the dump into database + self.Entropy.updateProgress( red("Injecting downloaded dump ") + darkgreen(etpConst[cmethod[3]])+red(" ..."), + importance = 0, + type = "info", + header = "\t" + ) + dbfile = os.path.join(etpRepositories[repo]['dbpath'],etpConst['etpdatabasefile']) + dumpfile = os.path.join(etpRepositories[repo]['dbpath'],etpConst[cmethod[3]]) + if os.path.isfile(dbfile): + os.remove(dbfile) + dbconn = self.Entropy.openGenericDatabase(dbfile, xcache = False, indexing_override = False) + dbconn.doDatabaseImport(dumpfile) + dbconn.closeDB() + del dbconn + # remove the dump + os.remove(dumpfile) + # download packages.db.mask self.Entropy.updateProgress( red("Downloading package mask ")+darkgreen(etpConst['etpdatabasemaskfile'])+red(" ..."), importance = 0, @@ -4005,7 +4078,6 @@ class RepoInterface: back = True ) dbconn = self.Entropy.openRepositoryDatabase(repo) - dbconn.doContentExtraction() dbconn.createAllIndexes() try: # client db can be absent self.Entropy.clientDbconn.createAllIndexes() diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index ec626735d..e9aa2cf88 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -522,19 +522,22 @@ def initConfig_entropyConstants(rootdir): 'etpdatabasemaskfile': ETP_DBFILE+".mask", # the local/remote database revision file 'etpdatabaserevisionfile': ETP_DBFILE+".revision", # the local/remote database revision file 'etpdatabasehashfile': ETP_DBFILE+".md5", # its checksum + 'etpdatabasedumphashfilebz2': ETP_DBFILE+".dump.bz2.md5", + 'etpdatabasedumphashfilegzip': ETP_DBFILE+".dump.gz.md5", 'etpdatabaselockfile': ETP_DBFILE+".lock", # the remote database lock file 'etpdatabasedownloadlockfile': ETP_DBFILE+".download.lock", # the remote database download lock file 'etpdatabasetaintfile': ETP_DBFILE+".tainted", # when this file exists, the database is not synced anymore with the online one 'etpdatabasefile': ETP_DBFILE, # Entropy sqlite database file ETP_DIR+ETP_DBDIR+"/packages.db" 'etpdatabasefilegzip': ETP_DBFILE+".gz", # Entropy sqlite database file (gzipped) 'etpdatabasefilebzip2': ETP_DBFILE+".bz2", # Entropy sqlite database file (bzipped2) + 'etpdatabasedumpbzip2': ETP_DBFILE+".dump.bz2", # Entropy sqlite database dump file (bzipped2) + 'etpdatabasedumpgzip': ETP_DBFILE+".dump.gz", # Entropy sqlite database dump file (bzipped2) 'etpdatabasefileformat': "bz2", # Entropy default compressed database format 'etpdatabasesupportedcformats': ["bz2","gz"], # Entropy compressed databases format support 'etpdatabasecompressclasses': { - "bz2": ("bz2.BZ2File","unpackBzip2","etpdatabasefilebzip2",), - "gz": ("gzip.GzipFile","unpackGzip","etpdatabasefilegzip",) + "bz2": ("bz2.BZ2File","unpackBzip2","etpdatabasefilebzip2","etpdatabasedumpbzip2","etpdatabasedumphashfilebz2"), + "gz": ("gzip.GzipFile","unpackGzip","etpdatabasefilegzip","etpdatabasedumpgzip","etpdatabasedumphashfilegzip") }, - 'metadata-compression': False, # enable/disable server-side database tables compression 'rss-feed': True, # enable/disable packages RSS feed feature 'rss-name': "packages.rss", # default name of the RSS feed 'rss-base-url': "http://packages.sabayonlinux.org/", # default URL to the entropy web interface (overridden in reagent.conf) diff --git a/libraries/serverConstants.py b/libraries/serverConstants.py index 457bc19cd..5b6892aaf 100644 --- a/libraries/serverConstants.py +++ b/libraries/serverConstants.py @@ -61,11 +61,6 @@ def initConfig_serverConstants(): etpConst['activatorloglevel'] = loglevel else: pass - elif line.startswith("metadata-compression|") and (len(line.split("metadata-compression|")) == 2): - mc = line.split("metadata-compression|")[1] - if feed in ("enable","enabled","true","1"): - etpConst['metadata-compression'] = True - # reagent section if (os.path.isfile(etpConst['reagentconf'])):