Manifest file and md5 digests support

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@78 cd1c1023-2f26-0410-ae45-c471fc1f0318

README

@@ -11,4 +11,29 @@ DEPENDENCIES:
 - >=app-portage/portage-utils-0.1.23 (not mandatory)
 
 INSTALLATION:
-- copy entropy.conf in /etc/
+- copy entropy.conf in /etc/
+
+STRUCTURE:
+
+     |           ENTROPY           // master CLI interface to own them all
+     |             |||
+     |             |||
+     |            enzym            // creates .tbz2 packages (automatically on some specified packages)
+     |              |
+     |              |
+    \ /          reagent           // transform metadata info of .tbz2 packages in entropy specification files
+     '              |
+     |              |
+     |          activator          // does some QA tests on .etp and .tbz2 files upload data in a consistent way
+     |              |
+     |              |
+     |          ========== ^ server side
+     |          ==========   client side
+    \ /             |
+     '              |
+                    |
+                equilibrium       // CLI client with interface libraries
+                    |
+                    |
+                    |
+                GUI CLIENTs

TODO

@@ -2,7 +2,7 @@ TODO list (for developers only):
 - timestamp management across database and packages servers
 - hash management (like Manifest in gentoo)
 - write rules for exit codes
-- store etpData['md5'] and use it to choose if there is the needing of a version bump of the .etp file
+- add digest funcionality and tool to reagent
 - a lot more :-P
 
 ~cvill64 suggestion

handlers/reagent

@@ -14,33 +14,42 @@ import entropyTools
 from entropyConstants import *
 
 # CONSTANTS
-APPNAME = "entropy-specifications-generator"
+APPNAME = "reagent"
 APPVERSION = "1.0"
 def print_help():
     print "* info  * : Sabayon Linux "+APPNAME+" (C - 2007)"
-    print "* usage * : "+APPNAME+" <valid .tbz2 file>"
+    print
+    print "* usage * : "+APPNAME+" <tool, not mandatory> <valid path or .tbz2 file>"
     print "* opts  * : --help\t\tthis output"
     print "* opts  * : --version\t\tprint version"
+    print
+    print "* info  * : tools available: "
+    print "* info  * : \tdigest\t to create digest of a specified directory"
+    print
 
 options = sys.argv[1:]
 
+# print version
 if (string.join(options).find("--version") != -1) or (string.join(options).find(" -V") != -1):
     entropyTools.print_generic(APPNAME+": "+APPVERSION)
     sys.exit(0)
 
-# parameters test
+# print help
 if len(options) < 1 or string.join(options).find("--help") != -1 or string.join(options).find(" -h") != -1:
     print_help()
     if len(options) < 1:
 	entropyTools.print_error("not enough parameters")
     sys.exit(1)
 
-if len(options) > 2:
-    print_help()
-    entropyTools.print_error("too many parameters")
-    sys.exit(1)
-
-if not string.join(options).find(".tbz2"):
+# digest tool (creates a digest on the specified directory)
+if (options[0].find("digest") != -1):
+    entropyTools.createDigest(options[1])
+    sys.exit(0)
+# add other options here
+# ...
+# ...
+# tbz2 tool
+if (not options[0].endswith(".tbz2")):
     entropyTools.print_error("no .tbz2 file specified")
     sys.exit(3)
 
@@ -69,7 +78,8 @@ if etpOutfilePath is not None:
     f.writelines(etpOutput)
     f.flush()
     f.close()
-
+    # digesting directory
+    entropyTools.createDigest(etpOutfilePath)
 else:
     entropyTools.print_info("not generating a new .etp file, it's not needed")
 

handlers/tests/reagent-test

@@ -1,7 +1,7 @@
 #!/bin/sh
 cd ..
 for tbz2 in `find $1 -name "*.tbz2"`; do
-  python entropy-specifications-generator $tbz2
+  python reagent $tbz2
   if [ "$?" != "0" ]; then
     echo
     echo "test of "$tbz2" failed"

libraries/entropyConstants.py

@@ -21,6 +21,7 @@ etpData = {
     'keywords': "", # supported ARCHs (by the SRC)
     'binkeywords': "", # supported ARCHs (by the BIN)
     'download': "", # link to download the binary package
+    'digest': "", # md5 hash of the .tbz2 package
     'sources': "", # link to the sources
     'mirrorlinks': "", # =mirror://openoffice|link1|link2|link3
     'dependencies': "", # dependencies
@@ -36,8 +37,8 @@ etpData = {
 # the ARCHs that we support
 ETP_ARCHS = ["x86", "amd64"] # maybe ppc someday
 ETP_API_MAJOR = "1"
-ETP_API_MINOR = "1"
-ETP_API_SUBLEVEL = "0"
+ETP_API_MINOR = "2"
+ETP_API_SUBLEVEL = "1"
 ETP_API = ETP_API_MAJOR+"."+ETP_API_MINOR+"."+ETP_API_SUBLEVEL
 ETP_ARCH_CONST = "%ARCH%"
 ETP_REVISION_CONST = "%ETPREV%"
@@ -59,6 +60,8 @@ etpConst = {
     'packagessuploaddir': ETP_DIR+ETP_UPDIR, # etpConst['packagessuploaddir'] --> directory where .tbz2 files are stored waiting for being uploaded to our main mirror
     'confdir': ETP_CONF_DIR, # directory where entropy stores its configuration
     'repositoriesconf': ETP_CONF_DIR+"/repositories.conf", # repositories.conf file
+    'digestfile': "Manifest", # file that contains md5 hashes
+    'extension': ".etp", # entropy files extension
 }
 
 import os

libraries/entropyTools.py

@@ -50,6 +50,46 @@ def removeSpaceAtTheEnd(string):
     else:
 	return string
 
+def md5sum(filepath):
+    import md5
+    m = md5.new()
+    readfile = file(filepath)
+    block = readfile.read(1024)
+    while block:
+        m.update(block)
+	block = readfile.read(1024)
+    return m.hexdigest()
+
+def createDigest(path):
+    if path.endswith(etpConst['extension']):
+        # remove file name and keep the rest of the path
+	_path = path.split("/")[:len(path.split("/"))-1]
+	path = ""
+	for i in _path:
+            if (i):
+	        path += "/"+i
+    if (not os.path.isdir(path)):
+	print_error(path+" does not exist")
+        sys.exit(102)
+    digestContent = os.listdir(path)
+    # only .etp files
+    _digestContent = digestContent
+    digestContent = []
+    for i in _digestContent:
+	if i.endswith(etpConst['extension']):
+            digestContent.append(i)
+    if (not digestContent[0].endswith(etpConst['extension'])):
+	print_error(path+" does not contain "+etpConst['extension']+" files")
+        sys.exit(103)
+    print_info("digesting files in "+path)
+    digestOut = []
+    for i in digestContent:
+        digestOut.append("MD5 "+md5sum(path+"/"+i)+" "+i+"\n")
+    f = open(path+"/"+etpConst['digestfile'],"w")
+    f.writelines(digestOut)
+    f.flush()
+    f.close()
+
 def print_error(msg):
     print "* erro *  : "+msg
 
@@ -88,6 +128,9 @@ def extractPkgData(package):
     etpData['name'] = pkgname
     etpData['version'] = pkgver
 
+    # .tbz2 md5
+    etpData['digest'] = md5sum(tbz2File)
+
     import xpak
     tbz2 = xpak.tbz2(tbz2File)
     tbz2TmpDir = etpConst['packagestmpdir']+"/"+etpData['name']+"-"+etpData['version']+"/"
@@ -395,7 +438,7 @@ def allocateFile(etpData):
     # locate directory structure
     etpOutfileDir = etpConst['packagesdatabasedir']+"/"+etpData['category']+"/"+etpData['name']
     etpOutfileDir = translateArch(etpOutfileDir,etpData['chost'])
-    etpOutfileName = etpData['name']+"-"+etpData['version']+"-etp"+ETP_REVISION_CONST+".etp"
+    etpOutfileName = etpData['name']+"-"+etpData['version']+"-etp"+ETP_REVISION_CONST+etpConst['extension']
     etpOutfilePath = etpOutfileDir+"/"+etpOutfileName
 
     # we've the directory, then create it
@@ -435,7 +478,7 @@ def allocateFile(etpData):
 		    etpOutfilePath = None
 		else:
 		    # add 1 to: packagename-1.2.3-r1-etpX.etp
-		    newFileCounter = int(possibleOldFile.split("-")[len(possibleOldFile.split("-"))-1].split(".etp")[0].split("etp")[1])
+		    newFileCounter = int(possibleOldFile.split("-")[len(possibleOldFile.split("-"))-1].split(etpConst['extension'])[0].split(etpConst['extension'][1:])[1])
 		    newFileCounter += 1
 		    etpOutfilePath = re.subn(ETP_REVISION_CONST,str(newFileCounter), etpOutfilePath)[0]
 	    except OSError: