#! /usr/bin/env python ## $Date$ ## $Revision$ __usage__ = """\ %prog [options] %prog is a helper script which downloads and installs Rivet and the other packages needed to get the Rivet generator validation system up and running. It tries to be relatively intelligent about what it downloads so that, e.g. for users on systems with the LCG applications mounted in /afs/cern.ch/sw/lcg/external it will try to build Rivet using those when possible, rather than downloading and building things that are already available. TODO: * Build logging: can we stream build output as it happens (in verbose mode)? * Add new SLC5 LCG platform tag scheme """ import os, sys import logging import shutil, commands def compute_lcg_tag(): import platform import re ## Get distribution slversion = None osxversion = None distribution = platform.system() ## SL tests rhreleasepath = "/etc/redhat-release" if os.path.exists(rhreleasepath): distribution = "redhat" f = open(rhreleasepath, "r") #sltest = commands.getoutput("lsb_release -ds") sltest = f.read() f.close() if "Scientific Linux" in sltest: #slversion = [int(i) for i in commands.getoutput("lsb_release -rs").split(".")] m = re.search(r'Scientific Linux.*? (\d)\.(\d).*', sltest) if m: slversion = [int(m.group(1)), int(m.group(2))] distribution = "slc%d" % slversion[0] ## Mac tests if distribution == "Darwin": osxversion = (10,4) macver = platform.mac_ver()[0] if len(macver) > 0: ver = macver.split(".") osxversion = [int(i) for i in ver] distribution = "mac%d%d" % (osxversion[0], osxversion[1]) ## Windows tests if distribution == "Windows": distribution = "winxp" logging.debug("OS: " + distribution) ## Get architecture machine = platform.machine() logging.debug("Architecture: " + machine) ## Get compiler version compiler_code = None vcversion = None if distribution != "winxp": ## Get GCC version GCC_CMD = "g++" if os.environ.has_key("CC"): GCC_CMD = os.environ["CC"] elif os.environ.has_key("CXX"): GCC_CMD = os.environ["CXX"] gcc_version_str = commands.getoutput(GCC_CMD + ' --version | head -1') gcc_version_match = re.search(r'.*? (\d)\.(\d)\.?(\d)?.*?', gcc_version_str) compiler_code = "gccXX" if gcc_version_match is not None: gcc_major = gcc_version_match.group(1) gcc_minor = gcc_version_match.group(2) gcc_micro = gcc_version_match.group(3) compiler_code = "gcc%s%s" % (gcc_major, gcc_minor) else: ## Try to find VC version... somehow! import distutils.msvccompiler as msvc vcversions = sorted(msvc.get_devstudio_versions()) vcversion = vcversions[0] compiler_code = "vc%s" % vcversion ## the "future-proof" platform tag LCGPLATFORM = "%s-%s-%s-%s" % (machine, distribution, compiler_code, "opt") ## Historical platform tags if vcversion is not None and vcversion < 9: logging.debug("Computing old-style Windows tag to replace " + LCGPLATFORM) LCGPLATFORM = "win32_vc71_dbg" elif slversion is not None and slversion[0] < 5 or \ osxversion is not None and osxversion[1] < 6: logging.debug("Computing old-style tag to replace " + LCGPLATFORM) ## Different arch codes if "64" in machine: machine = "amd64" else: machine = "ia32" ## Old Mac code is "osx" rather than "mac" if osxversion: distribution = distribution.replace("mac", "osx") ## Historical exceptions for GCC version if compiler_code in ["gcc32", "gcc40"] and gcc_micro is not None: compiler_code += gcc_micro ## Set legacy LCG platform LCGPLATFORM = "%s_%s_%s" % (distribution, machine, compiler_code) # ## For Macs, append "_dbg" if needed # if osxversion and opts.BUILD_TYPE == "dbg": # LCGPLATFORM += "_dbg" return LCGPLATFORM ############################## from optparse import OptionParser DEFAULTPREFIX = os.path.join(os.getcwd(), "local") parser = OptionParser(usage=__usage__) parser.add_option("--prefix", metavar="INSTALLDIR", default=DEFAULTPREFIX, dest="PREFIX", help="Location to install packages to [%default]") parser.add_option("--force", action="store_true", default=False, dest="FORCE", help="Overwrite existing tarballs [%default]") parser.add_option("-j", default="2", dest="JMAKE", help="Num of 'make' threads to run in parallel (the n in 'make -j') [%default]") parser.add_option("--dev-mode", action="store_true", default=False, dest="DEV_MODE", help="Use the SVN development head version of Rivet [%default]") parser.add_option("--devmode", action="store_true", default=False, dest="DEV_MODE", help="Deprecated: alias for --dev-mode [%default]") parser.add_option("--lcgextdir", default="/afs/cern.ch/sw/lcg/external", dest="LCGDIR", help="Standard location of LCG external packages [%default]") parser.add_option("--lcgtag", default=compute_lcg_tag(), dest="LCGTAG", help="Force the LCG platform tag if it's not being computed correctly [%default]") parser.add_option("--ignore-lcgext", action="store_true", default=False, dest="IGNORE_LCG", help="Always bootstrap from sources, even if LCG versions are available [%default]") parser.add_option("--no-install-rivet", action="store_false", default=True, dest="INSTALL_RIVET", help="Don't install Rivet! Useful to set up required packages, or AGILe only [%default]") parser.add_option("--rivet-version", default="1.3.0", dest="RIVET_VERSION", help="Explicitly specify version of Rivet to get and use [%default]") parser.add_option("--rivet-url", default="http://www.hepforge.org/archive/rivet/", dest="RIVET_URL", help="Base URL for Rivet tarball downloads [%default]") parser.add_option("--build-rivet-manual", default=False, action="store_true", dest="BUILD_RIVET_MANUAL", help="Try to build the Rivet PDF manual (requires LaTeX) [%default]") parser.add_option("--build-unvalidated", default=False, action="store_true", dest="BUILD_UNVALIDATED", help="Build the unvalidated collection of Rivet analyses [%default]") parser.add_option("--install-agile", action="store_true", default=False, dest="INSTALL_AGILE", help="Install the AGILe interface system for Fortran generators [%default]") parser.add_option("--agile-version", default="1.2.0", dest="AGILE_VERSION", help="Explicitly specify version of AGILe to get and use [%default]") parser.add_option("--agile-url", default="http://www.hepforge.org/archive/agile/", dest="AGILE_URL", help="Base URL for AGILe tarball downloads [%default]") parser.add_option("--hepmc-version", default="2.06.00", dest="HEPMC_VERSION", help="Explicitly specify version of HepMC to get and use [%default]") parser.add_option("--hepmc-url", default="http://lcgapp.cern.ch/project/simu/HepMC/download/", dest="HEPMC_URL", help="Base URL for HepMC tarball downloads [%default]") parser.add_option("--fastjet-version", default="2.4.2", dest="FASTJET_VERSION", help="Explicitly specify version of FastJet to get and use [%default]") parser.add_option("--fastjet-url", default="http://www.lpthe.jussieu.fr/~salam/repository/software/fastjet/", dest="FASTJET_URL", help="Base URL for FastJet tarball downloads [%default]") parser.add_option("--gsl-version", default="1.10", dest="GSL_VERSION", help="Version of GSL to be used from LCG if AFS is available and used [%default]") parser.add_option("--with-gsl", metavar="DIR", default=None, dest="GSL_DIR", help="Explicit path to find GSL -- overrides --gsl-version [%default]") parser.add_option("--with-boost", metavar="DIR", default=None, dest="BOOST_DIR", help="Explicit path to find Boost [%default]") parser.add_option("--install-boost", action="store_true", default=False, dest="INSTALL_BOOST", help="Don't use a system copy of Boost (NB. it takes a long time to build) [%default]") parser.add_option("--boost-version", default="1.42.0", dest="BOOST_VERSION", help="Explicitly specify version of Boost to use from LCG (or to get if --install-boost is used) [%default]") parser.add_option("-v", "--verbose", action="store_const", const=logging.DEBUG, dest="LOGLEVEL", default=logging.INFO, help="print debug (very verbose) messages") parser.add_option("-q", "--quiet", action="store_const", const=logging.WARNING, dest="LOGLEVEL", default=logging.INFO, help="be very quiet") opts, args = parser.parse_args() ## Configure logging try: logging.basicConfig(level=opts.LOGLEVEL, format="%(message)s") except: pass h = logging.StreamHandler() h.setFormatter(logging.Formatter("%(message)s")) logging.getLogger().setLevel(opts.LOGLEVEL) if logging.getLogger().handlers: logging.getLogger().handlers[0] = h else: logging.getLogger().addHandler(h) ## Build location ROOT = os.path.abspath(os.getcwd()) ## Build location DLDIR = os.path.abspath(os.path.join(ROOT, "downloads")) if not os.path.exists(DLDIR): os.makedirs(DLDIR) if not os.access(DLDIR, os.W_OK): logging.error("Can't write to downloads directory, %s... exiting" % DLDIR) ## Build location BUILDDIR = os.path.abspath(os.path.join(ROOT, "build")) if not os.path.exists(BUILDDIR): os.makedirs(BUILDDIR) if not os.access(BUILDDIR, os.W_OK): logging.error("Can't write to build directory, %s... exiting" % BUILDDIR) ## Install to the PREFIX location PREFIX = os.path.abspath(opts.PREFIX) if not os.path.exists(PREFIX): os.makedirs(PREFIX) if not os.access(PREFIX, os.W_OK): logging.error("Can't write to installation directory, %s... exiting" % PREFIX) INCDIR = os.path.join(PREFIX, "include") if not os.path.exists(INCDIR): os.makedirs(INCDIR) LIBDIR = os.path.join(PREFIX, "lib") if not os.path.exists(LIBDIR): os.makedirs(LIBDIR) ########################### ## Function to grab a tarball from the Web def get_tarball(url, outname=None): if not outname: import urlparse outname = os.path.basename(urlparse.urlparse(url)[2]) outpath = os.path.join(DLDIR, outname) if os.path.exists(outpath): if not opts.FORCE: logging.info("Not overwriting tarball at %s" % outpath) return outpath else: logging.info("Overwriting tarball at %s" % outpath) os.remove(outpath) import urllib2 hreq = None out = None try: logging.info("Downloading %s" % url) hreq = urllib2.urlopen(url) out = open(outpath, "w") out.write(hreq.read()) out.close() hreq.close() return outpath except urllib2.URLError: logging.error("Problem downloading tarball from %s" % url) if hreq: hreq.close() except IOError: logging.error("Problem while writing tarball to %s" % outpath) if out: out.close() if hreq: hreq.close() return None ## Function to unpack a tarball def unpack_tarball(path): import tarfile tar = tarfile.open(path) try: for i in tar.getnames(): if not os.path.exists(os.path.join(BUILDDIR, i)): tar.extract(i, path=BUILDDIR) except: return False tar.close() return True ## Convenience function to get and unpack the tarball def get_unpack_tarball(tarurl, outname=None): outfile = get_tarball(tarurl, outname) unpack_tarball(outfile) or sys.exit(1) return True ## Function to enter an expanded tarball and run the usual ## autotools ./configure, make, make install mantra def conf_mk_mkinst(d, extraopts=""): prevdir = os.getcwd() if os.access(d, os.W_OK): os.chdir(d) confcmd = "./configure --prefix=%s %s" % (PREFIX, extraopts) logging.info("Configuring in %s: %s" % (os.getcwd(), confcmd)) st, op = commands.getstatusoutput(confcmd) if st != 0: logging.error(op) sys.exit(1) logging.debug("Configure output:\n" + op) buildcmd = "make -j%s && make -j%s install" % (opts.JMAKE, opts.JMAKE) logging.info("Building in %s: %s" % (os.getcwd(), buildcmd)) st, op = commands.getstatusoutput(buildcmd) if st != 0: logging.error(op) sys.exit(1) logging.debug("Build output:\n" + op) os.chdir(prevdir) else: logging.error("Couldn't find $1... exiting") sys.exit(1) DEVTOOLS_OK = False def check_devtools(): global DEVTOOLS_OK if not DEVTOOLS_OK: logging.info("Checking for developer mode programs") path = [] if os.environ.has_key("PATH"): path = os.environ["PATH"].split(":") if os.environ.has_key("path"): path = os.environ["path"].split(":") for pkg in ["svn", "autoconf", "autoreconf", "automake", "libtool", "swig"]: found = False for d in path: if os.access(os.path.join(d, pkg), os.X_OK): found = True break if not found: logging.error("You must have %s installed to bootstrap in developer mode" % pkg) return False DEVTOOLS_OK = True return True def pkg_bootstrap_svn(svnurl, pkgname, displayname=None): st = check_devtools() if not st: return False # if displayname is None: displayname = pkgname os.chdir(BUILDDIR) if not os.path.exists(pkgname): logging.info("Checking out %s from SVN head" % displayname) st, op = commands.getstatusoutput("svn co %s %s" % (svnurl, pkgname)) if st != 0: logging.error(op) return False logging.debug("SVN checkout output:\n" + op) elif not os.path.exists(os.path.join(pkgname, ".svn")): logging.error("Non-SVN '%s' directory already exists, blocking SVN checkout" % pkgname) return False os.chdir(pkgname) logging.info("Updating %s SVN working copy" % displayname) st, op = commands.getstatusoutput("svn update") if st != 0: logging.error(op) return False logging.debug("SVN update output:\n" + op) if not os.path.exists("configure"): logging.info("Bootstrapping autoconf files") st, op = commands.getstatusoutput("autoreconf -i") if st != 0: logging.error(op) return False logging.debug("autoreconf output:\n" + op) os.chdir(BUILDDIR) return True try: ## Get Rivet if opts.INSTALL_RIVET: rivetname = "rivet" os.chdir(BUILDDIR) if os.path.islink(rivetname): os.remove(rivetname) if not opts.DEV_MODE: import urlparse RIVET_NAME = "Rivet-" + opts.RIVET_VERSION RIVET_URL = urlparse.urljoin(opts.RIVET_URL, "%s.tar.gz" % RIVET_NAME) logging.info("Getting %s" % RIVET_URL) get_unpack_tarball(RIVET_URL) os.chdir(BUILDDIR) if not os.path.exists(rivetname): os.symlink(RIVET_NAME, rivetname) else: if not os.path.islink(rivetname): logging.warn("A '%s' directory already exists in %s, but is not a symlink to an expanded tarball" % (rivetname, BUILDDIR)) sys.exit(1) else: pkg_bootstrap_svn("http://svn.hepforge.org/rivet/trunk", rivetname, "Rivet") or sys.exit(2) ## Get AGILe if opts.INSTALL_AGILE: agilename = "agile" os.chdir(BUILDDIR) if os.path.islink(agilename): os.remove(agilename) if not opts.DEV_MODE: import urlparse AGILE_NAME = "AGILe-" + opts.AGILE_VERSION AGILE_URL = urlparse.urljoin(opts.AGILE_URL, "%s.tar.gz" % AGILE_NAME) logging.info("Getting %s" % AGILE_URL) get_unpack_tarball(AGILE_URL) or sys.exit(2) os.chdir(BUILDDIR) if not os.path.exists(agilename): os.symlink(AGILE_NAME, agilename) else: if not os.path.islink(agilename): logging.warn("A '%s' directory already exists in %s, but is not a symlink to an expanded tarball" % (agilename, BUILDDIR)) sys.exit(1) else: pkg_bootstrap_svn("http://svn.hepforge.org/agile/trunk", agilename, "AGILe") or sys.exit(2) ## Get Boost BOOSTFLAGS = None if opts.INSTALL_BOOST: logging.info("Installing a local copy of Boost") boostname = "boost_%s" % opts.BOOST_VERSION.replace(".", "_") boosttarname = boostname + ".tar.gz" boosturl = "http://downloads.sourceforge.net/boost/%s?use_mirror=mesh" % boosttarname get_unpack_tarball(boosturl) boostbuilddir = os.path.join(BUILDDIR, boostname) ## Don't do a full install --- just copy the headers into place ## (the build system changes between versions, and usually cocks up the dir structure) import shutil boostincdir = os.path.join(INCDIR, "boost") if not os.path.exists(boostincdir): shutil.copytree(os.path.join(boostbuilddir, "boost"), boostincdir) else: logging.warning("Boost header directory %s already exists... exiting" % boostincdir) sys.exit(2) logging.debug("Setting BOOST_DIR = " + PREFIX) opts.BOOST_DIR = PREFIX ## Are we able to use pre-built packages from CERN AFS? if not opts.IGNORE_LCG and os.path.isdir(opts.LCGDIR): logging.info("LCG area available: using LCG-built packages") logging.info("Using LCG platform tag = " + opts.LCGTAG) ## Now work out paths to give to Rivet ## HepMC HEPMCPATH = os.path.join(opts.LCGDIR, "HepMC", opts.HEPMC_VERSION, opts.LCGTAG) if not os.path.exists(HEPMCPATH): logging.error("HepMC does not exist at path %s. You may wish to use the --ignore-lcgext option" % HEPMCPATH) sys.exit(1) ## FastJet FASTJETPATH = os.path.join(opts.LCGDIR, "fastjet", opts.FASTJET_VERSION, opts.LCGTAG) if not os.path.exists(FASTJETPATH): logging.error("FastJet does not exist at path %s. You may wish to use the --ignore-lcgext option" % FASTJETPATH) sys.exit(1) ## Boost if not opts.INSTALL_BOOST: lcg_boost_version = opts.BOOST_VERSION + "_python2.5" opts.BOOST_DIR = os.path.join(opts.LCGDIR, "Boost", lcg_boost_version, opts.LCGTAG) ## GSL if not opts.GSL_DIR: opts.GSL_DIR = os.path.join(opts.LCGDIR, "GSL", opts.GSL_VERSION, opts.LCGTAG) ## Automatically set up a nice SWIG version from LCG if available swigbin = os.path.join(opts.LCGDIR, "swig", "1.3.40", opts.LCGTAG, "bin") if os.access(swigbin, os.R_OK) and os.access(os.path.join(swigbin, "swig"), os.X_OK): ## TODO: Check if this is exported to the builds os.environ["PATH"] = os.environ["PATH"] + ":" + swigbin else: ## We don't have access to LCG AFS, or are ignoring it, so we download the packages... ## Get and build HepMC hepmcname = "HepMC-" + opts.HEPMC_VERSION os.chdir(BUILDDIR) if not os.path.exists(hepmcname): hepmctarname = hepmcname + ".tar.gz" hepmcurl = os.path.join(opts.HEPMC_URL, hepmctarname) get_unpack_tarball(hepmcurl) conf_mk_mkinst(os.path.join(BUILDDIR, hepmcname), "--with-momentum=GEV --with-length=MM") HEPMCPATH = PREFIX ## Get and build FastJet fastjetname = "fastjet-" + opts.FASTJET_VERSION os.chdir(BUILDDIR) if not os.path.exists(fastjetname): fastjettarname = fastjetname + ".tar.gz" fastjeturl = os.path.join(opts.FASTJET_URL, fastjettarname) get_unpack_tarball(fastjeturl) conf_mk_mkinst(os.path.join(BUILDDIR, fastjetname), "--enable-shared --enable-allcxxplugins") FASTJETPATH = PREFIX ## This wouldn't be needed if Boost followed normal installation conventions... if opts.BOOST_DIR: logging.debug("Working out if Boost's headers are installed properly in " + opts.BOOST_DIR) boostincdir = os.path.join(opts.BOOST_DIR, "include") if not os.path.exists(os.path.join(boostincdir, "boost")): logging.debug("Boost's headers are not installed properly in " + opts.BOOST_DIR) incdirs = [d for d in os.listdir(boostincdir) if d.startswith("boost-")] if len(incdirs) > 0: BOOSTFLAGS = "--with-boost-incpath=%s" % os.path.join(boostincdir, incdirs[0]) else: logging.error("Couldn't work out location of Boost headers in %s" % boostincdir) sys.exit(1) ## Get build flags for Rivet and AGILe RA_CONFIGURE_FLAGS = "" ## LCG tag RA_CONFIGURE_FLAGS += " --with-lcgtag=%s" % opts.LCGTAG ## HepMC logging.debug("HepMC path: " + HEPMCPATH) RA_CONFIGURE_FLAGS += " --with-hepmc=%s" % HEPMCPATH ## Boost if opts.BOOST_DIR: logging.debug("Boost path: " + opts.BOOST_DIR) RA_CONFIGURE_FLAGS += " --with-boost=%s" % opts.BOOST_DIR ## In case the Boost headers are not in the standard structure, also try this: if BOOSTFLAGS: logging.debug("Boost flags: " + BOOSTFLAGS) RA_CONFIGURE_FLAGS += " " + BOOSTFLAGS AGILE_CONFIGURE_FLAGS = RA_CONFIGURE_FLAGS RIVET_CONFIGURE_FLAGS = RA_CONFIGURE_FLAGS ## Build and install Rivet if opts.INSTALL_RIVET: logging.debug("FastJet path: " + FASTJETPATH) RIVET_CONFIGURE_FLAGS += " --with-fastjet=%s" % FASTJETPATH if not opts.BUILD_RIVET_MANUAL: RIVET_CONFIGURE_FLAGS += " --disable-pdfmanual" if opts.BUILD_UNVALIDATED: RIVET_CONFIGURE_FLAGS += " --enable-unvalidated" if opts.GSL_DIR: logging.debug("Using GSL path: " + opts.GSL_DIR) RIVET_CONFIGURE_FLAGS += " --with-gsl=%s" % opts.GSL_DIR ## TODO: Remove for Rivet >= 1.2.2 if not opts.DEV_MODE: RIVET_CONFIGURE_FLAGS += " --with-gslcblas=%s" % opts.GSL_DIR conf_mk_mkinst(os.path.join(BUILDDIR, "rivet"), RIVET_CONFIGURE_FLAGS) ## Build and install AGILe if opts.INSTALL_AGILE: conf_mk_mkinst(os.path.join(BUILDDIR, agilename), AGILE_CONFIGURE_FLAGS) ## Collect and write out environment variables ## TODO: just point at rivetenv.(c)sh from 1.2.2 onwards env = {} ## Path env env["PATH"] = ":".join([os.path.join(PREFIX, "bin"), "$PATH"]) ## Lib env libdirs = [] for d in [PREFIX, HEPMCPATH, FASTJETPATH]: # CGALPATH libdir = os.path.join(d, "lib") if libdir not in libdirs: libdirs.append(libdir) if commands.getoutput("uname") == "Darwin": env["DYLD_LIBRARY_PATH"] = ":".join(libdirs + ["$DYLD_LIBRARY_PATH"]) else: env["LD_LIBRARY_PATH"] = ":".join(libdirs + ["$LD_LIBRARY_PATH"]) ## Python env pyversion = "%d.%d" % (sys.version_info[0], sys.version_info[1]) pylibdirs = [] for ld in ["lib", "lib64", "lib32"]: pylibdir = os.path.join(PREFIX, ld, "python%s/site-packages" % pyversion) logging.debug("Possible Python lib dir: %s" % pylibdir) if os.path.exists(pylibdir): pylibdirs.append(pylibdir) env["PYTHONPATH"] = ":".join(pylibdirs + ["$PYTHONPATH"]) ## LaTeX env texmfdir = [os.path.join(PREFIX, "share", "Rivet", "texmf")] env["HOMETEXMF"] = ":".join(texmfdir + ["$HOMETEXMF"]) env["TEXMFHOME"] = ":".join(texmfdir + ["$TEXMFHOME"]) ## Write out env files ## sh SHENV = "" for k, v in env.iteritems(): SHENV += "export %s=%s\n" % (k,v) comppath = os.path.join(PREFIX, "share", "Rivet", "rivet-completion") if os.path.exists(comppath): SHENV += ". %s\n" % comppath if opts.INSTALL_AGILE: comppath = os.path.join(PREFIX, "share", "AGILe", "agile-completion") if os.path.exists(comppath): SHENV += ". %s\n" % comppath os.chdir(ROOT) f = open("rivetenv.sh", "w") f.write(SHENV) f.close() ## csh CSHENV = "" for k, v in env.iteritems(): CSHENV += "setenv %s %s\n" % (k,v) os.chdir(ROOT) f = open("rivetenv.csh", "w") f.write(CSHENV) f.close() ## Tell the user print logging.info("All done. Now set some variables in your shell:\n") logging.info("In sh shell:\n" + SHENV) logging.info("In csh shell:\n" + CSHENV) logging.info("These can be used by sourcing, e.g.\nsource rivetenv.sh\nor\nsource rivetenv.csh") except Exception, e: import traceback traceback.print_exc() logging.error("\n\n") logging.error("An error has occurred while bootstrapping Rivet or one of its dependencies. Sorry!") logging.error("Please contact the Rivet developers at rivet@projects.hepforge.org, with a \ description of your problem, a copy of this script and any error trace that may have appeared \ and we'll try to get it fixed as soon as possible. Thanks for your help!")