From 17b6fcd4467c88d51b2cb0b08b742588328f23a9 Mon Sep 17 00:00:00 2001 From: Rene Wagner Date: Tue, 16 May 2006 22:50:02 +0200 Subject: [PATCH] import bitbake from svn as of revision 323 Signed-off-by: Rene Wagner --- bitbake/.gitignore | 1 + bitbake/AUTHORS | 5 + bitbake/ChangeLog | 39 + bitbake/MANIFEST | 33 + bitbake/TODO | 18 + bitbake/bin/.gitignore | 1 + bitbake/bin/bbimage | 154 ++ bitbake/bin/bitbake | 933 ++++++++++++ bitbake/bin/bitdoc | 529 +++++++ bitbake/classes/.gitignore | 1 + bitbake/classes/base.bbclass | 79 + bitbake/conf/.gitignore | 1 + bitbake/conf/bitbake.conf | 55 + bitbake/contrib/.gitignore | 1 + bitbake/contrib/README | 1 + bitbake/contrib/bbdev.sh | 31 + bitbake/contrib/vim/.gitignore | 1 + bitbake/contrib/vim/syntax/.gitignore | 1 + bitbake/contrib/vim/syntax/bitbake.vim | 120 ++ bitbake/doc/.gitignore | 1 + bitbake/doc/COPYING.GPL | 340 +++++ bitbake/doc/COPYING.MIT | 17 + bitbake/doc/manual/.gitignore | 1 + bitbake/doc/manual/Makefile | 56 + bitbake/doc/manual/html.css | 281 ++++ bitbake/doc/manual/usermanual.xml | 361 +++++ bitbake/lib/.gitignore | 1 + bitbake/lib/bb/.gitignore | 1 + bitbake/lib/bb/__init__.py | 1266 +++++++++++++++++ bitbake/lib/bb/build.py | 395 +++++ bitbake/lib/bb/data.py | 580 ++++++++ bitbake/lib/bb/data_smart.py | 351 +++++ bitbake/lib/bb/event.py | 210 +++ bitbake/lib/bb/fetch/.gitignore | 1 + bitbake/lib/bb/fetch/__init__.py | 166 +++ bitbake/lib/bb/fetch/bk.py | 40 + bitbake/lib/bb/fetch/cvs.py | 214 +++ bitbake/lib/bb/fetch/git.py | 165 +++ bitbake/lib/bb/fetch/local.py | 61 + bitbake/lib/bb/fetch/svn.py | 189 +++ bitbake/lib/bb/fetch/wget.py | 164 +++ bitbake/lib/bb/manifest.py | 144 ++ bitbake/lib/bb/parse/.gitignore | 1 + bitbake/lib/bb/parse/__init__.py | 73 + bitbake/lib/bb/parse/parse_c/.gitignore | 1 + bitbake/lib/bb/parse/parse_c/bitbakeparser.l | 288 ++++ bitbake/lib/bb/parse/parse_c/bitbakeparser.py | 133 ++ bitbake/lib/bb/parse/parse_c/bitbakeparser.y | 161 +++ bitbake/lib/bb/parse/parse_c/lexer.h | 41 + bitbake/lib/bb/parse/parse_c/token.h | 83 ++ bitbake/lib/bb/parse/parse_py/.gitignore | 1 + bitbake/lib/bb/parse/parse_py/BBHandler.py | 378 +++++ bitbake/lib/bb/parse/parse_py/ConfHandler.py | 199 +++ bitbake/lib/bb/parse/parse_py/__init__.py | 32 + bitbake/lib/bb/shell.py | 808 +++++++++++ bitbake/lib/bb/utils.py | 71 + bitbake/setup.py | 69 + 57 files changed, 9348 insertions(+) create mode 100644 bitbake/.gitignore create mode 100644 bitbake/AUTHORS create mode 100644 bitbake/ChangeLog create mode 100644 bitbake/MANIFEST create mode 100644 bitbake/TODO create mode 100644 bitbake/bin/.gitignore create mode 100755 bitbake/bin/bbimage create mode 100755 bitbake/bin/bitbake create mode 100755 bitbake/bin/bitdoc create mode 100644 bitbake/classes/.gitignore create mode 100644 bitbake/classes/base.bbclass create mode 100644 bitbake/conf/.gitignore create mode 100644 bitbake/conf/bitbake.conf create mode 100644 bitbake/contrib/.gitignore create mode 100644 bitbake/contrib/README create mode 100644 bitbake/contrib/bbdev.sh create mode 100644 bitbake/contrib/vim/.gitignore create mode 100644 bitbake/contrib/vim/syntax/.gitignore create mode 100644 bitbake/contrib/vim/syntax/bitbake.vim create mode 100644 bitbake/doc/.gitignore create mode 100644 bitbake/doc/COPYING.GPL create mode 100644 bitbake/doc/COPYING.MIT create mode 100644 bitbake/doc/manual/.gitignore create mode 100644 bitbake/doc/manual/Makefile create mode 100644 bitbake/doc/manual/html.css create mode 100644 bitbake/doc/manual/usermanual.xml create mode 100644 bitbake/lib/.gitignore create mode 100644 bitbake/lib/bb/.gitignore create mode 100644 bitbake/lib/bb/__init__.py create mode 100644 bitbake/lib/bb/build.py create mode 100644 bitbake/lib/bb/data.py create mode 100644 bitbake/lib/bb/data_smart.py create mode 100644 bitbake/lib/bb/event.py create mode 100644 bitbake/lib/bb/fetch/.gitignore create mode 100644 bitbake/lib/bb/fetch/__init__.py create mode 100644 bitbake/lib/bb/fetch/bk.py create mode 100644 bitbake/lib/bb/fetch/cvs.py create mode 100644 bitbake/lib/bb/fetch/git.py create mode 100644 bitbake/lib/bb/fetch/local.py create mode 100644 bitbake/lib/bb/fetch/svn.py create mode 100644 bitbake/lib/bb/fetch/wget.py create mode 100644 bitbake/lib/bb/manifest.py create mode 100644 bitbake/lib/bb/parse/.gitignore create mode 100644 bitbake/lib/bb/parse/__init__.py create mode 100644 bitbake/lib/bb/parse/parse_c/.gitignore create mode 100644 bitbake/lib/bb/parse/parse_c/bitbakeparser.l create mode 100644 bitbake/lib/bb/parse/parse_c/bitbakeparser.py create mode 100644 bitbake/lib/bb/parse/parse_c/bitbakeparser.y create mode 100644 bitbake/lib/bb/parse/parse_c/lexer.h create mode 100644 bitbake/lib/bb/parse/parse_c/token.h create mode 100644 bitbake/lib/bb/parse/parse_py/.gitignore create mode 100644 bitbake/lib/bb/parse/parse_py/BBHandler.py create mode 100644 bitbake/lib/bb/parse/parse_py/ConfHandler.py create mode 100644 bitbake/lib/bb/parse/parse_py/__init__.py create mode 100644 bitbake/lib/bb/shell.py create mode 100644 bitbake/lib/bb/utils.py create mode 100755 bitbake/setup.py diff --git a/bitbake/.gitignore b/bitbake/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/AUTHORS b/bitbake/AUTHORS new file mode 100644 index 0000000..4129e4c --- /dev/null +++ b/bitbake/AUTHORS @@ -0,0 +1,5 @@ +Holger Freyther +Chris Larson +Mickey Lauer +Holger Schurig +Phil Blundell diff --git a/bitbake/ChangeLog b/bitbake/ChangeLog new file mode 100644 index 0000000..f4b65ff --- /dev/null +++ b/bitbake/ChangeLog @@ -0,0 +1,39 @@ +Changes in BitBake 1.3.?: + - Create a new Fetcher module to ease the + development of new Fetchers. + +Changes in BitBake 1.3.2: + - reintegration of make.py into BitBake + - bbread is gone, use bitbake -e + - lots of shell updates and bugfixes + - Introduction of the .= and =. operator + - Sort variables, keys and groups in bitdoc + - Fix regression in the handling of BBCOLLECTIONS + - Update the bitbake usermanual + +Changes in BitBake 1.3.0: + - add bitbake interactive shell (bitbake -i) + - refactor bitbake utility in OO style + - kill default arguments in methods in the bb.data module + - kill default arguments in methods in the bb.fetch module + - the http/https/ftp fetcher will fail if the to be + downloaded file was not found in DL_DIR (this is needed + to avoid unpacking the sourceforge mirror page) + - Switch to a cow like data instance for persistent and non + persisting mode (called data_smart.py) + - Changed the callback of bb.make.collect_bbfiles to carry + additional parameters + - Drastically reduced the amount of needed RAM by not holding + each data instance in memory when using a cache/persistent + storage + +Changes in BitBake 1.2.1: + The 1.2.1 release is meant as a intermediate release to lay the + ground for more radical changes. The most notable changes are: + + - Do not hardcode {}, use bb.data.init() instead if you want to + get a instance of a data class + - bb.data.init() is a factory and the old bb.data methods are delegates + - Do not use deepcopy use bb.data.createCopy() instead. + - Removed default arguments in bb.fetch + diff --git a/bitbake/MANIFEST b/bitbake/MANIFEST new file mode 100644 index 0000000..cf0aac9 --- /dev/null +++ b/bitbake/MANIFEST @@ -0,0 +1,33 @@ +AUTHORS +ChangeLog +MANIFEST +setup.py +bin/bbimage +bin/bitbake +lib/bb/__init__.py +lib/bb/build.py +lib/bb/data.py +lib/bb/data_smart.py +lib/bb/event.py +lib/bb/fetch/bk.py +lib/bb/fetch/cvs.py +lib/bb/fetch/git.py +lib/bb/fetch/__init__.py +lib/bb/fetch/local.py +lib/bb/fetch/svn.py +lib/bb/fetch/wget.py +lib/bb/manifest.py +lib/bb/parse/__init__.py +lib/bb/parse/parse_py/BBHandler.py +lib/bb/parse/parse_py/ConfHandler.py +lib/bb/parse/parse_py/__init__.py +lib/bb/shell.py +lib/bb/utils.py +doc/COPYING.GPL +doc/COPYING.MIT +doc/manual/html.css +doc/manual/Makefile +doc/manual/usermanual.xml +contrib/bbdev.sh +conf/bitbake.conf +classes/base.bbclass diff --git a/bitbake/TODO b/bitbake/TODO new file mode 100644 index 0000000..511fae4 --- /dev/null +++ b/bitbake/TODO @@ -0,0 +1,18 @@ +On popular request by popular people a list of tasks to-do: + + -Kill insecure usage of os.system either by properly escaping + the strings or a faster replacement not involving /bin/sh + -Introduce a -p option to automatically hotshot/profile the + run + -Cache dependencies separately and invalidate them when any file + changed. + -... + + +DONE: +· -On generating the inter package deps do not parse each file multiply +· times. + -We build the lists while parsing the data now +· (WAS: Do not generate the world dependency tree, only when someone +· requests it. + diff --git a/bitbake/bin/.gitignore b/bitbake/bin/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/bin/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/bin/bbimage b/bitbake/bin/bbimage new file mode 100755 index 0000000..df6caa2 --- /dev/null +++ b/bitbake/bin/bbimage @@ -0,0 +1,154 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2003 Chris Larson +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. + +import sys, os +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) +import bb +from bb import * + +__version__ = 1.0 +type = "jffs2" +cfg_bb = data.init() +cfg_oespawn = data.init() + + +def usage(): + print "Usage: bbimage [options ...]" + print "Creates an image for a target device from a root filesystem," + print "obeying configuration parameters from the BitBake" + print "configuration files, thereby easing handling of deviceisms." + print "" + print " %s\t\t%s" % ("-r [arg], --root [arg]", "root directory (default=${IMAGE_ROOTFS})") + print " %s\t\t%s" % ("-t [arg], --type [arg]", "image type (jffs2[default], cramfs)") + print " %s\t\t%s" % ("-n [arg], --name [arg]", "image name (override IMAGE_NAME variable)") + print " %s\t\t%s" % ("-v, --version", "output version information and exit") + sys.exit(0) + +def version(): + print "BitBake Build Tool Core version %s" % bb.__version__ + print "BBImage version %s" % __version__ + +def emit_bb(d, base_d = {}): + for v in d.keys(): + if d[v] != base_d[v]: + data.emit_var(v, d) + +def getopthash(l): + h = {} + for (opt, val) in l: + h[opt] = val + return h + +import getopt +try: + (opts, args) = getopt.getopt(sys.argv[1:], 'vr:t:e:n:', [ 'version', 'root=', 'type=', 'bbfile=', 'name=' ]) +except getopt.GetoptError: + usage() + +# handle opts +opthash = getopthash(opts) + +if '--version' in opthash or '-v' in opthash: + version() + sys.exit(0) + +try: + cfg_bb = parse.handle(os.path.join('conf', 'bitbake.conf'), cfg_bb) +except IOError: + fatal("Unable to open bitbake.conf") + +# sanity check +if cfg_bb is None: + fatal("Unable to open/parse %s" % os.path.join('conf', 'bitbake.conf')) + usage(1) + +rootfs = None +extra_files = [] + +if '--root' in opthash: + rootfs = opthash['--root'] +if '-r' in opthash: + rootfs = opthash['-r'] + +if '--type' in opthash: + type = opthash['--type'] +if '-t' in opthash: + type = opthash['-t'] + +if '--bbfile' in opthash: + extra_files.append(opthash['--bbfile']) +if '-e' in opthash: + extra_files.append(opthash['-e']) + +for f in extra_files: + try: + cfg_bb = parse.handle(f, cfg_bb) + except IOError: + print "unable to open %s" % f + +if not rootfs: + rootfs = data.getVar('IMAGE_ROOTFS', cfg_bb, 1) + +if not rootfs: + bb.fatal("IMAGE_ROOTFS not defined") + +data.setVar('IMAGE_ROOTFS', rootfs, cfg_bb) + +from copy import copy, deepcopy +localdata = data.createCopy(cfg_bb) + +overrides = data.getVar('OVERRIDES', localdata) +if not overrides: + bb.fatal("OVERRIDES not defined.") +data.setVar('OVERRIDES', '%s:%s' % (overrides, type), localdata) +data.update_data(localdata) +data.setVar('OVERRIDES', overrides, localdata) + +if '-n' in opthash: + data.setVar('IMAGE_NAME', opthash['-n'], localdata) +if '--name' in opthash: + data.setVar('IMAGE_NAME', opthash['--name'], localdata) + +topdir = data.getVar('TOPDIR', localdata, 1) or os.getcwd() + +cmd = data.getVar('IMAGE_CMD', localdata, 1) +if not cmd: + bb.fatal("IMAGE_CMD not defined") + +outdir = data.getVar('DEPLOY_DIR_IMAGE', localdata, 1) +if not outdir: + bb.fatal('DEPLOY_DIR_IMAGE not defined') +mkdirhier(outdir) + +#depends = data.getVar('IMAGE_DEPENDS', localdata, 1) or "" +#if depends: +# bb.note("Spawning bbmake to satisfy dependencies: %s" % depends) +# ret = os.system('bbmake %s' % depends) +# if ret != 0: +# bb.error("executing bbmake to satisfy dependencies") + +bb.note("Executing %s" % cmd) +data.setVar('image_cmd', cmd, localdata) +data.setVarFlag('image_cmd', 'func', 1, localdata) +try: + bb.build.exec_func('image_cmd', localdata) +except bb.build.FuncFailed: + sys.exit(1) +#ret = os.system(cmd) +#sys.exit(ret) diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake new file mode 100755 index 0000000..b060c4f --- /dev/null +++ b/bitbake/bin/bitbake @@ -0,0 +1,933 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2003, 2004 Phil Blundell +# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer +# Copyright (C) 2005 Holger Hans Peter Freyther +# Copyright (C) 2005 ROAD GmbH +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. + +import sys, os, getopt, glob, copy, os.path, re, time +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) +import bb +from bb import utils, data, parse, debug, event, fatal +from sets import Set +import itertools, optparse + +parsespin = itertools.cycle( r'|/-\\' ) +bbdebug = 0 + +__version__ = "1.3.2" + +#============================================================================# +# BBParsingStatus +#============================================================================# +class BBParsingStatus: + """ + The initial idea for this status class is to use the data when it is + already loaded instead of loading it from various place over and over + again. + """ + + def __init__(self): + self.cache_dirty = False + self.providers = {} + self.bbfile_priority = {} + self.bbfile_config_priorities = [] + self.ignored_dependencies = None + self.possible_world = [] + self.world_target = Set() + self.pkg_pn = {} + self.pkg_fn = {} + self.pkg_pvpr = {} + self.pkg_dp = {} + self.pn_provides = {} + self.all_depends = Set() + + def handle_bb_data(self, file_name, bb_data, cached): + """ + We will fill the dictionaries with the stuff we + need for building the tree more fast + """ + if bb_data == None: + return + + if not cached: + self.cache_dirty = True + + pn = bb.data.getVar('PN', bb_data, True) + pv = bb.data.getVar('PV', bb_data, True) + pr = bb.data.getVar('PR', bb_data, True) + dp = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0") + provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split()) + depends = (bb.data.getVar("DEPENDS", bb_data, True) or "").split() + + + # build PackageName to FileName lookup table + if pn not in self.pkg_pn: + self.pkg_pn[pn] = [] + self.pkg_pn[pn].append(file_name) + + # build FileName to PackageName lookup table + self.pkg_fn[file_name] = pn + self.pkg_pvpr[file_name] = (pv,pr) + self.pkg_dp[file_name] = dp + + # Build forward and reverse provider hashes + # Forward: virtual -> [filenames] + # Reverse: PN -> [virtuals] + if pn not in self.pn_provides: + self.pn_provides[pn] = Set() + self.pn_provides[pn] |= provides + + for provide in provides: + if provide not in self.providers: + self.providers[provide] = [] + self.providers[provide].append(file_name) + + for dep in depends: + self.all_depends.add(dep) + + # Collect files we may need for possible world-dep + # calculations + if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True): + self.possible_world.append(file_name) + + +#============================================================================# +# BBStatistics +#============================================================================# +class BBStatistics: + """ + Manage build statistics for one run + """ + def __init__(self ): + self.attempt = 0 + self.success = 0 + self.fail = 0 + self.deps = 0 + + def show( self ): + print "Build statistics:" + print " Attempted builds: %d" % self.attempt + if self.fail: + print " Failed builds: %d" % self.fail + if self.deps: + print " Dependencies not satisfied: %d" % self.deps + if self.fail or self.deps: return 1 + else: return 0 + + +#============================================================================# +# BBOptions +#============================================================================# +class BBConfiguration( object ): + """ + Manages build options and configurations for one run + """ + def __init__( self, options ): + for key, val in options.__dict__.items(): + setattr( self, key, val ) + self.data = data.init() + +#============================================================================# +# BBCooker +#============================================================================# +class BBCooker: + """ + Manages one bitbake build run + """ + + ParsingStatus = BBParsingStatus # make it visible from the shell + Statistics = BBStatistics # make it visible from the shell + + def __init__( self ): + self.build_cache_fail = [] + self.build_cache = [] + self.building_list = [] + self.build_path = [] + self.consider_msgs_cache = [] + self.preferred = {} + self.stats = BBStatistics() + self.status = None + + self.pkgdata = None + self.cache = None + + def tryBuildPackage( self, fn, item, the_data ): + """Build one package""" + bb.event.fire(bb.event.PkgStarted(item, the_data)) + try: + self.stats.attempt += 1 + if self.configuration.force: + bb.data.setVarFlag('do_%s' % self.configuration.cmd, 'force', 1, the_data) + if not self.configuration.dry_run: + bb.build.exec_task('do_%s' % self.configuration.cmd, the_data) + bb.event.fire(bb.event.PkgSucceeded(item, the_data)) + self.build_cache.append(fn) + return True + except bb.build.FuncFailed: + self.stats.fail += 1 + bb.error("task stack execution failed") + bb.event.fire(bb.event.PkgFailed(item, the_data)) + self.build_cache_fail.append(fn) + raise + except bb.build.EventException, e: + self.stats.fail += 1 + event = e.args[1] + bb.error("%s event exception, aborting" % bb.event.getName(event)) + bb.event.fire(bb.event.PkgFailed(item, the_data)) + self.build_cache_fail.append(fn) + raise + + def tryBuild( self, fn, virtual ): + """Build a provider and its dependencies""" + if fn in self.building_list: + bb.error("%s depends on itself (eventually)" % fn) + bb.error("upwards chain is: %s" % (" -> ".join(self.build_path))) + return False + + the_data = self.pkgdata[fn] + item = self.status.pkg_fn[fn] + + self.building_list.append(fn) + + pathstr = "%s (%s)" % (item, virtual) + self.build_path.append(pathstr) + + depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split() + if self.configuration.verbose: + bb.note("current path: %s" % (" -> ".join(self.build_path))) + bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list))) + + try: + failed = False + + depcmd = self.configuration.cmd + bbdepcmd = bb.data.getVarFlag('do_%s' % self.configuration.cmd, 'bbdepcmd', the_data) + if bbdepcmd is not None: + if bbdepcmd == "": + depcmd = None + else: + depcmd = bbdepcmd + + if depcmd: + oldcmd = self.configuration.cmd + self.configuration.cmd = depcmd + + for dependency in depends_list: + if dependency in self.status.ignored_dependencies: + continue + if not depcmd: + continue + if self.buildProvider( dependency ) == 0: + bb.error("dependency %s (for %s) not satisfied" % (dependency,item)) + failed = True + if self.configuration.abort: + break + + if depcmd: + self.configuration.cmd = oldcmd + + if failed: + self.stats.deps += 1 + return False + + if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data): + self.build_cache.append(fn) + return True + + return self.tryBuildPackage( fn, item, the_data ) + + finally: + self.building_list.remove(fn) + self.build_path.remove(pathstr) + + def findBestProvider( self, pn, pkg_pn = None): + """ + If there is a PREFERRED_VERSION, find the highest-priority bbfile + providing that version. If not, find the latest version provided by + an bbfile in the highest-priority set. + """ + if not pkg_pn: + pkg_pn = self.status.pkg_pn + + files = pkg_pn[pn] + priorities = {} + for f in files: + priority = self.status.bbfile_priority[f] + if priority not in priorities: + priorities[priority] = [] + priorities[priority].append(f) + p_list = priorities.keys() + p_list.sort(lambda a, b: a - b) + tmp_pn = [] + for p in p_list: + tmp_pn = [priorities[p]] + tmp_pn + + preferred_file = None + + localdata = data.createCopy(self.configuration.data) + bb.data.setVar('OVERRIDES', "%s:%s" % (pn, data.getVar('OVERRIDES', localdata)), localdata) + bb.data.update_data(localdata) + + preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, 1) + if preferred_v: + m = re.match('(.*)_(.*)', preferred_v) + if m: + preferred_v = m.group(1) + preferred_r = m.group(2) + else: + preferred_r = None + + for file_set in tmp_pn: + for f in file_set: + pv,pr = self.status.pkg_pvpr[f] + if preferred_v == pv and (preferred_r == pr or preferred_r == None): + preferred_file = f + preferred_ver = (pv, pr) + break + if preferred_file: + break; + if preferred_r: + pv_str = '%s-%s' % (preferred_v, preferred_r) + else: + pv_str = preferred_v + if preferred_file is None: + bb.note("preferred version %s of %s not available" % (pv_str, pn)) + else: + bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn)) + + del localdata + + # get highest priority file set + files = tmp_pn[0] + latest = None + latest_p = 0 + latest_f = None + for file_name in files: + pv,pr = self.status.pkg_pvpr[file_name] + dp = self.status.pkg_dp[file_name] + + if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p): + latest = (pv, pr) + latest_f = file_name + latest_p = dp + if preferred_file is None: + preferred_file = latest_f + preferred_ver = latest + + return (latest,latest_f,preferred_ver, preferred_file) + + def showVersions( self ): + pkg_pn = self.status.pkg_pn + preferred_versions = {} + latest_versions = {} + + # Sort by priority + for pn in pkg_pn.keys(): + (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn) + preferred_versions[pn] = (pref_ver, pref_file) + latest_versions[pn] = (last_ver, last_file) + + pkg_list = pkg_pn.keys() + pkg_list.sort() + + for p in pkg_list: + pref = preferred_versions[p] + latest = latest_versions[p] + + if pref != latest: + prefstr = pref[0][0] + "-" + pref[0][1] + else: + prefstr = "" + + print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1], + prefstr) + + def showEnvironment( self ): + """Show the outer or per-package environment""" + if self.configuration.buildfile: + try: + self.configuration.data, fromCache = self.load_bbfile( self.configuration.buildfile ) + except IOError, e: + fatal("Unable to read %s: %s" % ( self.configuration.buildfile, e )) + except Exception, e: + fatal("%s" % e) + # emit variables and shell functions + try: + data.update_data( self.configuration.data ) + data.emit_env(sys.__stdout__, self.configuration.data, True) + except Exception, e: + fatal("%s" % e) + # emit the metadata which isnt valid shell + for e in self.configuration.data.keys(): + if data.getVarFlag( e, 'python', self.configuration.data ): + sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1))) + + def buildProvider( self, item ): + fn = None + + discriminated = False + + if item not in self.status.providers: + bb.error("Nothing provides %s" % item) + return 0 + + all_p = self.status.providers[item] + + for p in all_p: + if p in self.build_cache: + bb.debug(1, "already built %s in this run\n" % p) + return 1 + + eligible = [] + preferred_versions = {} + + # Collate providers by PN + pkg_pn = {} + for p in all_p: + pn = self.status.pkg_fn[p] + if pn not in pkg_pn: + pkg_pn[pn] = [] + pkg_pn[pn].append(p) + + bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys())) + + for pn in pkg_pn.keys(): + preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4] + eligible.append(preferred_versions[pn][1]) + + for p in eligible: + if p in self.build_cache_fail: + bb.debug(1, "rejecting already-failed %s" % p) + eligible.remove(p) + + if len(eligible) == 0: + bb.error("no eligible providers for %s" % item) + return 0 + + # look to see if one of them is already staged, or marked as preferred. + # if so, bump it to the head of the queue + for p in all_p: + the_data = self.pkgdata[p] + pn = bb.data.getVar('PN', the_data, 1) + pv = bb.data.getVar('PV', the_data, 1) + pr = bb.data.getVar('PR', the_data, 1) + tmpdir = bb.data.getVar('TMPDIR', the_data, 1) + stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr) + if os.path.exists(stamp): + (newvers, fn) = preferred_versions[pn] + if not fn in eligible: + # package was made ineligible by already-failed check + continue + oldver = "%s-%s" % (pv, pr) + newver = '-'.join(newvers) + if (newver != oldver): + extra_chat = "; upgrading from %s to %s" % (oldver, newver) + else: + extra_chat = "" + if self.configuration.verbose: + bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat)) + eligible.remove(fn) + eligible = [fn] + eligible + discriminated = True + break + + prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1) + if prefervar: + self.preferred[item] = prefervar + + if item in self.preferred: + for p in eligible: + pn = self.status.pkg_fn[p] + if self.preferred[item] == pn: + if self.configuration.verbose: + bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item)) + eligible.remove(p) + eligible = [p] + eligible + discriminated = True + break + + if len(eligible) > 1 and discriminated == False: + if item not in self.consider_msgs_cache: + providers_list = [] + for fn in eligible: + providers_list.append(self.status.pkg_fn[fn]) + bb.note("multiple providers are available (%s);" % ", ".join(providers_list)) + bb.note("consider defining PREFERRED_PROVIDER_%s" % item) + self.consider_msgs_cache.append(item) + + + # run through the list until we find one that we can build + for fn in eligible: + bb.debug(2, "selecting %s to satisfy %s" % (fn, item)) + if self.tryBuild(fn, item): + return 1 + + bb.note("no buildable providers for %s" % item) + return 0 + + def buildDepgraph( self ): + all_depends = self.status.all_depends + pn_provides = self.status.pn_provides + + def calc_bbfile_priority(filename): + for (regex, pri) in self.status.bbfile_config_priorities: + if regex.match(filename): + return pri + return 0 + + # Handle PREFERRED_PROVIDERS + for p in (bb.data.getVar('PREFERRED_PROVIDERS', self.configuration.data, 1) or "").split(): + (providee, provider) = p.split(':') + if providee in self.preferred and self.preferred[providee] != provider: + bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee])) + self.preferred[providee] = provider + + # Calculate priorities for each file + for p in self.pkgdata.keys(): + self.status.bbfile_priority[p] = calc_bbfile_priority(p) + + # Build package list for "bitbake world" + bb.debug(1, "collating packages for \"world\"") + for f in self.status.possible_world: + terminal = True + pn = self.status.pkg_fn[f] + + for p in pn_provides[pn]: + if p.startswith('virtual/'): + bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p)) + terminal = False + break + for pf in self.status.providers[p]: + if self.status.pkg_fn[pf] != pn: + bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p)) + terminal = False + break + if terminal: + self.status.world_target.add(pn) + + # drop reference count now + self.status.possible_world = None + self.status.all_depends = None + + def myProgressCallback( self, x, y, f, file_data, from_cache ): + # feed the status with new input + self.status.handle_bb_data(f, file_data, from_cache) + + if bbdebug > 0: + return + if os.isatty(sys.stdout.fileno()): + sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) ) + sys.stdout.flush() + else: + if x == 1: + sys.stdout.write("Parsing .bb files, please wait...") + sys.stdout.flush() + if x == y: + sys.stdout.write("done.") + sys.stdout.flush() + + def interactiveMode( self ): + """Drop off into a shell""" + try: + from bb import shell + except ImportError, details: + bb.fatal("Sorry, shell not available (%s)" % details ) + else: + bb.data.update_data( self.configuration.data ) + shell.start( self ) + sys.exit( 0 ) + + def parseConfigurationFile( self, afile ): + try: + self.configuration.data = bb.parse.handle( afile, self.configuration.data ) + except IOError: + bb.fatal( "Unable to open %s" % afile ) + except bb.parse.ParseError, details: + bb.fatal( "Unable to parse %s (%s)" % (afile, details) ) + + def handleCollections( self, collections ): + """Handle collections""" + if collections: + collection_list = collections.split() + for c in collection_list: + regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1) + if regex == None: + bb.error("BBFILE_PATTERN_%s not defined" % c) + continue + priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1) + if priority == None: + bb.error("BBFILE_PRIORITY_%s not defined" % c) + continue + try: + cre = re.compile(regex) + except re.error: + bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex)) + continue + try: + pri = int(priority) + self.status.bbfile_config_priorities.append((cre, pri)) + except ValueError: + bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority)) + + + def cook( self, configuration, args ): + self.configuration = configuration + + if not self.configuration.cmd: + self.configuration.cmd = "build" + + if self.configuration.debug: + bb.debug_level = self.configuration.debug + + self.configuration.data = bb.data.init() + + for f in self.configuration.file: + self.parseConfigurationFile( f ) + + self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) ) + + if self.configuration.show_environment: + self.showEnvironment() + sys.exit( 0 ) + + # inject custom variables + if not bb.data.getVar("BUILDNAME", self.configuration.data): + bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data) + bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data) + + buildname = bb.data.getVar("BUILDNAME", self.configuration.data) + + if self.configuration.interactive: + self.interactiveMode() + + if self.configuration.buildfile is not None: + bf = os.path.abspath( self.configuration.buildfile ) + try: + bbfile_data = bb.parse.handle(bf, self.configuration.data) + except IOError: + bb.fatal("Unable to open %s" % bf) + + item = bb.data.getVar('PN', bbfile_data, 1) + try: + self.tryBuildPackage( bf, item, bbfile_data ) + except bb.build.EventException: + bb.error( "Build of '%s' failed" % item ) + + sys.exit( self.stats.show() ) + + # initialise the parsing status now we know we will need deps + self.status = BBParsingStatus() + + ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or "" + self.status.ignored_dependencies = Set( ignore.split() ) + + self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) ) + + pkgs_to_build = None + if args: + if not pkgs_to_build: + pkgs_to_build = [] + pkgs_to_build.extend(args) + if not pkgs_to_build: + bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1) + if bbpkgs: + pkgs_to_build = bbpkgs.split() + if not pkgs_to_build and not self.configuration.show_versions \ + and not self.configuration.interactive \ + and not self.configuration.show_environment: + print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'" + print "for usage information." + sys.exit(0) + + # Import Psyco if available and not disabled + if not self.configuration.disable_psyco: + try: + import psyco + except ImportError: + if bbdebug == 0: + bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.") + else: + psyco.bind( self.collect_bbfiles ) + else: + bb.note("You have disabled Psyco. This decreases performance.") + + try: + bb.debug(1, "collecting .bb files") + self.collect_bbfiles( self.myProgressCallback ) + bb.debug(1, "parsing complete") + if bbdebug == 0: + print + if self.configuration.parse_only: + print "Requested parsing .bb files only. Exiting." + return + + bb.data.update_data( self.configuration.data ) + self.buildDepgraph() + + if self.configuration.show_versions: + self.showVersions() + sys.exit( 0 ) + if 'world' in pkgs_to_build: + pkgs_to_build.remove('world') + for t in self.status.world_target: + pkgs_to_build.append(t) + + bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.data)) + + for k in pkgs_to_build: + failed = False + try: + if self.buildProvider( k ) == 0: + # already diagnosed + failed = True + except bb.build.EventException: + bb.error("Build of " + k + " failed") + failed = True + + if failed: + if self.configuration.abort: + sys.exit(1) + + bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.data)) + + sys.exit( self.stats.show() ) + + except KeyboardInterrupt: + print "\nNOTE: KeyboardInterrupt - Build not completed." + sys.exit(1) + + def get_bbfiles( self, path = os.getcwd() ): + """Get list of default .bb files by reading out the current directory""" + contents = os.listdir(path) + bbfiles = [] + for f in contents: + (root, ext) = os.path.splitext(f) + if ext == ".bb": + bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f))) + return bbfiles + + def find_bbfiles( self, path ): + """Find all the .bb files in a directory (uses find)""" + findcmd = 'find ' + path + ' -name *.bb | grep -v SCCS/' + try: + finddata = os.popen(findcmd) + except OSError: + return [] + return finddata.readlines() + + def deps_clean(self, d): + depstr = data.getVar('__depends', d) + if depstr: + deps = depstr.split(" ") + for dep in deps: + (f,old_mtime_s) = dep.split("@") + old_mtime = int(old_mtime_s) + new_mtime = parse.cached_mtime(f) + if (new_mtime > old_mtime): + return False + return True + + def load_bbfile( self, bbfile ): + """Load and parse one .bb build file""" + + if not self.cache in [None, '']: + # get the times + cache_mtime = data.init_db_mtime(self.cache, bbfile) + file_mtime = parse.cached_mtime(bbfile) + + if file_mtime > cache_mtime: + #print " : '%s' dirty. reparsing..." % bbfile + pass + else: + #print " : '%s' clean. loading from cache..." % bbfile + cache_data = data.init_db( self.cache, bbfile, False ) + if self.deps_clean(cache_data): + return cache_data, True + + topdir = data.getVar('TOPDIR', self.configuration.data) + if not topdir: + topdir = os.path.abspath(os.getcwd()) + # set topdir to here + data.setVar('TOPDIR', topdir, self.configuration) + bbfile = os.path.abspath(bbfile) + bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) + # expand tmpdir to include this topdir + data.setVar('TMPDIR', data.getVar('TMPDIR', self.configuration.data, 1) or "", self.configuration.data) + # set topdir to location of .bb file + topdir = bbfile_loc + #data.setVar('TOPDIR', topdir, cfg) + # go there + oldpath = os.path.abspath(os.getcwd()) + os.chdir(topdir) + bb = data.init_db(self.cache,bbfile, True, self.configuration.data) + try: + parse.handle(bbfile, bb) # read .bb data + if not self.cache in [None, '']: + bb.commit(parse.cached_mtime(bbfile)) # write cache + os.chdir(oldpath) + return bb, False + finally: + os.chdir(oldpath) + + def collect_bbfiles( self, progressCallback ): + """Collect all available .bb build files""" + self.cb = progressCallback + parsed, cached, skipped, masked = 0, 0, 0, 0 + self.cache = bb.data.getVar( "CACHE", self.configuration.data, 1 ) + self.pkgdata = data.pkgdata( not self.cache in [None, ''], self.cache, self.configuration.data ) + + if not self.cache in [None, '']: + if self.cb is not None: + print "NOTE: Using cache in '%s'" % self.cache + try: + os.stat( self.cache ) + except OSError: + bb.mkdirhier( self.cache ) + else: + if self.cb is not None: + print "NOTE: Not using a cache. Set CACHE = to enable." + files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split() + data.setVar("BBFILES", " ".join(files), self.configuration.data) + + if not len(files): + files = self.get_bbfiles() + + if not len(files): + bb.error("no files to build.") + + newfiles = [] + for f in files: + if os.path.isdir(f): + dirfiles = self.find_bbfiles(f) + if dirfiles: + newfiles += dirfiles + continue + newfiles += glob.glob(f) or [ f ] + + bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1) or "" + try: + bbmask_compiled = re.compile(bbmask) + except sre_constants.error: + bb.fatal("BBMASK is not a valid regular expression.") + + for i in xrange( len( newfiles ) ): + f = newfiles[i] + if bbmask and bbmask_compiled.search(f): + bb.debug(1, "bbmake: skipping %s" % f) + masked += 1 + continue + debug(1, "bbmake: parsing %s" % f) + + # read a file's metadata + try: + bb_data, fromCache = self.load_bbfile(f) + if fromCache: cached += 1 + else: parsed += 1 + deps = None + if bb_data is not None: + # allow metadata files to add items to BBFILES + #data.update_data(self.pkgdata[f]) + addbbfiles = data.getVar('BBFILES', bb_data) or None + if addbbfiles: + for aof in addbbfiles.split(): + if not files.count(aof): + if not os.path.isabs(aof): + aof = os.path.join(os.path.dirname(f),aof) + files.append(aof) + for var in bb_data.keys(): + if data.getVarFlag(var, "handler", bb_data) and data.getVar(var, bb_data): + event.register(data.getVar(var, bb_data)) + self.pkgdata[f] = bb_data + + # now inform the caller + if self.cb is not None: + self.cb( i + 1, len( newfiles ), f, bb_data, fromCache ) + + except IOError, e: + bb.error("opening %s: %s" % (f, e)) + pass + except bb.parse.SkipPackage: + skipped += 1 + pass + except KeyboardInterrupt: + raise + except Exception, e: + bb.error("%s while parsing %s" % (e, f)) + + if self.cb is not None: + print "\rNOTE: Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ), + +#============================================================================# +# main +#============================================================================# + +if __name__ == "__main__": + + parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ), + usage = """%prog [options] [package ...] + +Executes the specified task (default is 'build') for a given set of BitBake files. +It expects that BBFILES is defined, which is a space seperated list of files to +be executed. BBFILES does support wildcards. +Default BBFILES are the .bb files in the current directory.""" ) + + parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.", + action = "store", dest = "buildfile", default = None ) + + parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.", + action = "store_false", dest = "abort", default = True ) + + parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status", + action = "store_true", dest = "force", default = False ) + + parser.add_option( "-i", "--interactive", help = "drop into the interactive mode.", + action = "store_true", dest = "interactive", default = False ) + + parser.add_option( "-c", "--cmd", help = "Specify task to execute. Note that this only executes the specified task for the providee and the packages it depends on, i.e. 'compile' does not implicitly call stage for the dependencies (IOW: use only if you know what you are doing)", + action = "store", dest = "cmd", default = "build" ) + + parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf", + action = "append", dest = "file", default = [] ) + + parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal", + action = "store_true", dest = "verbose", default = False ) + + parser.add_option( "-D", "--debug", help = "Increase the debug level", + action = "count", dest="debug", default = 0) + + parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions", + action = "store_true", dest = "dry_run", default = False ) + + parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)", + action = "store_true", dest = "parse_only", default = False ) + + parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)", + action = "store_true", dest = "disable_psyco", default = False ) + + parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages", + action = "store_true", dest = "show_versions", default = False ) + + parser.add_option( "-e", "--environment", help = "show the global or per-package environment (this is what used to be bbread)", + action = "store_true", dest = "show_environment", default = False ) + + options, args = parser.parse_args( sys.argv ) + + cooker = BBCooker() + cooker.cook( BBConfiguration( options ), args[1:] ) diff --git a/bitbake/bin/bitdoc b/bitbake/bin/bitdoc new file mode 100755 index 0000000..64d3294 --- /dev/null +++ b/bitbake/bin/bitdoc @@ -0,0 +1,529 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2005 Holger Hans Peter Freyther +# +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +# THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# + +import optparse, os, sys + +# bitbake +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) +import bb +from bb import make +from string import split, join + +__version__ = "0.0.2" + +class HTMLFormatter: + """ + Simple class to help to generate some sort of HTML files. It is + quite inferior solution compared to docbook, gtkdoc, doxygen but it + should work for now. + We've a global introduction site (index.html) and then one site for + the list of keys (alphabetical sorted) and one for the list of groups, + one site for each key with links to the relations and groups. + + index.html + keys.html + groups.html + groupNAME.html + keyNAME.html + """ + + def replace(self, text, *pairs): + """ + From pydoc... almost identical at least + """ + while pairs: + (a,b) = pairs[0] + text = join(split(text, a), b) + pairs = pairs[1:] + return text + def escape(self, text): + """ + Escape string to be conform HTML + """ + return self.replace(text, + ('&', '&'), + ('<', '<' ), + ('>', '>' ) ) + def createNavigator(self): + """ + Create the navgiator + """ + return """ + + + + +
HomeGroupsKeys
+""" + + def relatedKeys(self, item): + """ + Create HTML to link to foreign keys + """ + + if len(item.related()) == 0: + return "" + + txt = "

See also:
" + for it in item.related(): + txt += """%s, """ % (it, it) + + return txt + + def groups(self,item): + """ + Create HTML to link to related groups + """ + + if len(item.groups()) == 0: + return "" + + + txt = "

Seel also:
" + for group in item.groups(): + txt += """%s, """ % (group,group) + + return txt + + + def createKeySite(self,item): + """ + Create a site for a key. It contains the header/navigator, a heading, + the description, links to related keys and to the groups. + """ + + return """ +Key %s + + +%s +

%s

+ +
+

Synopsis

+
+%s
+
+
+ +
+

Related Keys

+
+%s
+
+
+ +
+

Groups

+
+%s
+
+
+ + + +""" % (item.name(), self.createNavigator(), item.name(), + self.escape(item.description()), self.relatedKeys(item), self.groups(item)) + + def createGroupsSite(self, doc): + """ + Create the Group Overview site + """ + + groups = "" + sorted_groups = doc.groups() + sorted_groups.sort() + for group in sorted_groups: + groups += """%s
""" % (group, group) + + return """ +Group overview + + +%s +

Available Groups

+%s + +""" % (self.createNavigator(), groups) + + def createIndex(self): + """ + Create the index file + """ + + return """ +Bitbake Documentation + + +%s +

Documentation Entrance

+All available groups
+All available keys
+ +""" % self.createNavigator() + + def createKeysSite(self, doc): + """ + Create Overview of all avilable keys + """ + keys = "" + sorted_keys = doc.doc_keys() + sorted_keys.sort() + for key in sorted_keys: + keys += """%s
""" % (key, key) + + return """ +Key overview + + +%s +

Available Keys

+%s + +""" % (self.createNavigator(), keys) + + def createGroupSite(self,gr, items): + """ + Create a site for a group: + Group the name of the group, items contain the name of the keys + inside this group + """ + groups = "" + for group in items: + groups += """%s
""" % (group.name(), group.name()) + + return """ +Group %s + + +%s +
+

Keys in Group %s

+
+%s
+
+
+ +""" % (gr, self.createNavigator(), gr, groups) + + + + def createCSS(self): + """ + Create the CSS file + """ + return """.synopsis, .classsynopsis +{ + background: #eeeeee; + border: solid 1px #aaaaaa; + padding: 0.5em; +} +.programlisting +{ + background: #eeeeff; + border: solid 1px #aaaaff; + padding: 0.5em; +} +.variablelist +{ + padding: 4px; + margin-left: 3em; +} +.variablelist td:first-child +{ + vertical-align: top; +} +table.navigation +{ + background: #ffeeee; + border: solid 1px #ffaaaa; + margin-top: 0.5em; + margin-bottom: 0.5em; +} +.navigation a +{ + color: #770000; +} +.navigation a:visited +{ + color: #550000; +} +.navigation .title +{ + font-size: 200%; +} +div.refnamediv +{ + margin-top: 2em; +} +div.gallery-float +{ + float: left; + padding: 10px; +} +div.gallery-float img +{ + border-style: none; +} +div.gallery-spacer +{ + clear: both; +} +a +{ + text-decoration: none; +} +a:hover +{ + text-decoration: underline; + color: #FF0000; +} +""" + + + +class DocumentationItem: + """ + A class to hold information about a configuration + item. It contains the key name, description, a list of related names, + and the group this item is contained in. + """ + + def __init__(self): + self._groups = [] + self._related = [] + self._name = "" + self._desc = "" + + def groups(self): + return self._groups + + def name(self): + return self._name + + def description(self): + return self._desc + + def related(self): + return self._related + + def setName(self, name): + self._name = name + + def setDescription(self, desc): + self._desc = desc + + def addGroup(self, group): + self._groups.append(group) + + def addRelation(self,relation): + self._related.append(relation) + + def sort(self): + self._related.sort() + self._groups.sort() + + +class Documentation: + """ + Holds the documentation... with mappings from key to items... + """ + + def __init__(self): + self.__keys = {} + self.__groups = {} + + def insert_doc_item(self, item): + """ + Insert the Doc Item into the internal list + of representation + """ + item.sort() + self.__keys[item.name()] = item + + for group in item.groups(): + if not group in self.__groups: + self.__groups[group] = [] + self.__groups[group].append(item) + self.__groups[group].sort() + + + def doc_item(self, key): + """ + Return the DocumentationInstance describing the key + """ + try: + return self.__keys[key] + except KeyError: + return None + + def doc_keys(self): + """ + Return the documented KEYS (names) + """ + return self.__keys.keys() + + def groups(self): + """ + Return the names of available groups + """ + return self.__groups.keys() + + def group_content(self,group_name): + """ + Return a list of keys/names that are in a specefic + group or the empty list + """ + try: + return self.__groups[group_name] + except KeyError: + return [] + + +def parse_cmdline(args): + """ + Parse the CMD line and return the result as a n-tuple + """ + + parser = optparse.OptionParser( version = "Bitbake Documentation Tool Core version %s, %%prog version %s" % (bb.__version__,__version__)) + usage = """%prog [options] + +Create a set of html pages (documentation) for a bitbake.conf.... +""" + + # Add the needed options + parser.add_option( "-c", "--config", help = "Use the specified configuration file as source", + action = "store", dest = "config", default = os.path.join("conf", "documentation.conf") ) + + parser.add_option( "-o", "--output", help = "Output directory for html files", + action = "store", dest = "output", default = "html/" ) + + parser.add_option( "-D", "--debug", help = "Increase the debug level", + action = "count", dest = "debug", default = 0 ) + + parser.add_option( "-v","--verbose", help = "output more chit-char to the terminal", + action = "store_true", dest = "verbose", default = False ) + + options, args = parser.parse_args( sys.argv ) + + if options.debug: + bb.debug_level = options.debug + + return options.config, options.output + +def main(): + """ + The main Method + """ + + (config_file,output_dir) = parse_cmdline( sys.argv ) + + # right to let us load the file now + try: + documentation = bb.parse.handle( config_file, bb.data.init() ) + except IOError: + bb.fatal( "Unable to open %s" % config_file ) + except bb.parse.ParseError: + bb.fatal( "Unable to parse %s" % config_file ) + + + # Assuming we've the file loaded now, we will initialize the 'tree' + doc = Documentation() + + # defined states + state_begin = 0 + state_see = 1 + state_group = 2 + + for key in bb.data.keys(documentation): + data = bb.data.getVarFlag(key, "doc", documentation) + if not data: + continue + + # The Documentation now starts + doc_ins = DocumentationItem() + doc_ins.setName(key) + + + tokens = data.split(' ') + state = state_begin + string= "" + for token in tokens: + token = token.strip(',') + + if not state == state_see and token == "@see": + state = state_see + continue + elif not state == state_group and token == "@group": + state = state_group + continue + + if state == state_begin: + string += " %s" % token + elif state == state_see: + doc_ins.addRelation(token) + elif state == state_group: + doc_ins.addGroup(token) + + # set the description + doc_ins.setDescription(string) + doc.insert_doc_item(doc_ins) + + # let us create the HTML now + bb.mkdirhier(output_dir) + os.chdir(output_dir) + + # Let us create the sites now. We do it in the following order + # Start with the index.html. It will point to sites explaining all + # keys and groups + html_slave = HTMLFormatter() + + f = file('style.css', 'w') + print >> f, html_slave.createCSS() + + f = file('index.html', 'w') + print >> f, html_slave.createIndex() + + f = file('groups.html', 'w') + print >> f, html_slave.createGroupsSite(doc) + + f = file('keys.html', 'w') + print >> f, html_slave.createKeysSite(doc) + + # now for each group create the site + for group in doc.groups(): + f = file('group%s.html' % group, 'w') + print >> f, html_slave.createGroupSite(group, doc.group_content(group)) + + # now for the keys + for key in doc.doc_keys(): + f = file('key%s.html' % doc.doc_item(key).name(), 'w') + print >> f, html_slave.createKeySite(doc.doc_item(key)) + + +if __name__ == "__main__": + main() diff --git a/bitbake/classes/.gitignore b/bitbake/classes/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/classes/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/classes/base.bbclass b/bitbake/classes/base.bbclass new file mode 100644 index 0000000..1d75964 --- /dev/null +++ b/bitbake/classes/base.bbclass @@ -0,0 +1,79 @@ +# Copyright (C) 2003 Chris Larson +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +die() { + bbfatal "$*" +} + +bbnote() { + echo "NOTE:" "$*" +} + +bbwarn() { + echo "WARNING:" "$*" +} + +bbfatal() { + echo "FATAL:" "$*" + exit 1 +} + +bbdebug() { + test $# -ge 2 || { + echo "Usage: bbdebug level \"message\"" + exit 1 + } + + test ${@bb.debug_level} -ge $1 && { + shift + echo "DEBUG:" $* + } +} + +addtask showdata +do_showdata[nostamp] = "1" +python do_showdata() { + import sys + # emit variables and shell functions + bb.data.emit_env(sys.__stdout__, d, True) + # emit the metadata which isnt valid shell + for e in bb.data.keys(d): + if bb.data.getVarFlag(e, 'python', d): + sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, bb.data.getVar(e, d, 1))) +} + +addtask listtasks +do_listtasks[nostamp] = "1" +python do_listtasks() { + import sys + for e in bb.data.keys(d): + if bb.data.getVarFlag(e, 'task', d): + sys.__stdout__.write("%s\n" % e) +} + +addtask build +do_build[dirs] = "${TOPDIR}" +do_build[nostamp] = "1" +python base_do_build () { + bb.note("The included, default BB base.bbclass does not define a useful default task.") + bb.note("Try running the 'listtasks' task against a .bb to see what tasks are defined.") +} + +EXPORT_FUNCTIONS do_clean do_mrproper do_build diff --git a/bitbake/conf/.gitignore b/bitbake/conf/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/conf/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/conf/bitbake.conf b/bitbake/conf/bitbake.conf new file mode 100644 index 0000000..fd216a3 --- /dev/null +++ b/bitbake/conf/bitbake.conf @@ -0,0 +1,55 @@ +# Copyright (C) 2003 Chris Larson +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +B = "${S}" +CVSDIR = "${DL_DIR}/cvs" +DEPENDS = "" +DEPLOY_DIR = "${TMPDIR}/deploy" +DEPLOY_DIR_IMAGE = "${DEPLOY_DIR}/images" +DL_DIR = "${TMPDIR}/downloads" +FETCHCOMMAND = "" +FETCHCOMMAND_cvs = "/usr/bin/env cvs -d${CVSROOT} co ${CVSCOOPTS} ${CVSMODULE}" +FETCHCOMMAND_svn = "/usr/bin/env svn co http://${SVNROOT} ${SVNCOOPTS} ${SVNMODULE}" +FETCHCOMMAND_wget = "/usr/bin/env wget -t 5 --passive-ftp -P ${DL_DIR} ${URI}" +FILESDIR = "${@bb.which(bb.data.getVar('FILESPATH', d, 1), '.')}" +FILESPATH = "${FILE_DIRNAME}/${PF}:${FILE_DIRNAME}/${P}:${FILE_DIRNAME}/${PN}:${FILE_DIRNAME}/files:${FILE_DIRNAME}" +FILE_DIRNAME = "${@os.path.dirname(bb.data.getVar('FILE', d))}" +IMAGE_CMD = "_NO_DEFINED_IMAGE_TYPES_" +IMAGE_ROOTFS = "${TMPDIR}/rootfs" +MKTEMPCMD = "mktemp -q ${TMPBASE}" +MKTEMPDIRCMD = "mktemp -d -q ${TMPBASE}" +OVERRIDES = "local:${MACHINE}:${TARGET_OS}:${TARGET_ARCH}" +P = "${PN}-${PV}" +PF = "${PN}-${PV}-${PR}" +PN = "${@bb.parse.BBHandler.vars_from_file(bb.data.getVar('FILE',d),d)[0] or 'defaultpkgname'}" +PR = "${@bb.parse.BBHandler.vars_from_file(bb.data.getVar('FILE',d),d)[2] or 'r0'}" +PROVIDES = "" +PV = "${@bb.parse.BBHandler.vars_from_file(bb.data.getVar('FILE',d),d)[1] or '1.0'}" +RESUMECOMMAND = "" +RESUMECOMMAND_wget = "/usr/bin/env wget -c -t 5 --passive-ftp -P ${DL_DIR} ${URI}" +S = "${WORKDIR}/${P}" +SRC_URI = "file://${FILE}" +STAMP = "${TMPDIR}/stamps/${PF}" +T = "${WORKDIR}/temp" +TARGET_ARCH = "${BUILD_ARCH}" +TMPDIR = "${TOPDIR}/tmp" +UPDATECOMMAND = "" +UPDATECOMMAND_cvs = "/usr/bin/env cvs update ${CVSCOOPTS}" +WORKDIR = "${TMPDIR}/work/${PF}" diff --git a/bitbake/contrib/.gitignore b/bitbake/contrib/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/contrib/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/contrib/README b/bitbake/contrib/README new file mode 100644 index 0000000..25e5156 --- /dev/null +++ b/bitbake/contrib/README @@ -0,0 +1 @@ +This directory is for additional contributed files which may be useful. diff --git a/bitbake/contrib/bbdev.sh b/bitbake/contrib/bbdev.sh new file mode 100644 index 0000000..33a7853 --- /dev/null +++ b/bitbake/contrib/bbdev.sh @@ -0,0 +1,31 @@ +# This is a shell function to be sourced into your shell or placed in your .profile, +# which makes setting things up for BitBake a bit easier. +# +# The author disclaims copyright to the contents of this file and places it in the +# public domain. + +bbdev () { + local BBDIR PKGDIR BUILDDIR + if test x"$1" = "x--help"; then echo >&2 "syntax: bbdev [bbdir [pkgdir [builddir]]]"; return 1; fi + if test x"$1" = x; then BBDIR=`pwd`; else BBDIR=$1; fi + if test x"$2" = x; then PKGDIR=`pwd`; else PKGDIR=$2; fi + if test x"$3" = x; then BUILDDIR=`pwd`; else BUILDDIR=$3; fi + + BBDIR=`readlink -f $BBDIR` + PKGDIR=`readlink -f $PKGDIR` + BUILDDIR=`readlink -f $BUILDDIR` + if ! (test -d $BBDIR && test -d $PKGDIR && test -d $BUILDDIR); then + echo >&2 "syntax: bbdev [bbdir [pkgdir [builddir]]]" + return 1 + fi + + PATH=$BBDIR/bin:$PATH + BBPATH=$BBDIR + if test x"$BBDIR" != x"$PKGDIR"; then + BBPATH=$PKGDIR:$BBPATH + fi + if test x"$PKGDIR" != x"$BUILDDIR"; then + BBPATH=$BUILDDIR:$BBPATH + fi + export BBPATH +} diff --git a/bitbake/contrib/vim/.gitignore b/bitbake/contrib/vim/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/contrib/vim/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/contrib/vim/syntax/.gitignore b/bitbake/contrib/vim/syntax/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/contrib/vim/syntax/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/contrib/vim/syntax/bitbake.vim b/bitbake/contrib/vim/syntax/bitbake.vim new file mode 100644 index 0000000..5d2bc63 --- /dev/null +++ b/bitbake/contrib/vim/syntax/bitbake.vim @@ -0,0 +1,120 @@ +" Vim syntax file +" +" Copyright (C) 2004 Chris Larson +" This file is licensed under the MIT license, see COPYING.MIT in +" this source distribution for the terms. +" +" Language: BitBake +" Maintainer: Chris Larson +" Filenames: *.bb, *.bbclass + +if version < 600 + syntax clear +elseif exists("b:current_syntax") + finish +endif + +syn case match + + +" Catch incorrect syntax (only matches if nothing else does) +" +syn match bbUnmatched "." + + +" Other + +syn match bbComment "^#.*$" display contains=bbTodo +syn keyword bbTodo TODO FIXME XXX contained +syn match bbDelimiter "[(){}=]" contained +syn match bbQuote /['"]/ contained +syn match bbArrayBrackets "[\[\]]" contained + + +" BitBake strings + +syn match bbContinue "\\$" +syn region bbString matchgroup=bbQuote start=/"/ skip=/\\$/ excludenl end=/"/ contained keepend contains=bbTodo,bbContinue,bbVarDeref +syn region bbString matchgroup=bbQuote start=/'/ skip=/\\$/ excludenl end=/'/ contained keepend contains=bbTodo,bbContinue,bbVarDeref + + +" BitBake variable metadata + +syn keyword bbExportFlag export contained nextgroup=bbIdentifier skipwhite +syn match bbVarDeref "${[a-zA-Z0-9\-_\.]\+}" contained +syn match bbVarDef "^\(export\s*\)\?\([a-zA-Z0-9\-_\.]\+\(_[${}a-zA-Z0-9\-_\.]\+\)\?\)\s*\(\(:=\)\|\(+=\)\|\(=+\)\|\(?=\)\|=\)\@=" contains=bbExportFlag,bbIdentifier,bbVarDeref nextgroup=bbVarEq + +syn match bbIdentifier "[a-zA-Z0-9\-_\.]\+" display contained +"syn keyword bbVarEq = display contained nextgroup=bbVarValue +syn match bbVarEq "\(:=\)\|\(+=\)\|\(=+\)\|\(?=\)\|=" contained nextgroup=bbVarValue +syn match bbVarValue ".*$" contained contains=bbString,bbVarDeref + + +" BitBake variable metadata flags +syn match bbVarFlagDef "^\([a-zA-Z0-9\-_\.]\+\)\(\[[a-zA-Z0-9\-_\.]\+\]\)\@=" contains=bbIdentifier nextgroup=bbVarFlagFlag +syn region bbVarFlagFlag matchgroup=bbArrayBrackets start="\[" end="\]\s*\(=\)\@=" keepend excludenl contained contains=bbIdentifier nextgroup=bbVarEq +"syn match bbVarFlagFlag "\[\([a-zA-Z0-9\-_\.]\+\)\]\s*\(=\)\@=" contains=bbIdentifier nextgroup=bbVarEq + + +" Functions! +syn match bbFunction "\h\w*" display contained + + +" BitBake python metadata +syn include @python syntax/python.vim +if exists("b:current_syntax") + unlet b:current_syntax +endif + +syn keyword bbPythonFlag python contained nextgroup=bbFunction +syn match bbPythonFuncDef "^\(python\s\+\)\(\w\+\)\?\(\s*()\s*\)\({\)\@=" contains=bbPythonFlag,bbFunction,bbDelimiter nextgroup=bbPythonFuncRegion skipwhite +syn region bbPythonFuncRegion matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" keepend contained contains=@python +"hi def link bbPythonFuncRegion Comment + + +" BitBake shell metadata +syn include @shell syntax/sh.vim +if exists("b:current_syntax") + unlet b:current_syntax +endif + +syn keyword bbFakerootFlag fakeroot contained nextgroup=bbFunction +syn match bbShellFuncDef "^\(fakeroot\s*\)\?\(\w\+\)\(python\)\@ + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/bitbake/doc/COPYING.MIT b/bitbake/doc/COPYING.MIT new file mode 100644 index 0000000..7e7d574 --- /dev/null +++ b/bitbake/doc/COPYING.MIT @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/bitbake/doc/manual/.gitignore b/bitbake/doc/manual/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/doc/manual/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/doc/manual/Makefile b/bitbake/doc/manual/Makefile new file mode 100644 index 0000000..a43c025 --- /dev/null +++ b/bitbake/doc/manual/Makefile @@ -0,0 +1,56 @@ +topdir = . +manual = $(topdir)/usermanual.xml +# types = pdf txt rtf ps xhtml html man tex texi dvi +# types = pdf txt +types = $(xmltotypes) $(htmltypes) +xmltotypes = pdf txt +htmltypes = html xhtml +htmlxsl = $(if $(filter $@,$(foreach type,$(htmltypes),$(type)-nochunks)),http://docbook.sourceforge.net/release/xsl/current/xhtml/docbook.xsl,http://docbook.sourceforge.net/release/xsl/current/$@/chunk.xsl) +htmlcssfile = docbook.css +htmlcss = $(topdir)/html.css +# htmlcssfile = +# htmlcss = +cleanfiles = $(foreach i,$(types),$(topdir)/$(i)) + +ifdef DEBUG +define command + $(1) +endef +else +define command + @echo $(2) $(3) $(4) + @$(1) >/dev/null +endef +endif + +all: $(types) + +lint: $(manual) FORCE + $(call command,xmllint --xinclude --postvalid --noout $(manual),XMLLINT $(manual)) + +$(types) $(foreach type,$(htmltypes),$(type)-nochunks): lint FORCE + +$(foreach type,$(htmltypes),$(type)-nochunks): $(if $(htmlcss),$(htmlcss)) $(manual) + @mkdir -p $@ +ifdef htmlcss + $(call command,install -m 0644 $(htmlcss) $@/$(htmlcssfile),CP $(htmlcss) $@/$(htmlcssfile)) +endif + $(call command,xsltproc --stringparam base.dir $@/ $(if $(htmlcssfile),--stringparam html.stylesheet $(htmlcssfile)) $(htmlxsl) $(manual) > $@/index.$(patsubst %-nochunks,%,$@),XSLTPROC $@ $(manual)) + +$(htmltypes): $(if $(htmlcss),$(htmlcss)) $(manual) + @mkdir -p $@ +ifdef htmlcss + $(call command,install -m 0644 $(htmlcss) $@/$(htmlcssfile),CP $(htmlcss) $@/$(htmlcssfile)) +endif + $(call command,xsltproc --stringparam base.dir $@/ $(if $(htmlcssfile),--stringparam html.stylesheet $(htmlcssfile)) $(htmlxsl) $(manual),XSLTPROC $@ $(manual)) + +$(xmltotypes): $(manual) + $(call command,xmlto --extensions -o $(topdir)/$@ $@ $(manual),XMLTO $@ $(manual)) + +clean: + rm -rf $(cleanfiles) + +$(foreach i,$(types) $(foreach type,$(htmltypes),$(type)-nochunks),clean-$(i)): + rm -rf $(patsubst clean-%,%,$@) + +FORCE: diff --git a/bitbake/doc/manual/html.css b/bitbake/doc/manual/html.css new file mode 100644 index 0000000..6eedfd3 --- /dev/null +++ b/bitbake/doc/manual/html.css @@ -0,0 +1,281 @@ +/* Feuille de style DocBook du projet Traduc.org */ +/* DocBook CSS stylesheet of the Traduc.org project */ + +/* (c) Jean-Philippe Guérard - 14 août 2004 */ +/* (c) Jean-Philippe Guérard - 14 August 2004 */ + +/* Cette feuille de style est libre, vous pouvez la */ +/* redistribuer et la modifier selon les termes de la Licence */ +/* Art Libre. Vous trouverez un exemplaire de cette Licence sur */ +/* http://tigreraye.org/Petit-guide-du-traducteur.html#licence-art-libre */ + +/* This work of art is free, you can redistribute it and/or */ +/* modify it according to terms of the Free Art license. You */ +/* will find a specimen of this license on the Copyleft */ +/* Attitude web site: http://artlibre.org as well as on other */ +/* sites. */ +/* Please note that the French version of this licence as shown */ +/* on http://tigreraye.org/Petit-guide-du-traducteur.html#licence-art-libre */ +/* is only official licence of this document. The English */ +/* is only provided to help you understand this licence. */ + +/* La dernière version de cette feuille de style est toujours */ +/* disponible sur : http://tigreraye.org/style.css */ +/* Elle est également disponible sur : */ +/* http://www.traduc.org/docs/HOWTO/lecture/style.css */ + +/* The latest version of this stylesheet is available from: */ +/* http://tigreraye.org/style.css */ +/* It is also available on: */ +/* http://www.traduc.org/docs/HOWTO/lecture/style.css */ + +/* N'hésitez pas à envoyer vos commentaires et corrections à */ +/* Jean-Philippe Guérard */ + +/* Please send feedback and bug reports to */ +/* Jean-Philippe Guérard */ + +/* $Id: style.css,v 1.14 2004/09/10 20:12:09 fevrier Exp fevrier $ */ + +/* Présentation générale du document */ +/* Overall document presentation */ + +body { + /* + font-family: Apolline, "URW Palladio L", Garamond, jGaramond, + "Bitstream Cyberbit", "Palatino Linotype", serif; + */ + margin: 7%; + background-color: white; +} + +/* Taille du texte */ +/* Text size */ + +* { font-size: 100%; } + +/* Gestion des textes mis en relief imbriqués */ +/* Embedded emphasis */ + +em { font-style: italic; } +em em { font-style: normal; } +em em em { font-style: italic; } + +/* Titres */ +/* Titles */ + +h1 { font-size: 200%; font-weight: 900; } +h2 { font-size: 160%; font-weight: 900; } +h3 { font-size: 130%; font-weight: bold; } +h4 { font-size: 115%; font-weight: bold; } +h5 { font-size: 108%; font-weight: bold; } +h6 { font-weight: bold; } + +/* Nom de famille en petites majuscules (uniquement en français) */ +/* Last names in small caps (for French only) */ + +*[class~="surname"]:lang(fr) { font-variant: small-caps; } + +/* Blocs de citation */ +/* Quotation blocs */ + +div[class~="blockquote"] { + border: solid 2px #AAA; + padding: 5px; + margin: 5px; +} + +div[class~="blockquote"] > table { + border: none; +} + +/* Blocs litéraux : fond gris clair */ +/* Literal blocs: light gray background */ + +*[class~="literallayout"] { + background: #f0f0f0; + padding: 5px; + margin: 5px; +} + +/* Programmes et captures texte : fond bleu clair */ +/* Listing and text screen snapshots: light blue background */ + +*[class~="programlisting"], *[class~="screen"] { + background: #f0f0ff; + padding: 5px; + margin: 5px; +} + +/* Les textes à remplacer sont surlignés en vert pâle */ +/* Replaceable text in highlighted in pale green */ + +*[class~="replaceable"] { + background-color: #98fb98; + font-style: normal; } + +/* Tables : fonds gris clair & bords simples */ +/* Tables: light gray background and solid borders */ + +*[class~="table"] *[class~="title"] { width:100%; border: 0px; } + +table { + border: 1px solid #aaa; + border-collapse: collapse; + padding: 2px; + margin: 5px; +} + +/* Listes simples en style table */ +/* Simples lists in table presentation */ + +table[class~="simplelist"] { + background-color: #F0F0F0; + margin: 5px; + border: solid 1px #AAA; +} + +table[class~="simplelist"] td { + border: solid 1px #AAA; +} + +/* Les tables */ +/* Tables */ + +*[class~="table"] table { + background-color: #F0F0F0; + border: solid 1px #AAA; +} +*[class~="informaltable"] table { background-color: #F0F0F0; } + +th,td { + vertical-align: baseline; + text-align: left; + padding: 0.1em 0.3em; + empty-cells: show; +} + +/* Alignement des colonnes */ +/* Colunms alignment */ + +td[align=center] , th[align=center] { text-align: center; } +td[align=right] , th[align=right] { text-align: right; } +td[align=left] , th[align=left] { text-align: left; } +td[align=justify] , th[align=justify] { text-align: justify; } + +/* Pas de marge autour des images */ +/* No inside margins for images */ + +img { border: 0; } + +/* Les liens ne sont pas soulignés */ +/* No underlines for links */ + +:link , :visited , :active { text-decoration: none; } + +/* Prudence : cadre jaune et fond jaune clair */ +/* Caution: yellow border and light yellow background */ + +*[class~="caution"] { + border: solid 2px yellow; + background-color: #ffffe0; + padding: 1em 6px 1em ; + margin: 5px; +} + +*[class~="caution"] th { + vertical-align: middle +} + +*[class~="caution"] table { + background-color: #ffffe0; + border: none; +} + +/* Note importante : cadre jaune et fond jaune clair */ +/* Important: yellow border and light yellow background */ + +*[class~="important"] { + border: solid 2px yellow; + background-color: #ffffe0; + padding: 1em 6px 1em; + margin: 5px; +} + +*[class~="important"] th { + vertical-align: middle +} + +*[class~="important"] table { + background-color: #ffffe0; + border: none; +} + +/* Mise en évidence : texte légèrement plus grand */ +/* Highlights: slightly larger texts */ + +*[class~="highlights"] { + font-size: 110%; +} + +/* Note : cadre bleu et fond bleu clair */ +/* Notes: blue border and light blue background */ + +*[class~="note"] { + border: solid 2px #7099C5; + background-color: #f0f0ff; + padding: 1em 6px 1em ; + margin: 5px; +} + +*[class~="note"] th { + vertical-align: middle +} + +*[class~="note"] table { + background-color: #f0f0ff; + border: none; +} + +/* Astuce : cadre vert et fond vert clair */ +/* Tip: green border and light green background */ + +*[class~="tip"] { + border: solid 2px #00ff00; + background-color: #f0ffff; + padding: 1em 6px 1em ; + margin: 5px; +} + +*[class~="tip"] th { + vertical-align: middle; +} + +*[class~="tip"] table { + background-color: #f0ffff; + border: none; +} + +/* Avertissement : cadre rouge et fond rouge clair */ +/* Warning: red border and light red background */ + +*[class~="warning"] { + border: solid 2px #ff0000; + background-color: #fff0f0; + padding: 1em 6px 1em ; + margin: 5px; +} + +*[class~="warning"] th { + vertical-align: middle; +} + + +*[class~="warning"] table { + background-color: #fff0f0; + border: none; +} + +/* Fin */ +/* The End */ + diff --git a/bitbake/doc/manual/usermanual.xml b/bitbake/doc/manual/usermanual.xml new file mode 100644 index 0000000..b96863c --- /dev/null +++ b/bitbake/doc/manual/usermanual.xml @@ -0,0 +1,361 @@ + + + + + + BitBake User Manual + + BitBake Team + + + 2004, 2005 + Chris Larson + Phil Blundell + + + This work is licensed under the Creative Commons Attribution License. To view a copy of this license, visit http://creativecommons.org/licenses/by/2.0/ or send a letter to Creative Commons, 559 Nathan Abbott Way, Stanford, California 94305, USA. + + + + Introduction +
+ Overview + BitBake is, at its simplest, a tool for executing +tasks and managing metadata. As such, its similarities to GNU make and other +build tools are readily apparent. It was inspired by Portage, the package management system used by the Gentoo Linux distribution. BitBake is the basis of the OpenEmbedded project, which is being used to build and maintain a number of embedded Linux distributions, including OpenZaurus and Familiar. +
+
+ Background and Goals + Prior to BitBake, no other build tool adequately met +the needs of an aspiring embedded Linux distribution. All of the +buildsystems used by traditional desktop Linux distributions lacked +important functionality, and none of the ad-hoc +buildroot systems, prevalent in the +embedded space, were scalable or maintainable. + + Some important goals for BitBake were: + + Handle crosscompilation. + Handle interpackage dependencies (build time on target architecture, build time on native architecture, and runtime). + Support running any number of tasks within a given package, including, but not limited to, fetching upstream sources, unpacking them, patching them, configuring them, et cetera. + Must be linux distribution agnostic (both build and target). + Must be architecture agnostic + Must support multiple build and target operating systems (including cygwin, the BSDs, etc). + Must be able to be self contained, rather than tightly integrated into the build machine's root filesystem. + There must be a way to handle conditional metadata (on target architecture, operating system, distribution, machine). + It must be easy for the person using the tools to supply their own local metadata and packages to operate against. + Must make it easy to collaborate +between multiple projects using BitBake for their +builds. + Should provide an inheritance mechanism to +share common metadata between many packages. + Et cetera... + + + BitBake satisfies all these and many more. Flexibility and power have always been the priorities. It is highly extensible, supporting embedded Python code and execution of any arbitrary tasks. +
+ + + Metadata +
+ Description + + BitBake metadata can be classified into 3 major areas: + + Configuration Files + + + .bb Files + + + Classes + + + What follows are a large number of examples of BitBake metadata. Any syntax which isn't supported in any of the aforementioned areas will be documented as such. +
+ Basic variable setting + VARIABLE = "value" + In this example, VARIABLE is value. +
+
+ Variable expansion + BitBake supports variables referencing one another's contents using a syntax which is similar to shell scripting + A = "aval" +B = "pre${A}post" + This results in A containing aval and B containing preavalpost. +
+
+ Immediate variable expansion (:=) + := results in a variable's contents being expanded immediately, rather than when the variable is actually used. + T = "123" +A := "${B} ${A} test ${T}" +T = "456" +B = "${T} bval" + +C = "cval" +C := "${C}append" + In that example, A would contain test 123, B would contain 456 bval, and C would be cvalappend. +
+
+ Appending (+=) and prepending (=+) + B = "bval" +B += "additionaldata" +C = "cval" +C =+ "test" + In this example, B is now bval additionaldata and C is test cval. +
+
+ Appending (.=) and prepending (=.) without spaces + B = "bval" +B += "additionaldata" +C = "cval" +C =+ "test" + In this example, B is now bvaladditionaldata and C is testcval. In contrast to the above Appending and Prepending operators no additional space +will be introduced. +
+
+ Conditional metadata set + OVERRIDES is a : seperated variable containing each item you want to satisfy conditions. So, if you have a variable which is conditional on arm, and arm is in OVERRIDES, then the arm specific version of the variable is used rather than the non-conditional version. Example: + OVERRIDES = "architecture:os:machine" +TEST = "defaultvalue" +TEST_os = "osspecificvalue" +TEST_condnotinoverrides = "othercondvalue" + In this example, TEST would be osspecificvalue, due to the condition os being in OVERRIDES. +
+
+ Conditional appending + BitBake also supports appending and prepending to variables based on whether something is in OVERRIDES. Example: + DEPENDS = "glibc ncurses" +OVERRIDES = "machine:local" +DEPENDS_append_machine = " libmad" + In this example, DEPENDS is set to glibc ncurses libmad. +
+
+ Inclusion + Next, there is the include directive, which causes BitBake to parse in whatever file you specify, and insert it at that location, which is not unlike make. However, if the path specified on the include line is a relative path, BitBake will locate the first one it can find within BBPATH. +
+
+ Python variable expansion + DATE = "${@time.strftime('%Y%m%d',time.gmtime())}" + This would result in the DATE variable containing today's date. +
+
+ Defining executable metadata + NOTE: This is only supported in .bb and .bbclass files. + do_mytask () { + echo "Hello, world!" +} + This is essentially identical to setting a variable, except that this variable happens to be executable shell code. + python do_printdate () { + import time + print time.strftime('%Y%m%d', time.gmtime()) +} + This is the similar to the previous, but flags it as python so that BitBake knows it is python code. +
+
+ Defining python functions into the global python namespace + NOTE: This is only supported in .bb and .bbclass files. + def get_depends(bb, d): + if bb.data.getVar('SOMECONDITION', d, True): + return "dependencywithcond" + else: + return "dependency" + +SOMECONDITION = "1" +DEPENDS = "${@get_depends(bb, d)}" + This would result in DEPENDS containing dependencywithcond. +
+
+ Inheritance + NOTE: This is only supported in .bb and .bbclass files. + The inherit directive is a means of specifying what classes of functionality your .bb requires. It is a rudamentary form of inheritence. For example, you can easily abstract out the tasks involved in building a package that uses autoconf and automake, and put that into a bbclass for your packages to make use of. A given bbclass is located by searching for classes/filename.oeclass in BBPATH, where filename is what you inherited. +
+
+ Tasks + NOTE: This is only supported in .bb and .bbclass files. + In BitBake, each step that needs to be run for a given .bb is known as a task. There is a command addtask to add new tasks (must be a defined python executable metadata and must start with do_) and describe intertask dependencies. + python do_printdate () { + import time + print time.strftime('%Y%m%d', time.gmtime()) +} + +addtask printdate before do_build + This defines the necessary python function and adds it as a task which is now a dependency of do_build (the default task). If anyone executes the do_build task, that will result in do_printdate being run first. +
+
+ Events + NOTE: This is only supported in .bb and .bbclass files. + BitBake also implements a means of registering event handlers. Events are triggered at certain points during operation, such as, the beginning of operation against a given .bb, the start of a given task, task failure, task success, et cetera. The intent was to make it easy to do things like email notifications on build failure. + addhandler myclass_eventhandler +python myclass_eventhandler() { + from bb.event import NotHandled, getName + from bb import data + + print "The name of the Event is %s" % getName(e) + print "The file we run for is %s" % data.getVar('FILE', e.data, True) + + return NotHandled + +This event handler gets called every time an event is triggered. A global variable e is defined. e.data contains an instance of bb.data. With the getName(e) +method one can get the name of the triggered event.The above event handler prints the name +of the event and the content of the FILE variable. +
+
+
+ Parsing +
+ Configuration Files + The first of the classifications of metadata in BitBake is configuration metadata. This metadata is global, and therefore affects all packages and tasks which are executed. Currently, BitBake has hardcoded knowledge of a single configuration file. It expects to find 'conf/bitbake.conf' somewhere in the user specified BBPATH. That configuration file generally has include directives to pull in any other metadata (generally files specific to architecture, machine, local and so on. + Only variable definitions and include directives are allowed in .conf files. +
+
+ Classes + BitBake classes are our rudamentary inheritence mechanism. As briefly mentioned in the metadata introduction, they're parsed when an inherit directive is encountered, and they are located in classes/ relative to the dirs in BBPATH. +
+
+ .bb Files + A BitBake (.bb) file is a logical unit of tasks to be executed. Normally this is a package to be built. Inter-.bb dependencies are obeyed. The files themselves are located via the BBFILES variable, which is set to a space seperated list of .bb files, and does handle wildcards. +
+
+ + + Commands +
+ bbread + bbread is a command for displaying BitBake metadata. When run with no arguments, it has the core parse 'conf/bitbake.conf', as located in BBPATH, and displays that. If you supply a file on the commandline, such as a .bb, then it parses that afterwards, using the aforementioned configuration metadata. + NOTE: the stand a lone bbread command was removed. Instead of bbread use bitbake -e. + +
+
+ bitbake +
+ Introduction + bitbake is the primary command in the system. It facilitates executing tasks in a single .bb file, or executing a given task on a set of multiple .bb files, accounting for interdependencies amongst them. +
+
+ Usage and Syntax + + $ bitbake --help +usage: bitbake [options] [package ...] + +Executes the specified task (default is 'build') for a given set of BitBake files. +It expects that BBFILES is defined, which is a space seperated list of files to +be executed. BBFILES does support wildcards. +Default BBFILES are the .bb files in the current directory. + +options: + --version show program's version number and exit + -h, --help show this help message and exit + -b BUILDFILE, --buildfile=BUILDFILE + execute the task against this .bb file, rather than a + package from BBFILES. + -k, --continue continue as much as possible after an error. While the + target that failed, and those that depend on it, + cannot be remade, the other dependencies of these + targets can be processed all the same. + -f, --force force run of specified cmd, regardless of stamp status + -i, --interactive drop into the interactive mode. + -c CMD, --cmd=CMD Specify task to execute. Note that this only executes + the specified task for the providee and the packages + it depends on, i.e. 'compile' does not implicitly call + stage for the dependencies (IOW: use only if you know + what you are doing) + -r FILE, --read=FILE read the specified file before bitbake.conf + -v, --verbose output more chit-chat to the terminal + -D, --debug Increase the debug level + -n, --dry-run don't execute, just go through the motions + -p, --parse-only quit after parsing the BB files (developers only) + -d, --disable-psyco disable using the psyco just-in-time compiler (not + recommended) + -s, --show-versions show current and preferred versions of all packages + -e, --environment show the global or per-package environment (this is + what used to be bbread) + + + + + + Executing a task against a single .bb + Executing tasks for a single file is relatively simple. You specify the file in question, and bitbake parses it and executes the specified task (or build by default). It obeys intertask dependencies when doing so. + clean task: + $ bitbake -b blah_1.0.bb -c clean + build task: + $ bitbake -b blah_1.0.bb + + + + + Executing tasks against a set of .bb files + There are a number of additional complexities introduced when one wants to manage multiple .bb files. Clearly there needs to be a way to tell bitbake what files are available, and of those, which we want to execute at this time. There also needs to be a way for each .bb to express its dependencies, both for build time and runtime. There must be a way for the user to express their preferences when multiple .bb's provide the same functionality, or when there are multiple versions of a .bb. + The next section, Metadata, outlines how one goes about specifying such things. + Note that the bitbake command, when not using --buildfile, accepts a PROVIDER, not a filename or anything else. By default, a .bb generally PROVIDES its packagename, packagename-version, and packagename-version-revision. + $ bitbake blah + $ bitbake blah-1.0 + $ bitbake blah-1.0-r0 + $ bitbake -c clean blah + $ bitbake virtual/whatever + $ bitbake -c clean virtual/whatever + + +
+
+ Metadata + As you may have seen in the usage information, or in the information about .bb files, the BBFILES variable is how the bitbake tool locates its files. This variable is a space seperated list of files that are available, and supports wildcards. + + Setting BBFILES + BBFILES = "/path/to/bbfiles/*.bb" + + With regard to dependencies, it expects the .bb to define a DEPENDS variable, which contains a space seperated list of package names, which themselves are the PN variable. The PN variable is, in general, by default, set to a component of the .bb filename. + + Depending on another .bb + a.bb: + PN = "package-a" + DEPENDS += "package-b" + + b.bb: + PN = "package-b" + + + + Using PROVIDES + This example shows the usage of the PROVIDES variable, which allows a given .bb to specify what functionality it provides. + package1.bb: + PROVIDES += "virtual/package" + + package2.bb: + DEPENDS += "virtual/package" + + package3.bb: + PROVIDES += "virtual/package" + + As you can see, here there are two different .bb's that provide the same functionality (virtual/package). Clearly, there needs to be a way for the person running bitbake to control which of those providers gets used. There is, indeed, such a way. + The following would go into a .conf file, to select package1: + PREFERRED_PROVIDER_virtual/package = "package1" + + + + Specifying version preference + When there are multiple versions of a given package, bitbake defaults to selecting the most recent version, unless otherwise specified. If the .bb in question has a DEFAULT_PREFERENCE set lower than the other .bb's (default is 0), then it will not be selected. This allows the person or persons maintaining the repository of .bb files to specify their preferences for the default selected version. In addition, the user can specify their preferences with regard to version. + If the first .bb is named a_1.1.bb, then the PN variable will be set to a, and the PV variable will be set to 1.1. + If we then have an a_1.2.bb, bitbake will choose 1.2 by default. However, if we define the following variable in a .conf that bitbake parses, we can change that. + PREFERRED_VERSION_a = "1.1" + + + + Using <quote>bbfile collections</quote> + bbfile collections exist to allow the user to have multiple repositories of bbfiles that contain the same exact package. For example, one could easily use them to make one's own local copy of an upstream repository, but with custom modifications that one does not want upstream. Usage: + BBFILES = "/stuff/openembedded/*/*.bb /stuff/openembedded.modified/*/*.bb" +BBFILE_COLLECTIONS = "upstream local" +BBFILE_PATTERN_upstream = "^/stuff/openembedded/" +BBFILE_PATTERN_local = "^/stuff/openembedded.modified/" +BBFILE_PRIORITY_upstream = "5" +BBFILE_PRIORITY_local = "10" + +
+
+ + diff --git a/bitbake/lib/.gitignore b/bitbake/lib/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/lib/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/lib/bb/.gitignore b/bitbake/lib/bb/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/lib/bb/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py new file mode 100644 index 0000000..f27f53b --- /dev/null +++ b/bitbake/lib/bb/__init__.py @@ -0,0 +1,1266 @@ +#!/usr/bin/python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake Build System Python Library + +Copyright (C) 2003 Holger Schurig +Copyright (C) 2003, 2004 Chris Larson + +Based on Gentoo's portage.py. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. +""" + +__version__ = "1.3.2.1" + +__all__ = [ + + "debug", + "note", + "error", + "fatal", + + "mkdirhier", + "movefile", + + "tokenize", + "evaluate", + "flatten", + "relparse", + "ververify", + "isjustname", + "isspecific", + "pkgsplit", + "catpkgsplit", + "vercmp", + "pkgcmp", + "dep_parenreduce", + "dep_opconvert", + "digraph", + +# fetch + "decodeurl", + "encodeurl", + +# modules + "parse", + "data", + "event", + "build", + "fetch", + "manifest" + ] + +whitespace = '\t\n\x0b\x0c\r ' +lowercase = 'abcdefghijklmnopqrstuvwxyz' + +import sys, os, types, re, string + +#projectdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) +projectdir = os.getcwd() + +debug_level = 0 + +if "BBDEBUG" in os.environ: + level = int(os.environ["BBDEBUG"]) + if level: + debug_level = level + else: + debug_level = 0 + +class VarExpandError(Exception): + pass + +class MalformedUrl(Exception): + """Exception raised when encountering an invalid url""" + + +####################################################################### +####################################################################### +# +# SECTION: Debug +# +# PURPOSE: little functions to make yourself known +# +####################################################################### +####################################################################### + +debug_prepend = '' + + +def debug(lvl, *args): + if debug_level >= lvl: + print debug_prepend + 'DEBUG:', ''.join(args) + +def note(*args): + print debug_prepend + 'NOTE:', ''.join(args) + +def error(*args): + print debug_prepend + 'ERROR:', ''.join(args) + +def fatal(*args): + print debug_prepend + 'ERROR:', ''.join(args) + sys.exit(1) + + +####################################################################### +####################################################################### +# +# SECTION: File +# +# PURPOSE: Basic file and directory tree related functions +# +####################################################################### +####################################################################### + +def mkdirhier(dir): + """Create a directory like 'mkdir -p', but does not complain if + directory already exists like os.makedirs + """ + + debug(3, "mkdirhier(%s)" % dir) + try: + os.makedirs(dir) + debug(2, "created " + dir) + except OSError, e: + if e.errno != 17: raise e + + +####################################################################### + +import stat + +def movefile(src,dest,newmtime=None,sstat=None): + """Moves a file from src to dest, preserving all permissions and + attributes; mtime will be preserved even when moving across + filesystems. Returns true on success and false on failure. Move is + atomic. + """ + + #print "movefile("+src+","+dest+","+str(newmtime)+","+str(sstat)+")" + try: + if not sstat: + sstat=os.lstat(src) + except Exception, e: + print "!!! Stating source file failed... movefile()" + print "!!!",e + return None + + destexists=1 + try: + dstat=os.lstat(dest) + except: + dstat=os.lstat(os.path.dirname(dest)) + destexists=0 + + if destexists: + if stat.S_ISLNK(dstat[stat.ST_MODE]): + try: + os.unlink(dest) + destexists=0 + except Exception, e: + pass + + if stat.S_ISLNK(sstat[stat.ST_MODE]): + try: + target=os.readlink(src) + if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): + os.unlink(dest) + os.symlink(target,dest) +# os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) + os.unlink(src) + return os.lstat(dest) + except Exception, e: + print "!!! failed to properly create symlink:" + print "!!!",dest,"->",target + print "!!!",e + return None + + renamefailed=1 + if sstat[stat.ST_DEV]==dstat[stat.ST_DEV]: + try: + ret=os.rename(src,dest) + renamefailed=0 + except Exception, e: + import errno + if e[0]!=errno.EXDEV: + # Some random error. + print "!!! Failed to move",src,"to",dest + print "!!!",e + return None + # Invalid cross-device-link 'bind' mounted or actually Cross-Device + + if renamefailed: + didcopy=0 + if stat.S_ISREG(sstat[stat.ST_MODE]): + try: # For safety copy then move it over. + shutil.copyfile(src,dest+"#new") + os.rename(dest+"#new",dest) + didcopy=1 + except Exception, e: + print '!!! copy',src,'->',dest,'failed.' + print "!!!",e + return None + else: + #we don't yet handle special, so we need to fall back to /bin/mv + a=getstatusoutput("/bin/mv -f "+"'"+src+"' '"+dest+"'") + if a[0]!=0: + print "!!! Failed to move special file:" + print "!!! '"+src+"' to '"+dest+"'" + print "!!!",a + return None # failure + try: + if didcopy: + missingos.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) + os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown + os.unlink(src) + except Exception, e: + print "!!! Failed to chown/chmod/unlink in movefile()" + print "!!!",dest + print "!!!",e + return None + + if newmtime: + os.utime(dest,(newmtime,newmtime)) + else: + os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME])) + newmtime=sstat[stat.ST_MTIME] + return newmtime + + + +####################################################################### +####################################################################### +# +# SECTION: Download +# +# PURPOSE: Download via HTTP, FTP, CVS, BITKEEPER, handling of MD5-signatures +# and mirrors +# +####################################################################### +####################################################################### + +def decodeurl(url): + """Decodes an URL into the tokens (scheme, network location, path, + user, password, parameters). + + >>> decodeurl("http://www.google.com/index.html") + ('http', 'www.google.com', '/index.html', '', '', {}) + + CVS url with username, host and cvsroot. The cvs module to check out is in the + parameters: + + >>> decodeurl("cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg") + ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}) + + Dito, but this time the username has a password part. And we also request a special tag + to check out. + + >>> decodeurl("cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;module=familiar/dist/ipkg;tag=V0-99-81") + ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}) + """ + + m = re.compile('(?P[^:]*)://((?P.+)@)?(?P[^;]+)(;(?P.*))?').match(url) + if not m: + raise MalformedUrl(url) + + type = m.group('type') + location = m.group('location') + if not location: + raise MalformedUrl(url) + user = m.group('user') + parm = m.group('parm') + m = re.compile('(?P[^/;]+)(?P/[^;]+)').match(location) + if m: + host = m.group('host') + path = m.group('path') + else: + host = "" + path = location + if user: + m = re.compile('(?P[^:]+)(:?(?P.*))').match(user) + if m: + user = m.group('user') + pswd = m.group('pswd') + else: + user = '' + pswd = '' + + p = {} + if parm: + for s in parm.split(';'): + s1,s2 = s.split('=') + p[s1] = s2 + + return (type, host, path, user, pswd, p) + +####################################################################### + +def encodeurl(decoded): + """Encodes a URL from tokens (scheme, network location, path, + user, password, parameters). + + >>> encodeurl(['http', 'www.google.com', '/index.html', '', '', {}]) + 'http://www.google.com/index.html' + + CVS with username, host and cvsroot. The cvs module to check out is in the + parameters: + + >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}]) + 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg' + + Dito, but this time the username has a password part. And we also request a special tag + to check out. + + >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}]) + 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg' + """ + + (type, host, path, user, pswd, p) = decoded + + if not type or not path: + fatal("invalid or missing parameters for url encoding") + url = '%s://' % type + if user: + url += "%s" % user + if pswd: + url += ":%s" % pswd + url += "@" + if host: + url += "%s" % host + url += "%s" % path + if p: + for parm in p.keys(): + url += ";%s=%s" % (parm, p[parm]) + + return url + +####################################################################### + +def which(path, item, direction = 0): + """Useful function for locating a file in a PATH""" + found = "" + for p in (path or "").split(':'): + if os.path.exists(os.path.join(p, item)): + found = os.path.join(p, item) + if direction == 0: + break + return found + +####################################################################### + + + + +####################################################################### +####################################################################### +# +# SECTION: Dependency +# +# PURPOSE: Compare build & run dependencies +# +####################################################################### +####################################################################### + +def tokenize(mystring): + """Breaks a string like 'foo? (bar) oni? (blah (blah))' into (possibly embedded) lists: + + >>> tokenize("x") + ['x'] + >>> tokenize("x y") + ['x', 'y'] + >>> tokenize("(x y)") + [['x', 'y']] + >>> tokenize("(x y) b c") + [['x', 'y'], 'b', 'c'] + >>> tokenize("foo? (bar) oni? (blah (blah))") + ['foo?', ['bar'], 'oni?', ['blah', ['blah']]] + >>> tokenize("sys-apps/linux-headers nls? (sys-devel/gettext)") + ['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']] + """ + + newtokens = [] + curlist = newtokens + prevlists = [] + level = 0 + accum = "" + for x in mystring: + if x=="(": + if accum: + curlist.append(accum) + accum="" + prevlists.append(curlist) + curlist=[] + level=level+1 + elif x==")": + if accum: + curlist.append(accum) + accum="" + if level==0: + print "!!! tokenizer: Unmatched left parenthesis in:\n'"+mystring+"'" + return None + newlist=curlist + curlist=prevlists.pop() + curlist.append(newlist) + level=level-1 + elif x in whitespace: + if accum: + curlist.append(accum) + accum="" + else: + accum=accum+x + if accum: + curlist.append(accum) + if (level!=0): + print "!!! tokenizer: Exiting with unterminated parenthesis in:\n'"+mystring+"'" + return None + return newtokens + + +####################################################################### + +def evaluate(tokens,mydefines,allon=0): + """Removes tokens based on whether conditional definitions exist or not. + Recognizes ! + + >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {}) + ['sys-apps/linux-headers'] + + Negate the flag: + + >>> evaluate(['sys-apps/linux-headers', '!nls?', ['sys-devel/gettext']], {}) + ['sys-apps/linux-headers', ['sys-devel/gettext']] + + Define 'nls': + + >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {"nls":1}) + ['sys-apps/linux-headers', ['sys-devel/gettext']] + + Turn allon on: + + >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {}, True) + ['sys-apps/linux-headers', ['sys-devel/gettext']] + """ + + if tokens == None: + return None + mytokens = tokens + [] # this copies the list + pos = 0 + while pos < len(mytokens): + if type(mytokens[pos]) == types.ListType: + evaluate(mytokens[pos], mydefines) + if not len(mytokens[pos]): + del mytokens[pos] + continue + elif mytokens[pos][-1] == "?": + cur = mytokens[pos][:-1] + del mytokens[pos] + if allon: + if cur[0] == "!": + del mytokens[pos] + else: + if cur[0] == "!": + if (cur[1:] in mydefines) and (pos < len(mytokens)): + del mytokens[pos] + continue + elif (cur not in mydefines) and (pos < len(mytokens)): + del mytokens[pos] + continue + pos = pos + 1 + return mytokens + + +####################################################################### + +def flatten(mytokens): + """Converts nested arrays into a flat arrays: + + >>> flatten([1,[2,3]]) + [1, 2, 3] + >>> flatten(['sys-apps/linux-headers', ['sys-devel/gettext']]) + ['sys-apps/linux-headers', 'sys-devel/gettext'] + """ + + newlist=[] + for x in mytokens: + if type(x)==types.ListType: + newlist.extend(flatten(x)) + else: + newlist.append(x) + return newlist + + +####################################################################### + +_package_weights_ = {"pre":-2,"p":0,"alpha":-4,"beta":-3,"rc":-1} # dicts are unordered +_package_ends_ = ["pre", "p", "alpha", "beta", "rc", "cvs", "bk", "HEAD" ] # so we need ordered list + +def relparse(myver): + """Parses the last elements of a version number into a triplet, that can + later be compared: + + >>> relparse('1.2_pre3') + [1.2, -2, 3.0] + >>> relparse('1.2b') + [1.2, 98, 0] + >>> relparse('1.2') + [1.2, 0, 0] + """ + + number = 0 + p1 = 0 + p2 = 0 + mynewver = myver.split('_') + if len(mynewver)==2: + # an _package_weights_ + number = float(mynewver[0]) + match = 0 + for x in _package_ends_: + elen = len(x) + if mynewver[1][:elen] == x: + match = 1 + p1 = _package_weights_[x] + try: + p2 = float(mynewver[1][elen:]) + except: + p2 = 0 + break + if not match: + # normal number or number with letter at end + divider = len(myver)-1 + if myver[divider:] not in "1234567890": + # letter at end + p1 = ord(myver[divider:]) + number = float(myver[0:divider]) + else: + number = float(myver) + else: + # normal number or number with letter at end + divider = len(myver)-1 + if myver[divider:] not in "1234567890": + #letter at end + p1 = ord(myver[divider:]) + number = float(myver[0:divider]) + else: + number = float(myver) + return [number,p1,p2] + + +####################################################################### + +__ververify_cache__ = {} + +def ververify(myorigval,silent=1): + """Returns 1 if given a valid version string, els 0. Valid versions are in the format + + ....[a-z,_{_package_weights_}[vy]] + + >>> ververify('2.4.20') + 1 + >>> ververify('2.4..20') # two dots + 0 + >>> ververify('2.x.20') # 'x' is not numeric + 0 + >>> ververify('2.4.20a') + 1 + >>> ververify('2.4.20cvs') # only one trailing letter + 0 + >>> ververify('1a') + 1 + >>> ververify('test_a') # no version at all + 0 + >>> ververify('2.4.20_beta1') + 1 + >>> ververify('2.4.20_beta') + 1 + >>> ververify('2.4.20_wrongext') # _wrongext is no valid trailer + 0 + """ + + # Lookup the cache first + try: + return __ververify_cache__[myorigval] + except KeyError: + pass + + if len(myorigval) == 0: + if not silent: + error("package version is empty") + __ververify_cache__[myorigval] = 0 + return 0 + myval = myorigval.split('.') + if len(myval)==0: + if not silent: + error("package name has empty version string") + __ververify_cache__[myorigval] = 0 + return 0 + # all but the last version must be a numeric + for x in myval[:-1]: + if not len(x): + if not silent: + error("package version has two points in a row") + __ververify_cache__[myorigval] = 0 + return 0 + try: + foo = int(x) + except: + if not silent: + error("package version contains non-numeric '"+x+"'") + __ververify_cache__[myorigval] = 0 + return 0 + if not len(myval[-1]): + if not silent: + error("package version has trailing dot") + __ververify_cache__[myorigval] = 0 + return 0 + try: + foo = int(myval[-1]) + __ververify_cache__[myorigval] = 1 + return 1 + except: + pass + + # ok, our last component is not a plain number or blank, let's continue + if myval[-1][-1] in lowercase: + try: + foo = int(myval[-1][:-1]) + return 1 + __ververify_cache__[myorigval] = 1 + # 1a, 2.0b, etc. + except: + pass + # ok, maybe we have a 1_alpha or 1_beta2; let's see + ep=string.split(myval[-1],"_") + if len(ep)!= 2: + if not silent: + error("package version has more than one letter at then end") + __ververify_cache__[myorigval] = 0 + return 0 + try: + foo = string.atoi(ep[0]) + except: + # this needs to be numeric, i.e. the "1" in "1_alpha" + if not silent: + error("package version must have numeric part before the '_'") + __ververify_cache__[myorigval] = 0 + return 0 + + for mye in _package_ends_: + if ep[1][0:len(mye)] == mye: + if len(mye) == len(ep[1]): + # no trailing numeric is ok + __ververify_cache__[myorigval] = 1 + return 1 + else: + try: + foo = string.atoi(ep[1][len(mye):]) + __ververify_cache__[myorigval] = 1 + return 1 + except: + # if no _package_weights_ work, *then* we return 0 + pass + if not silent: + error("package version extension after '_' is invalid") + __ververify_cache__[myorigval] = 0 + return 0 + + +def isjustname(mypkg): + myparts = string.split(mypkg,'-') + for x in myparts: + if ververify(x): + return 0 + return 1 + + +_isspecific_cache_={} + +def isspecific(mypkg): + "now supports packages with no category" + try: + return __isspecific_cache__[mypkg] + except: + pass + + mysplit = string.split(mypkg,"/") + if not isjustname(mysplit[-1]): + __isspecific_cache__[mypkg] = 1 + return 1 + __isspecific_cache__[mypkg] = 0 + return 0 + + +####################################################################### + +__pkgsplit_cache__={} + +def pkgsplit(mypkg, silent=1): + + """This function can be used as a package verification function. If + it is a valid name, pkgsplit will return a list containing: + [pkgname, pkgversion(norev), pkgrev ]. + + >>> pkgsplit('') + >>> pkgsplit('x') + >>> pkgsplit('x-') + >>> pkgsplit('-1') + >>> pkgsplit('glibc-1.2-8.9-r7') + >>> pkgsplit('glibc-2.2.5-r7') + ['glibc', '2.2.5', 'r7'] + >>> pkgsplit('foo-1.2-1') + >>> pkgsplit('Mesa-3.0') + ['Mesa', '3.0', 'r0'] + """ + + try: + return __pkgsplit_cache__[mypkg] + except KeyError: + pass + + myparts = string.split(mypkg,'-') + if len(myparts) < 2: + if not silent: + error("package name without name or version part") + __pkgsplit_cache__[mypkg] = None + return None + for x in myparts: + if len(x) == 0: + if not silent: + error("package name with empty name or version part") + __pkgsplit_cache__[mypkg] = None + return None + # verify rev + revok = 0 + myrev = myparts[-1] + ververify(myrev, silent) + if len(myrev) and myrev[0] == "r": + try: + string.atoi(myrev[1:]) + revok = 1 + except: + pass + if revok: + if ververify(myparts[-2]): + if len(myparts) == 2: + __pkgsplit_cache__[mypkg] = None + return None + else: + for x in myparts[:-2]: + if ververify(x): + __pkgsplit_cache__[mypkg]=None + return None + # names can't have versiony looking parts + myval=[string.join(myparts[:-2],"-"),myparts[-2],myparts[-1]] + __pkgsplit_cache__[mypkg]=myval + return myval + else: + __pkgsplit_cache__[mypkg] = None + return None + + elif ververify(myparts[-1],silent): + if len(myparts)==1: + if not silent: + print "!!! Name error in",mypkg+": missing name part." + __pkgsplit_cache__[mypkg]=None + return None + else: + for x in myparts[:-1]: + if ververify(x): + if not silent: error("package name has multiple version parts") + __pkgsplit_cache__[mypkg] = None + return None + myval = [string.join(myparts[:-1],"-"), myparts[-1],"r0"] + __pkgsplit_cache__[mypkg] = myval + return myval + else: + __pkgsplit_cache__[mypkg] = None + return None + + +####################################################################### + +__catpkgsplit_cache__ = {} + +def catpkgsplit(mydata,silent=1): + """returns [cat, pkgname, version, rev ] + + >>> catpkgsplit('sys-libs/glibc-1.2-r7') + ['sys-libs', 'glibc', '1.2', 'r7'] + >>> catpkgsplit('glibc-1.2-r7') + [None, 'glibc', '1.2', 'r7'] + """ + + try: + return __catpkgsplit_cache__[mydata] + except KeyError: + pass + + cat = os.path.basename(os.path.dirname(mydata)) + mydata = os.path.join(cat, os.path.basename(mydata)) + if mydata[-3:] == '.bb': + mydata = mydata[:-3] + + mysplit = mydata.split("/") + p_split = None + splitlen = len(mysplit) + if splitlen == 1: + retval = [None] + p_split = pkgsplit(mydata,silent) + else: + retval = [mysplit[splitlen - 2]] + p_split = pkgsplit(mysplit[splitlen - 1],silent) + if not p_split: + __catpkgsplit_cache__[mydata] = None + return None + retval.extend(p_split) + __catpkgsplit_cache__[mydata] = retval + return retval + + +####################################################################### + +__vercmp_cache__ = {} + +def vercmp(val1,val2): + """This takes two version strings and returns an integer to tell you whether + the versions are the same, val1>val2 or val2>val1. + + >>> vercmp('1', '2') + -1.0 + >>> vercmp('2', '1') + 1.0 + >>> vercmp('1', '1.0') + 0 + >>> vercmp('1', '1.1') + -1.0 + >>> vercmp('1.1', '1_p2') + 1.0 + """ + + # quick short-circuit + if val1 == val2: + return 0 + valkey = val1+" "+val2 + + # cache lookup + try: + return __vercmp_cache__[valkey] + try: + return - __vercmp_cache__[val2+" "+val1] + except KeyError: + pass + except KeyError: + pass + + # consider 1_p2 vc 1.1 + # after expansion will become (1_p2,0) vc (1,1) + # then 1_p2 is compared with 1 before 0 is compared with 1 + # to solve the bug we need to convert it to (1,0_p2) + # by splitting _prepart part and adding it back _after_expansion + + val1_prepart = val2_prepart = '' + if val1.count('_'): + val1, val1_prepart = val1.split('_', 1) + if val2.count('_'): + val2, val2_prepart = val2.split('_', 1) + + # replace '-' by '.' + # FIXME: Is it needed? can val1/2 contain '-'? + + val1 = string.split(val1,'-') + if len(val1) == 2: + val1[0] = val1[0] +"."+ val1[1] + val2 = string.split(val2,'-') + if len(val2) == 2: + val2[0] = val2[0] +"."+ val2[1] + + val1 = string.split(val1[0],'.') + val2 = string.split(val2[0],'.') + + # add back decimal point so that .03 does not become "3" ! + for x in range(1,len(val1)): + if val1[x][0] == '0' : + val1[x] = '.' + val1[x] + for x in range(1,len(val2)): + if val2[x][0] == '0' : + val2[x] = '.' + val2[x] + + # extend varion numbers + if len(val2) < len(val1): + val2.extend(["0"]*(len(val1)-len(val2))) + elif len(val1) < len(val2): + val1.extend(["0"]*(len(val2)-len(val1))) + + # add back _prepart tails + if val1_prepart: + val1[-1] += '_' + val1_prepart + if val2_prepart: + val2[-1] += '_' + val2_prepart + # The above code will extend version numbers out so they + # have the same number of digits. + for x in range(0,len(val1)): + cmp1 = relparse(val1[x]) + cmp2 = relparse(val2[x]) + for y in range(0,3): + myret = cmp1[y] - cmp2[y] + if myret != 0: + __vercmp_cache__[valkey] = myret + return myret + __vercmp_cache__[valkey] = 0 + return 0 + + +####################################################################### + +def pkgcmp(pkg1,pkg2): + """ Compares two packages, which should have been split via + pkgsplit(). if the return value val is less than zero, then pkg2 is + newer than pkg1, zero if equal and positive if older. + + >>> pkgcmp(['glibc', '2.2.5', 'r7'], ['glibc', '2.2.5', 'r7']) + 0 + >>> pkgcmp(['glibc', '2.2.5', 'r4'], ['glibc', '2.2.5', 'r7']) + -1 + >>> pkgcmp(['glibc', '2.2.5', 'r7'], ['glibc', '2.2.5', 'r2']) + 1 + """ + + mycmp = vercmp(pkg1[1],pkg2[1]) + if mycmp > 0: + return 1 + if mycmp < 0: + return -1 + r1=string.atoi(pkg1[2][1:]) + r2=string.atoi(pkg2[2][1:]) + if r1 > r2: + return 1 + if r2 > r1: + return -1 + return 0 + + +####################################################################### + +def dep_parenreduce(mysplit, mypos=0): + """Accepts a list of strings, and converts '(' and ')' surrounded items to sub-lists: + + >>> dep_parenreduce(['']) + [''] + >>> dep_parenreduce(['1', '2', '3']) + ['1', '2', '3'] + >>> dep_parenreduce(['1', '(', '2', '3', ')', '4']) + ['1', ['2', '3'], '4'] + """ + + while mypos < len(mysplit): + if mysplit[mypos] == "(": + firstpos = mypos + mypos = mypos + 1 + while mypos < len(mysplit): + if mysplit[mypos] == ")": + mysplit[firstpos:mypos+1] = [mysplit[firstpos+1:mypos]] + mypos = firstpos + break + elif mysplit[mypos] == "(": + # recurse + mysplit = dep_parenreduce(mysplit,mypos) + mypos = mypos + 1 + mypos = mypos + 1 + return mysplit + + +def dep_opconvert(mysplit, myuse): + "Does dependency operator conversion" + + mypos = 0 + newsplit = [] + while mypos < len(mysplit): + if type(mysplit[mypos]) == types.ListType: + newsplit.append(dep_opconvert(mysplit[mypos],myuse)) + mypos += 1 + elif mysplit[mypos] == ")": + # mismatched paren, error + return None + elif mysplit[mypos]=="||": + if ((mypos+1)>=len(mysplit)) or (type(mysplit[mypos+1])!=types.ListType): + # || must be followed by paren'd list + return None + try: + mynew = dep_opconvert(mysplit[mypos+1],myuse) + except Exception, e: + error("unable to satisfy OR dependancy: " + string.join(mysplit," || ")) + raise e + mynew[0:0] = ["||"] + newsplit.append(mynew) + mypos += 2 + elif mysplit[mypos][-1] == "?": + # use clause, i.e "gnome? ( foo bar )" + # this is a quick and dirty hack so that repoman can enable all USE vars: + if (len(myuse) == 1) and (myuse[0] == "*"): + # enable it even if it's ! (for repoman) but kill it if it's + # an arch variable that isn't for this arch. XXX Sparc64? + if (mysplit[mypos][:-1] not in settings.usemask) or \ + (mysplit[mypos][:-1]==settings["ARCH"]): + enabled=1 + else: + enabled=0 + else: + if mysplit[mypos][0] == "!": + myusevar = mysplit[mypos][1:-1] + enabled = not myusevar in myuse + #if myusevar in myuse: + # enabled = 0 + #else: + # enabled = 1 + else: + myusevar=mysplit[mypos][:-1] + enabled = myusevar in myuse + #if myusevar in myuse: + # enabled=1 + #else: + # enabled=0 + if (mypos +2 < len(mysplit)) and (mysplit[mypos+2] == ":"): + # colon mode + if enabled: + # choose the first option + if type(mysplit[mypos+1]) == types.ListType: + newsplit.append(dep_opconvert(mysplit[mypos+1],myuse)) + else: + newsplit.append(mysplit[mypos+1]) + else: + # choose the alternate option + if type(mysplit[mypos+1]) == types.ListType: + newsplit.append(dep_opconvert(mysplit[mypos+3],myuse)) + else: + newsplit.append(mysplit[mypos+3]) + mypos += 4 + else: + # normal use mode + if enabled: + if type(mysplit[mypos+1]) == types.ListType: + newsplit.append(dep_opconvert(mysplit[mypos+1],myuse)) + else: + newsplit.append(mysplit[mypos+1]) + # otherwise, continue + mypos += 2 + else: + # normal item + newsplit.append(mysplit[mypos]) + mypos += 1 + return newsplit + +class digraph: + """beautiful directed graph object""" + + def __init__(self): + self.dict={} + #okeys = keys, in order they were added (to optimize firstzero() ordering) + self.okeys=[] + self.__callback_cache=[] + + def __str__(self): + str = "" + for key in self.okeys: + str += "%s:\t%s\n" % (key, self.dict[key][1]) + return str + + def addnode(self,mykey,myparent): + if not mykey in self.dict: + self.okeys.append(mykey) + if myparent==None: + self.dict[mykey]=[0,[]] + else: + self.dict[mykey]=[0,[myparent]] + self.dict[myparent][0]=self.dict[myparent][0]+1 + return + if myparent and (not myparent in self.dict[mykey][1]): + self.dict[mykey][1].append(myparent) + self.dict[myparent][0]=self.dict[myparent][0]+1 + + def delnode(self,mykey, ref = 1): + """Delete a node + + If ref is 1, remove references to this node from other nodes. + If ref is 2, remove nodes that reference this node.""" + if not mykey in self.dict: + return + for x in self.dict[mykey][1]: + self.dict[x][0]=self.dict[x][0]-1 + del self.dict[mykey] + while 1: + try: + self.okeys.remove(mykey) + except ValueError: + break + if ref: + __kill = [] + for k in self.okeys: + if mykey in self.dict[k][1]: + if ref == 1 or ref == 2: + self.dict[k][1].remove(mykey) + if ref == 2: + __kill.append(k) + for l in __kill: + self.delnode(l, ref) + + def allnodes(self): + "returns all nodes in the dictionary" + return self.dict.keys() + + def firstzero(self): + "returns first node with zero references, or NULL if no such node exists" + for x in self.okeys: + if self.dict[x][0]==0: + return x + return None + + def firstnonzero(self): + "returns first node with nonzero references, or NULL if no such node exists" + for x in self.okeys: + if self.dict[x][0]!=0: + return x + return None + + + def allzeros(self): + "returns all nodes with zero references, or NULL if no such node exists" + zerolist = [] + for x in self.dict.keys(): + if self.dict[x][0]==0: + zerolist.append(x) + return zerolist + + def hasallzeros(self): + "returns 0/1, Are all nodes zeros? 1 : 0" + zerolist = [] + for x in self.dict.keys(): + if self.dict[x][0]!=0: + return 0 + return 1 + + def empty(self): + if len(self.dict)==0: + return 1 + return 0 + + def hasnode(self,mynode): + return mynode in self.dict + + def getparents(self, item): + if not self.hasnode(item): + return [] + return self.dict[item][1] + + def getchildren(self, item): + if not self.hasnode(item): + return [] + children = [i for i in self.okeys if item in self.getparents(i)] + return children + + def walkdown(self, item, callback, debug = None, usecache = False): + if not self.hasnode(item): + return 0 + + if usecache: + if self.__callback_cache.count(item): + if debug: + print "hit cache for item: %s" % item + return 1 + + parents = self.getparents(item) + children = self.getchildren(item) + for p in parents: + if p in children: +# print "%s is both parent and child of %s" % (p, item) + if usecache: + self.__callback_cache.append(p) + ret = callback(self, p) + if ret == 0: + return 0 + continue + if item == p: + print "eek, i'm my own parent!" + return 0 + if debug: + print "item: %s, p: %s" % (item, p) + ret = self.walkdown(p, callback, debug, usecache) + if ret == 0: + return 0 + if usecache: + self.__callback_cache.append(item) + return callback(self, item) + + def walkup(self, item, callback): + if not self.hasnode(item): + return 0 + + parents = self.getparents(item) + children = self.getchildren(item) + for c in children: + if c in parents: + ret = callback(self, item) + if ret == 0: + return 0 + continue + if item == c: + print "eek, i'm my own child!" + return 0 + ret = self.walkup(c, callback) + if ret == 0: + return 0 + return callback(self, item) + + def copy(self): + mygraph=digraph() + for x in self.dict.keys(): + mygraph.dict[x]=self.dict[x][:] + mygraph.okeys=self.okeys[:] + return mygraph + +####################################################################### +####################################################################### +# +# SECTION: Config +# +# PURPOSE: Reading and handling of system/target-specific/local configuration +# reading of package configuration +# +####################################################################### +####################################################################### + +def reader(cfgfile, feeder): + """Generic configuration file reader that opens a file, reads the lines, + handles continuation lines, comments, empty lines and feed all read lines + into the function feeder(lineno, line). + """ + + f = open(cfgfile,'r') + lineno = 0 + while 1: + lineno = lineno + 1 + s = f.readline() + if not s: break + w = s.strip() + if not w: continue # skip empty lines + s = s.rstrip() + if s[0] == '#': continue # skip comments + while s[-1] == '\\': + s2 = f.readline()[:-1].strip() + s = s[:-1] + s2 + feeder(lineno, s) + +if __name__ == "__main__": + import doctest, bb + doctest.testmod(bb) diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py new file mode 100644 index 0000000..599b45d --- /dev/null +++ b/bitbake/lib/bb/build.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Build' implementation + +Core code for function execution and task handling in the +BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson + +Based on Gentoo's portage.py. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with + +Based on functions from the base bb module, Copyright 2003 Holger Schurig +""" + +from bb import debug, data, fetch, fatal, error, note, event, mkdirhier +import bb, os + +# data holds flags and function name for a given task +_task_data = data.init() + +# graph represents task interdependencies +_task_graph = bb.digraph() + +# stack represents execution order, excepting dependencies +_task_stack = [] + +# events +class FuncFailed(Exception): + """Executed function failed""" + +class EventException(Exception): + """Exception which is associated with an Event.""" + + def __init__(self, msg, event): + self.args = msg, event + +class TaskBase(event.Event): + """Base class for task events""" + + def __init__(self, t, d ): + self._task = t + event.Event.__init__(self, d) + + def getTask(self): + return self._task + + def setTask(self, task): + self._task = task + + task = property(getTask, setTask, None, "task property") + +class TaskStarted(TaskBase): + """Task execution started""" + +class TaskSucceeded(TaskBase): + """Task execution completed""" + +class TaskFailed(TaskBase): + """Task execution failed""" + +class InvalidTask(TaskBase): + """Invalid Task""" + +# functions + +def init(data): + global _task_data, _task_graph, _task_stack + _task_data = data.init() + _task_graph = bb.digraph() + _task_stack = [] + + +def exec_func(func, d, dirs = None): + """Execute an BB 'function'""" + + body = data.getVar(func, d) + if not body: + return + + if not dirs: + dirs = (data.getVarFlag(func, 'dirs', d) or "").split() + for adir in dirs: + adir = data.expand(adir, d) + mkdirhier(adir) + + if len(dirs) > 0: + adir = dirs[-1] + else: + adir = data.getVar('B', d, 1) + + adir = data.expand(adir, d) + + try: + prevdir = os.getcwd() + except OSError: + prevdir = data.expand('${TOPDIR}', d) + if adir and os.access(adir, os.F_OK): + os.chdir(adir) + + if data.getVarFlag(func, "python", d): + exec_func_python(func, d) + else: + exec_func_shell(func, d) + + if os.path.exists(prevdir): + os.chdir(prevdir) + +def exec_func_python(func, d): + """Execute a python BB 'function'""" + import re, os + + tmp = "def " + func + "():\n%s" % data.getVar(func, d) + comp = compile(tmp + '\n' + func + '()', bb.data.getVar('FILE', d, 1) + ':' + func, "exec") + prevdir = os.getcwd() + g = {} # globals + g['bb'] = bb + g['os'] = os + g['d'] = d + exec comp in g + if os.path.exists(prevdir): + os.chdir(prevdir) + +def exec_func_shell(func, d): + """Execute a shell BB 'function' Returns true if execution was successful. + + For this, it creates a bash shell script in the tmp dectory, writes the local + data into it and finally executes. The output of the shell will end in a log file and stdout. + + Note on directory behavior. The 'dirs' varflag should contain a list + of the directories you need created prior to execution. The last + item in the list is where we will chdir/cd to. + """ + import sys + + deps = data.getVarFlag(func, 'deps', d) + check = data.getVarFlag(func, 'check', d) + if check in globals(): + if globals()[check](func, deps): + return + + global logfile + t = data.getVar('T', d, 1) + if not t: + return 0 + mkdirhier(t) + logfile = "%s/log.%s.%s" % (t, func, str(os.getpid())) + runfile = "%s/run.%s.%s" % (t, func, str(os.getpid())) + + f = open(runfile, "w") + f.write("#!/bin/sh -e\n") + if bb.debug_level > 0: f.write("set -x\n") + data.emit_env(f, d) + + f.write("cd %s\n" % os.getcwd()) + if func: f.write("%s\n" % func) + f.close() + os.chmod(runfile, 0775) + if not func: + error("Function not specified") + raise FuncFailed() + + # open logs + si = file('/dev/null', 'r') + try: + if bb.debug_level > 0: + so = os.popen("tee \"%s\"" % logfile, "w") + else: + so = file(logfile, 'w') + except OSError, e: + bb.error("opening log file: %s" % e) + pass + + se = so + + # dup the existing fds so we dont lose them + osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] + oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] + ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] + + # replace those fds with our own + os.dup2(si.fileno(), osi[1]) + os.dup2(so.fileno(), oso[1]) + os.dup2(se.fileno(), ose[1]) + + # execute function + prevdir = os.getcwd() + if data.getVarFlag(func, "fakeroot", d): + maybe_fakeroot = "PATH=\"%s\" fakeroot " % bb.data.getVar("PATH", d, 1) + else: + maybe_fakeroot = '' + ret = os.system('%ssh -e %s' % (maybe_fakeroot, runfile)) + os.chdir(prevdir) + + # restore the backups + os.dup2(osi[0], osi[1]) + os.dup2(oso[0], oso[1]) + os.dup2(ose[0], ose[1]) + + # close our logs + si.close() + so.close() + se.close() + + # close the backup fds + os.close(osi[0]) + os.close(oso[0]) + os.close(ose[0]) + + if ret==0: + if bb.debug_level > 0: + os.remove(runfile) +# os.remove(logfile) + return + else: + error("function %s failed" % func) + if data.getVar("BBINCLUDELOGS", d): + error("log data follows (%s)" % logfile) + f = open(logfile, "r") + while True: + l = f.readline() + if l == '': + break + l = l.rstrip() + print '| %s' % l + f.close() + else: + error("see log in %s" % logfile) + raise FuncFailed( logfile ) + + +def exec_task(task, d): + """Execute an BB 'task' + + The primary difference between executing a task versus executing + a function is that a task exists in the task digraph, and therefore + has dependencies amongst other tasks.""" + + # check if the task is in the graph.. + task_graph = data.getVar('_task_graph', d) + if not task_graph: + task_graph = bb.digraph() + data.setVar('_task_graph', task_graph, d) + task_cache = data.getVar('_task_cache', d) + if not task_cache: + task_cache = [] + data.setVar('_task_cache', task_cache, d) + if not task_graph.hasnode(task): + raise EventException("Missing node in task graph", InvalidTask(task, d)) + + # check whether this task needs executing.. + if not data.getVarFlag(task, 'force', d): + if stamp_is_current(task, d): + return 1 + + # follow digraph path up, then execute our way back down + def execute(graph, item): + if data.getVarFlag(item, 'task', d): + if item in task_cache: + return 1 + + if task != item: + # deeper than toplevel, exec w/ deps + exec_task(item, d) + return 1 + + try: + debug(1, "Executing task %s" % item) + old_overrides = data.getVar('OVERRIDES', d, 0) + localdata = data.createCopy(d) + data.setVar('OVERRIDES', 'task_%s:%s' % (item, old_overrides), localdata) + data.update_data(localdata) + event.fire(TaskStarted(item, localdata)) + exec_func(item, localdata) + event.fire(TaskSucceeded(item, localdata)) + task_cache.append(item) + data.setVar('_task_cache', task_cache, d) + except FuncFailed, reason: + note( "Task failed: %s" % reason ) + failedevent = TaskFailed(item, d) + event.fire(failedevent) + raise EventException("Function failed in task: %s" % reason, failedevent) + + # execute + task_graph.walkdown(task, execute) + + # make stamp, or cause event and raise exception + if not data.getVarFlag(task, 'nostamp', d): + mkstamp(task, d) + + +def stamp_is_current(task, d, checkdeps = 1): + """Check status of a given task's stamp. returns 0 if it is not current and needs updating.""" + task_graph = data.getVar('_task_graph', d) + if not task_graph: + task_graph = bb.digraph() + data.setVar('_task_graph', task_graph, d) + stamp = data.getVar('STAMP', d) + if not stamp: + return 0 + stampfile = "%s.%s" % (data.expand(stamp, d), task) + if not os.access(stampfile, os.F_OK): + return 0 + + if checkdeps == 0: + return 1 + + import stat + tasktime = os.stat(stampfile)[stat.ST_MTIME] + + _deps = [] + def checkStamp(graph, task): + # check for existance + if data.getVarFlag(task, 'nostamp', d): + return 1 + + if not stamp_is_current(task, d, 0): + return 0 + + depfile = "%s.%s" % (data.expand(stamp, d), task) + deptime = os.stat(depfile)[stat.ST_MTIME] + if deptime > tasktime: + return 0 + return 1 + + return task_graph.walkdown(task, checkStamp) + + +def md5_is_current(task): + """Check if a md5 file for a given task is current""" + + +def mkstamp(task, d): + """Creates/updates a stamp for a given task""" + stamp = data.getVar('STAMP', d) + if not stamp: + return + stamp = "%s.%s" % (data.expand(stamp, d), task) + mkdirhier(os.path.dirname(stamp)) + open(stamp, "w+") + + +def add_task(task, deps, d): + task_graph = data.getVar('_task_graph', d) + if not task_graph: + task_graph = bb.digraph() + data.setVarFlag(task, 'task', 1, d) + task_graph.addnode(task, None) + for dep in deps: + if not task_graph.hasnode(dep): + task_graph.addnode(dep, None) + task_graph.addnode(task, dep) + # don't assume holding a reference + data.setVar('_task_graph', task_graph, d) + + +def remove_task(task, kill, d): + """Remove an BB 'task'. + + If kill is 1, also remove tasks that depend on this task.""" + + task_graph = data.getVar('_task_graph', d) + if not task_graph: + task_graph = bb.digraph() + if not task_graph.hasnode(task): + return + + data.delVarFlag(task, 'task', d) + ref = 1 + if kill == 1: + ref = 2 + task_graph.delnode(task, ref) + data.setVar('_task_graph', task_graph, d) + +def task_exists(task, d): + task_graph = data.getVar('_task_graph', d) + if not task_graph: + task_graph = bb.digraph() + data.setVar('_task_graph', task_graph, d) + return task_graph.hasnode(task) + +def get_task_data(): + return _task_data diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py new file mode 100644 index 0000000..b7d707a --- /dev/null +++ b/bitbake/lib/bb/data.py @@ -0,0 +1,580 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Data' implementations + +Functions for interacting with the data structure used by the +BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson +Copyright (C) 2005 Holger Hans Peter Freyther + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +Based on functions from the base bb module, Copyright 2003 Holger Schurig +""" + +import sys, os, re, time, types +if sys.argv[0][-5:] == "pydoc": + path = os.path.dirname(os.path.dirname(sys.argv[1])) +else: + path = os.path.dirname(os.path.dirname(sys.argv[0])) +sys.path.append(path) + +from bb import note, debug, data_smart + +_dict_type = data_smart.DataSmart +_dict_p_type = data_smart.DataSmartPackage + +class DataDictFull(dict): + """ + This implements our Package Data Storage Interface. + setDirty is a no op as all items are held in memory + """ + def setDirty(self, bbfile, data): + """ + No-Op we assume data was manipulated as some sort of + reference + """ + if not bbfile in self: + raise Exception("File %s was not in dictionary before" % bbfile) + + self[bbfile] = data + +class DataDictCache: + """ + Databacked Dictionary implementation + """ + def __init__(self, cache_dir, config): + self.cache_dir = cache_dir + self.files = [] + self.dirty = {} + self.config = config + + def has_key(self,key): + return key in self.files + + def keys(self): + return self.files + + def __setitem__(self, key, data): + """ + Add the key to the list of known files and + place the data in the cache? + """ + if key in self.files: + return + + self.files.append(key) + + def __getitem__(self, key): + if not key in self.files: + return None + + # if it was dirty we will + if key in self.dirty: + return self.dirty[key] + + # not cached yet + return _dict_p_type(self.cache_dir, key,False,self.config) + + def setDirty(self, bbfile, data): + """ + Only already added items can be declared dirty!!! + """ + + if not bbfile in self.files: + raise Exception("File %s was not in dictionary before" % bbfile) + + self.dirty[bbfile] = data + + + +def init(): + return _dict_type() + +def init_db(cache,name,clean,parent = None): + return _dict_p_type(cache,name,clean,parent) + +def init_db_mtime(cache,cache_bbfile): + return _dict_p_type.mtime(cache,cache_bbfile) + +def pkgdata(use_cache, cache, config = None): + """ + Return some sort of dictionary to lookup parsed dictionaires + """ + if use_cache: + return DataDictCache(cache, config) + return DataDictFull() + +def createCopy(source): + """Link the source set to the destination + If one does not find the value in the destination set, + search will go on to the source set to get the value. + Value from source are copy-on-write. i.e. any try to + modify one of them will end up putting the modified value + in the destination set. + """ + return source.createCopy() + +def initVar(var, d): + """Non-destructive var init for data structure""" + d.initVar(var) + + +def setVar(var, value, d): + """Set a variable to a given value + + Example: + >>> d = init() + >>> setVar('TEST', 'testcontents', d) + >>> print getVar('TEST', d) + testcontents + """ + d.setVar(var,value) + + +def getVar(var, d, exp = 0): + """Gets the value of a variable + + Example: + >>> d = init() + >>> setVar('TEST', 'testcontents', d) + >>> print getVar('TEST', d) + testcontents + """ + return d.getVar(var,exp) + +def delVar(var, d): + """Removes a variable from the data set + + Example: + >>> d = init() + >>> setVar('TEST', 'testcontents', d) + >>> print getVar('TEST', d) + testcontents + >>> delVar('TEST', d) + >>> print getVar('TEST', d) + None + """ + d.delVar(var) + +def setVarFlag(var, flag, flagvalue, d): + """Set a flag for a given variable to a given value + + Example: + >>> d = init() + >>> setVarFlag('TEST', 'python', 1, d) + >>> print getVarFlag('TEST', 'python', d) + 1 + """ + d.setVarFlag(var,flag,flagvalue) + +def getVarFlag(var, flag, d): + """Gets given flag from given var + + Example: + >>> d = init() + >>> setVarFlag('TEST', 'python', 1, d) + >>> print getVarFlag('TEST', 'python', d) + 1 + """ + return d.getVarFlag(var,flag) + +def delVarFlag(var, flag, d): + """Removes a given flag from the variable's flags + + Example: + >>> d = init() + >>> setVarFlag('TEST', 'testflag', 1, d) + >>> print getVarFlag('TEST', 'testflag', d) + 1 + >>> delVarFlag('TEST', 'testflag', d) + >>> print getVarFlag('TEST', 'testflag', d) + None + + """ + d.delVarFlag(var,flag) + +def setVarFlags(var, flags, d): + """Set the flags for a given variable + + Example: + >>> d = init() + >>> myflags = {} + >>> myflags['test'] = 'blah' + >>> setVarFlags('TEST', myflags, d) + >>> print getVarFlag('TEST', 'test', d) + blah + """ + d.setVarFlags(var,flags) + +def getVarFlags(var, d): + """Gets a variable's flags + + Example: + >>> d = init() + >>> setVarFlag('TEST', 'test', 'blah', d) + >>> print getVarFlags('TEST', d)['test'] + blah + """ + return d.getVarFlags(var) + +def delVarFlags(var, d): + """Removes a variable's flags + + Example: + >>> data = init() + >>> setVarFlag('TEST', 'testflag', 1, data) + >>> print getVarFlag('TEST', 'testflag', data) + 1 + >>> delVarFlags('TEST', data) + >>> print getVarFlags('TEST', data) + None + + """ + d.delVarFlags(var) + +def keys(d): + """Return a list of keys in d + + Example: + >>> d = init() + >>> setVar('TEST', 1, d) + >>> setVar('MOO' , 2, d) + >>> setVarFlag('TEST', 'test', 1, d) + >>> keys(d) + ['TEST', 'MOO'] + """ + return d.keys() + +def getData(d): + """Returns the data object used""" + return d + +def setData(newData, d): + """Sets the data object to the supplied value""" + d = newData + +__expand_var_regexp__ = re.compile(r"\${[^{}]+}") +__expand_python_regexp__ = re.compile(r"\${@.+?}") + +def expand(s, d, varname = None): + """Variable expansion using the data store. + + Example: + Standard expansion: + >>> d = init() + >>> setVar('A', 'sshd', d) + >>> print expand('/usr/bin/${A}', d) + /usr/bin/sshd + + Python expansion: + >>> d = init() + >>> print expand('result: ${@37 * 72}', d) + result: 2664 + + Shell expansion: + >>> d = init() + >>> print expand('${TARGET_MOO}', d) + ${TARGET_MOO} + >>> setVar('TARGET_MOO', 'yupp', d) + >>> print expand('${TARGET_MOO}',d) + yupp + >>> setVar('SRC_URI', 'http://somebug.${TARGET_MOO}', d) + >>> delVar('TARGET_MOO', d) + >>> print expand('${SRC_URI}', d) + http://somebug.${TARGET_MOO} + """ + def var_sub(match): + key = match.group()[2:-1] + if varname and key: + if varname == key: + raise Exception("variable %s references itself!" % varname) + var = getVar(key, d, 1) + if var is not None: + return var + else: + return match.group() + + def python_sub(match): + import bb + code = match.group()[3:-1] + locals()['d'] = d + s = eval(code) + if type(s) == types.IntType: s = str(s) + return s + + if type(s) is not types.StringType: # sanity check + return s + + while s.find('$') != -1: + olds = s + try: + s = __expand_var_regexp__.sub(var_sub, s) + s = __expand_python_regexp__.sub(python_sub, s) + if s == olds: break + if type(s) is not types.StringType: # sanity check + import bb + bb.error('expansion of %s returned non-string %s' % (olds, s)) + except KeyboardInterrupt: + raise + except: + note("%s:%s while evaluating:\n%s" % (sys.exc_info()[0], sys.exc_info()[1], s)) + raise + return s + +def expandKeys(alterdata, readdata = None): + if readdata == None: + readdata = alterdata + + for key in keys(alterdata): + ekey = expand(key, readdata) + if key == ekey: + continue + val = getVar(key, alterdata) + if val is None: + continue +# import copy +# setVarFlags(ekey, copy.copy(getVarFlags(key, readdata)), alterdata) + setVar(ekey, val, alterdata) + + for i in ('_append', '_prepend', '_delete'): + dest = getVarFlag(ekey, i, alterdata) or [] + src = getVarFlag(key, i, readdata) or [] + dest.extend(src) + setVarFlag(ekey, i, dest, alterdata) + + delVar(key, alterdata) + +def expandData(alterdata, readdata = None): + """For each variable in alterdata, expand it, and update the var contents. + Replacements use data from readdata. + + Example: + >>> a=init() + >>> b=init() + >>> setVar("dlmsg", "dl_dir is ${DL_DIR}", a) + >>> setVar("DL_DIR", "/path/to/whatever", b) + >>> expandData(a, b) + >>> print getVar("dlmsg", a) + dl_dir is /path/to/whatever + """ + if readdata == None: + readdata = alterdata + + for key in keys(alterdata): + val = getVar(key, alterdata) + if type(val) is not types.StringType: + continue + expanded = expand(val, readdata) +# print "key is %s, val is %s, expanded is %s" % (key, val, expanded) + if val != expanded: + setVar(key, expanded, alterdata) + +import os + +def inheritFromOS(d): + """Inherit variables from the environment.""" +# fakeroot needs to be able to set these + non_inherit_vars = [ "LD_LIBRARY_PATH", "LD_PRELOAD" ] + for s in os.environ.keys(): + if not s in non_inherit_vars: + try: + setVar(s, os.environ[s], d) + setVarFlag(s, 'matchesenv', '1', d) + except TypeError: + pass + +import sys + +def emit_var(var, o=sys.__stdout__, d = init(), all=False): + """Emit a variable to be sourced by a shell.""" + if getVarFlag(var, "python", d): + return 0 + + try: + if all: + oval = getVar(var, d, 0) + val = getVar(var, d, 1) + except KeyboardInterrupt: + raise + except: + excname = str(sys.exc_info()[0]) + if excname == "bb.build.FuncFailed": + raise + o.write('# expansion of %s threw %s\n' % (var, excname)) + return 0 + + if all: + o.write('# %s=%s\n' % (var, oval)) + + if type(val) is not types.StringType: + return 0 + + if getVarFlag(var, 'matchesenv', d): + return 0 + + if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all: + return 0 + + val.rstrip() + if not val: + return 0 + + if getVarFlag(var, "func", d): +# NOTE: should probably check for unbalanced {} within the var + o.write("%s() {\n%s\n}\n" % (var, val)) + else: + if getVarFlag(var, "export", d): + o.write('export ') + else: + if not all: + return 0 +# if we're going to output this within doublequotes, +# to a shell, we need to escape the quotes in the var + alter = re.sub('"', '\\"', val.strip()) + o.write('%s="%s"\n' % (var, alter)) + return 1 + + +def emit_env(o=sys.__stdout__, d = init(), all=False): + """Emits all items in the data store in a format such that it can be sourced by a shell.""" + + env = keys(d) + + for e in env: + if getVarFlag(e, "func", d): + continue + emit_var(e, o, d, all) and o.write('\n') + + for e in env: + if not getVarFlag(e, "func", d): + continue + emit_var(e, o, d) and o.write('\n') + +def update_data(d): + """Modifies the environment vars according to local overrides and commands. + Examples: + Appending to a variable: + >>> d = init() + >>> setVar('TEST', 'this is a', d) + >>> setVar('TEST_append', ' test', d) + >>> setVar('TEST_append', ' of the emergency broadcast system.', d) + >>> update_data(d) + >>> print getVar('TEST', d) + this is a test of the emergency broadcast system. + + Prepending to a variable: + >>> setVar('TEST', 'virtual/libc', d) + >>> setVar('TEST_prepend', 'virtual/tmake ', d) + >>> setVar('TEST_prepend', 'virtual/patcher ', d) + >>> update_data(d) + >>> print getVar('TEST', d) + virtual/patcher virtual/tmake virtual/libc + + Overrides: + >>> setVar('TEST_arm', 'target', d) + >>> setVar('TEST_ramses', 'machine', d) + >>> setVar('TEST_local', 'local', d) + >>> setVar('OVERRIDES', 'arm', d) + + >>> setVar('TEST', 'original', d) + >>> update_data(d) + >>> print getVar('TEST', d) + target + + >>> setVar('OVERRIDES', 'arm:ramses:local', d) + >>> setVar('TEST', 'original', d) + >>> update_data(d) + >>> print getVar('TEST', d) + local + """ + + debug(2, "update_data()") + +# can't do delete env[...] while iterating over the dictionary, so remember them + dodel = [] + overrides = (getVar('OVERRIDES', d, 1) or "").split(':') or [] + + def applyOverrides(var, d): + if not overrides: + debug(1, "OVERRIDES not defined, nothing to do") + return + val = getVar(var, d) + for o in overrides: + if var.endswith("_" + o): + l = len(o)+1 + name = var[:-l] + d[name] = d[var] + + for s in keys(d): + applyOverrides(s, d) + sval = getVar(s, d) or "" + +# Handle line appends: + for (a, o) in getVarFlag(s, '_append', d) or []: + # maybe the OVERRIDE was not yet added so keep the append + if (o and o in overrides) or not o: + delVarFlag(s, '_append', d) + if o: + if not o in overrides: + continue + sval+=a + setVar(s, sval, d) + +# Handle line prepends + for (a, o) in getVarFlag(s, '_prepend', d) or []: + # maybe the OVERRIDE was not yet added so keep the append + if (o and o in overrides) or not o: + delVarFlag(s, '_prepend', d) + if o: + if not o in overrides: + continue + sval=a+sval + setVar(s, sval, d) + +# Handle line deletions + name = s + "_delete" + nameval = getVar(name, d) + if nameval: + sval = getVar(s, d) + if sval: + new = '' + pattern = nameval.replace('\n','').strip() + for line in sval.split('\n'): + if line.find(pattern) == -1: + new = new + '\n' + line + setVar(s, new, d) + dodel.append(name) + +# delete all environment vars no longer needed + for s in dodel: + delVar(s, d) + +def inherits_class(klass, d): + val = getVar('__inherit_cache', d) or "" + if os.path.join('classes', '%s.bbclass' % klass) in val.split(): + return True + return False + +def _test(): + """Start a doctest run on this module""" + import doctest + from bb import data + doctest.testmod(data) + +if __name__ == "__main__": + _test() diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py new file mode 100644 index 0000000..7417905 --- /dev/null +++ b/bitbake/lib/bb/data_smart.py @@ -0,0 +1,351 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake Smart Dictionary Implementation + +Functions for interacting with the data structure used by the +BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson +Copyright (C) 2004, 2005 Seb Frankengul +Copyright (C) 2005 Holger Hans Peter Freyther +Copyright (C) 2005 Uli Luckas +Copyright (C) 2005 ROAD GmbH + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +Based on functions from the base bb module, Copyright 2003 Holger Schurig +""" + +import copy, os, re, sys, time, types +from bb import note, debug, fatal + +try: + import cPickle as pickle +except ImportError: + import pickle + print "NOTE: Importing cPickle failed. Falling back to a very slow implementation." + + +__setvar_keyword__ = ["_append","_prepend","_delete"] +__setvar_regexp__ = re.compile('(?P.*?)(?P_append|_prepend|_delete)(_(?P.*))?') +__expand_var_regexp__ = re.compile(r"\${[^{}]+}") +__expand_python_regexp__ = re.compile(r"\${@.+?}") + + +class DataSmart: + def __init__(self): + self.dict = {} + + def expand(self,s, varname): + def var_sub(match): + key = match.group()[2:-1] + if varname and key: + if varname == key: + raise Exception("variable %s references itself!" % varname) + var = self.getVar(key, 1) + if var is not None: + return var + else: + return match.group() + + def python_sub(match): + import bb + code = match.group()[3:-1] + locals()['d'] = self + s = eval(code) + if type(s) == types.IntType: s = str(s) + return s + + if type(s) is not types.StringType: # sanity check + return s + + while s.find('$') != -1: + olds = s + try: + s = __expand_var_regexp__.sub(var_sub, s) + s = __expand_python_regexp__.sub(python_sub, s) + if s == olds: break + if type(s) is not types.StringType: # sanity check + import bb + bb.error('expansion of %s returned non-string %s' % (olds, s)) + except KeyboardInterrupt: + raise + except: + note("%s:%s while evaluating:\n%s" % (sys.exc_info()[0], sys.exc_info()[1], s)) + raise + return s + + def initVar(self, var): + if not var in self.dict: + self.dict[var] = {} + + def pickle_prep(self, cfg): + if "_data" in self.dict: + if self.dict["_data"] == cfg: + self.dict["_data"] = "cfg"; + else: # this is an unknown array for the moment + pass + + def unpickle_prep(self, cfg): + if "_data" in self.dict: + if self.dict["_data"] == "cfg": + self.dict["_data"] = cfg; + + def _findVar(self,var): + _dest = self.dict + + while (_dest and var not in _dest): + if not "_data" in _dest: + _dest = None + break + _dest = _dest["_data"] + + if _dest and var in _dest: + return _dest[var] + return None + + def _copyVar(self,var,name): + local_var = self._findVar(var) + if local_var: + self.dict[name] = copy.copy(local_var) + else: + debug(1,"Warning, _copyVar %s to %s, %s does not exists" % (var,name,var)) + + + def _makeShadowCopy(self, var): + if var in self.dict: + return + + local_var = self._findVar(var) + + if local_var: + self.dict[var] = copy.copy(local_var) + else: + self.initVar(var) + + def setVar(self,var,value): + match = __setvar_regexp__.match(var) + if match and match.group("keyword") in __setvar_keyword__: + base = match.group('base') + keyword = match.group("keyword") + override = match.group('add') + l = self.getVarFlag(base, keyword) or [] + if override == 'delete': + if l.count([value, None]): + del l[l.index([value, None])] + l.append([value, override]) + self.setVarFlag(base, match.group("keyword"), l) + return + + if not var in self.dict: + self._makeShadowCopy(var) + if self.getVarFlag(var, 'matchesenv'): + self.delVarFlag(var, 'matchesenv') + self.setVarFlag(var, 'export', 1) + + # setting var + self.dict[var]["content"] = value + + def getVar(self,var,exp): + value = self.getVarFlag(var,"content") + + if exp and value: + return self.expand(value,var) + return value + + def delVar(self,var): + self.dict[var] = {} + + def setVarFlag(self,var,flag,flagvalue): + if not var in self.dict: + self._makeShadowCopy(var) + self.dict[var][flag] = flagvalue + + def getVarFlag(self,var,flag): + local_var = self._findVar(var) + if local_var: + if flag in local_var: + return copy.copy(local_var[flag]) + return None + + def delVarFlag(self,var,flag): + local_var = self._findVar(var) + if not local_var: + return + if not var in self.dict: + self._makeShadowCopy(var) + + if var in self.dict and flag in self.dict[var]: + del self.dict[var][flag] + + def setVarFlags(self,var,flags): + if not var in self.dict: + self._makeShadowCopy(var) + + for i in flags.keys(): + if i == "content": + continue + self.dict[var][i] = flags[i] + + def getVarFlags(self,var): + local_var = self._findVar(var) + flags = {} + + if local_var: + for i in self.dict[var].keys(): + if i == "content": + continue + flags[i] = self.dict[var][i] + + if len(flags) == 0: + return None + return flags + + + def delVarFlags(self,var): + if not var in self.dict: + self._makeShadowCopy(var) + + if var in self.dict: + content = None + + # try to save the content + if "content" in self.dict[var]: + content = self.dict[var]["content"] + self.dict[var] = {} + self.dict[var]["content"] = content + else: + del self.dict[var] + + + def createCopy(self): + """ + Create a copy of self by setting _data to self + """ + # we really want this to be a DataSmart... + data = DataSmart() + data.dict["_data"] = self.dict + + return data + + # Dictionary Methods + def keys(self): + def _keys(d, mykey): + if "_data" in d: + _keys(d["_data"],mykey) + + for key in d.keys(): + if key != "_data": + mykey[key] = None + keytab = {} + _keys(self.dict,keytab) + return keytab.keys() + + def __getitem__(self,item): + start = self.dict + while start: + if item in start: + return start[item] + elif "_data" in start: + start = start["_data"] + else: + start = None + return None + + def __setitem__(self,var,data): + self._makeShadowCopy(var) + self.dict[var] = data + + +class DataSmartPackage(DataSmart): + """ + Persistent Data Storage + """ + def sanitize_filename(bbfile): + return bbfile.replace( '/', '_' ) + sanitize_filename = staticmethod(sanitize_filename) + + def unpickle(self): + """ + Restore the dict from memory + """ + cache_bbfile = self.sanitize_filename(self.bbfile) + p = pickle.Unpickler( file("%s/%s"%(self.cache,cache_bbfile),"rb")) + self.dict = p.load() + self.unpickle_prep() + funcstr = self.getVar('__functions__', 0) + if funcstr: + comp = compile(funcstr, "", "exec") + exec comp in __builtins__ + + def linkDataSet(self): + if not self.parent == None: + # assume parent is a DataSmartInstance + self.dict["_data"] = self.parent.dict + + + def __init__(self,cache,name,clean,parent): + """ + Construct a persistent data instance + """ + #Initialize the dictionary + DataSmart.__init__(self) + + self.cache = cache + self.bbfile = os.path.abspath( name ) + self.parent = parent + + # Either unpickle the data or do copy on write + if clean: + self.linkDataSet() + else: + self.unpickle() + + def commit(self, mtime): + """ + Save the package to a permanent storage + """ + self.pickle_prep() + + cache_bbfile = self.sanitize_filename(self.bbfile) + p = pickle.Pickler(file("%s/%s" %(self.cache,cache_bbfile), "wb" ), -1 ) + p.dump( self.dict ) + + self.unpickle_prep() + + def mtime(cache,bbfile): + cache_bbfile = DataSmartPackage.sanitize_filename(bbfile) + try: + return os.stat( "%s/%s" % (cache,cache_bbfile) )[8] + except OSError: + return 0 + mtime = staticmethod(mtime) + + def pickle_prep(self): + """ + If self.dict contains a _data key and it is a configuration + we will remember we had a configuration instance attached + """ + if "_data" in self.dict: + if self.dict["_data"] == self.parent: + dest["_data"] = "cfg" + + def unpickle_prep(self): + """ + If we had a configuration instance attached, we will reattach it + """ + if "_data" in self.dict: + if self.dict["_data"] == "cfg": + self.dict["_data"] = self.parent diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py new file mode 100644 index 0000000..c4e88fa --- /dev/null +++ b/bitbake/lib/bb/event.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Event' implementation + +Classes and functions for manipulating 'events' in the +BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. +""" + +import os, re +import bb.data + +class Event: + """Base class for events""" + type = "Event" + + def __init__(self, d = bb.data.init()): + self._data = d + + def getData(self): + return self._data + + def setData(self, data): + self._data = data + + data = property(getData, setData, None, "data property") + +NotHandled = 0 +Handled = 1 +handlers = [] + +def tmpHandler(event): + """Default handler for code events""" + return NotHandled + +def defaultTmpHandler(): + tmp = "def tmpHandler(e):\n\t\"\"\"heh\"\"\"\n\treturn 0" + comp = compile(tmp, "tmpHandler(e)", "exec") + return comp + +def fire(event): + """Fire off an Event""" + for h in handlers: + if type(h).__name__ == "code": + exec(h) + if tmpHandler(event) == Handled: + return Handled + else: + if h(event) == Handled: + return Handled + return NotHandled + +def register(handler): + """Register an Event handler""" + if handler is not None: +# handle string containing python code + if type(handler).__name__ == "str": + return registerCode(handler) +# prevent duplicate registration + if not handler in handlers: + handlers.append(handler) + +def registerCode(handlerStr): + """Register a 'code' Event. + Deprecated interface; call register instead. + + Expects to be passed python code as a string, which will + be passed in turn to compile() and then exec(). Note that + the code will be within a function, so should have had + appropriate tabbing put in place.""" + tmp = "def tmpHandler(e):\n%s" % handlerStr + comp = compile(tmp, "tmpHandler(e)", "exec") +# prevent duplicate registration + if not comp in handlers: + handlers.append(comp) + +def remove(handler): + """Remove an Event handler""" + for h in handlers: + if type(handler).__name__ == "str": + return removeCode(handler) + + if handler is h: + handlers.remove(handler) + +def removeCode(handlerStr): + """Remove a 'code' Event handler + Deprecated interface; call remove instead.""" + tmp = "def tmpHandler(e):\n%s" % handlerStr + comp = compile(tmp, "tmpHandler(e)", "exec") + handlers.remove(comp) + +def getName(e): + """Returns the name of a class or class instance""" + if getattr(e, "__name__", None) == None: + return e.__class__.__name__ + else: + return e.__name__ + + +class PkgBase(Event): + """Base class for package events""" + + def __init__(self, t, d = {}): + self._pkg = t + Event.__init__(self, d) + + def getPkg(self): + return self._pkg + + def setPkg(self, pkg): + self._pkg = pkg + + pkg = property(getPkg, setPkg, None, "pkg property") + + +class BuildBase(Event): + """Base class for bbmake run events""" + + def __init__(self, n, p, c): + self._name = n + self._pkgs = p + Event.__init__(self, c) + + def getPkgs(self): + return self._pkgs + + def setPkgs(self, pkgs): + self._pkgs = pkgs + + def getName(self): + return self._name + + def setName(self, name): + self._name = name + + def getCfg(self): + return self.data + + def setCfg(self, cfg): + self.data = cfg + + pkgs = property(getPkgs, setPkgs, None, "pkgs property") + name = property(getName, setName, None, "name property") + cfg = property(getCfg, setCfg, None, "cfg property") + + +class DepBase(PkgBase): + """Base class for dependency events""" + + def __init__(self, t, data, d): + self._dep = d + PkgBase.__init__(self, t, data) + + def getDep(self): + return self._dep + + def setDep(self, dep): + self._dep = dep + + dep = property(getDep, setDep, None, "dep property") + + +class PkgStarted(PkgBase): + """Package build started""" + + +class PkgFailed(PkgBase): + """Package build failed""" + + +class PkgSucceeded(PkgBase): + """Package build completed""" + + +class BuildStarted(BuildBase): + """bbmake build run started""" + + +class BuildCompleted(BuildBase): + """bbmake build run completed""" + + +class UnsatisfiedDep(DepBase): + """Unsatisfied Dependency""" + + +class RecursiveDep(DepBase): + """Recursive Dependency""" + + +class MultipleProviders(PkgBase): + """Multiple Providers""" + diff --git a/bitbake/lib/bb/fetch/.gitignore b/bitbake/lib/bb/fetch/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/lib/bb/fetch/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py new file mode 100644 index 0000000..d68a1f1 --- /dev/null +++ b/bitbake/lib/bb/fetch/__init__.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +Based on functions from the base bb module, Copyright 2003 Holger Schurig +""" + +import os, re +import bb +from bb import data + +class FetchError(Exception): + """Exception raised when a download fails""" + +class NoMethodError(Exception): + """Exception raised when there is no method to obtain a supplied url or set of urls""" + +class MissingParameterError(Exception): + """Exception raised when a fetch method is missing a critical parameter in the url""" + +class MD5SumError(Exception): + """Exception raised when a MD5SUM of a file does not match the expected one""" + +def uri_replace(uri, uri_find, uri_replace, d): +# bb.note("uri_replace: operating on %s" % uri) + if not uri or not uri_find or not uri_replace: + bb.debug(1, "uri_replace: passed an undefined value, not replacing") + uri_decoded = list(bb.decodeurl(uri)) + uri_find_decoded = list(bb.decodeurl(uri_find)) + uri_replace_decoded = list(bb.decodeurl(uri_replace)) + result_decoded = ['','','','','',{}] + for i in uri_find_decoded: + loc = uri_find_decoded.index(i) + result_decoded[loc] = uri_decoded[loc] + import types + if type(i) == types.StringType: + import re + if (re.match(i, uri_decoded[loc])): + result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc]) + if uri_find_decoded.index(i) == 2: + if d: + localfn = bb.fetch.localpath(uri, d) + if localfn: + result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d)) +# bb.note("uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc])) + else: +# bb.note("uri_replace: no match") + return uri +# else: +# for j in i.keys(): +# FIXME: apply replacements against options + return bb.encodeurl(result_decoded) + +methods = [] + +def init(urls = [], d = None): + if d == None: + bb.debug(2,"BUG init called with None as data object!!!") + return + + for m in methods: + m.urls = [] + + for u in urls: + for m in methods: + m.data = d + if m.supports(u, d): + m.urls.append(u) + +def go(d): + """Fetch all urls""" + for m in methods: + if m.urls: + m.go(d) + +def localpaths(d): + """Return a list of the local filenames, assuming successful fetch""" + local = [] + for m in methods: + for u in m.urls: + local.append(m.localpath(u, d)) + return local + +def localpath(url, d): + for m in methods: + if m.supports(url, d): + return m.localpath(url, d) + return url + +class Fetch(object): + """Base class for 'fetch'ing data""" + + def __init__(self, urls = []): + self.urls = [] + for url in urls: + if self.supports(bb.decodeurl(url), d) is 1: + self.urls.append(url) + + def supports(url, d): + """Check to see if this fetch class supports a given url. + Expects supplied url in list form, as outputted by bb.decodeurl(). + """ + return 0 + supports = staticmethod(supports) + + def localpath(url, d): + """Return the local filename of a given url assuming a successful fetch. + """ + return url + localpath = staticmethod(localpath) + + def setUrls(self, urls): + self.__urls = urls + + def getUrls(self): + return self.__urls + + urls = property(getUrls, setUrls, None, "Urls property") + + def setData(self, data): + self.__data = data + + def getData(self): + return self.__data + + data = property(getData, setData, None, "Data property") + + def go(self, urls = []): + """Fetch urls""" + raise NoMethodError("Missing implementation for url") + +#if __name__ == "__main__": + +import bk +import cvs +import git +import local +import svn +import wget + +methods.append(bk.Bk()) +methods.append(cvs.Cvs()) +methods.append(git.Git()) +methods.append(local.Local()) +methods.append(svn.Svn()) +methods.append(wget.Wget()) diff --git a/bitbake/lib/bb/fetch/bk.py b/bitbake/lib/bb/fetch/bk.py new file mode 100644 index 0000000..6bd6c01 --- /dev/null +++ b/bitbake/lib/bb/fetch/bk.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +Based on functions from the base bb module, Copyright 2003 Holger Schurig +""" + +import os, re +import bb +from bb import data +from bb.fetch import Fetch + +class Bk(Fetch): + def supports(url, d): + """Check to see if a given url can be fetched via bitkeeper. + Expects supplied url in list form, as outputted by bb.decodeurl(). + """ + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) + return type in ['bk'] + supports = staticmethod(supports) diff --git a/bitbake/lib/bb/fetch/cvs.py b/bitbake/lib/bb/fetch/cvs.py new file mode 100644 index 0000000..7935744 --- /dev/null +++ b/bitbake/lib/bb/fetch/cvs.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +Based on functions from the base bb module, Copyright 2003 Holger Schurig +""" + +import os, re +import bb +from bb import data +from bb.fetch import Fetch +from bb.fetch import FetchError +from bb.fetch import MissingParameterError + +class Cvs(Fetch): + """Class to fetch a module or modules from cvs repositories""" + def supports(url, d): + """Check to see if a given url can be fetched with cvs. + Expects supplied url in list form, as outputted by bb.decodeurl(). + """ + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) + return type in ['cvs', 'pserver'] + supports = staticmethod(supports) + + def localpath(url, d): + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) + if "localpath" in parm: +# if user overrides local path, use it. + return parm["localpath"] + + if not "module" in parm: + raise MissingParameterError("cvs method needs a 'module' parameter") + else: + module = parm["module"] + if 'tag' in parm: + tag = parm['tag'] + else: + tag = "" + if 'date' in parm: + date = parm['date'] + else: + if not tag: + date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1) + else: + date = "" + + return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, tag, date), d)) + localpath = staticmethod(localpath) + + def go(self, d, urls = []): + """Fetch urls""" + if not urls: + urls = self.urls + + localdata = data.createCopy(d) + data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata) + data.update_data(localdata) + + for loc in urls: + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata)) + if not "module" in parm: + raise MissingParameterError("cvs method needs a 'module' parameter") + else: + module = parm["module"] + + dlfile = self.localpath(loc, localdata) + dldir = data.getVar('DL_DIR', localdata, 1) +# if local path contains the cvs +# module, consider the dir above it to be the +# download directory +# pos = dlfile.find(module) +# if pos: +# dldir = dlfile[:pos] +# else: +# dldir = os.path.dirname(dlfile) + +# setup cvs options + options = [] + if 'tag' in parm: + tag = parm['tag'] + else: + tag = "" + + if 'date' in parm: + date = parm['date'] + else: + if not tag: + date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1) + else: + date = "" + + if "method" in parm: + method = parm["method"] + else: + method = "pserver" + + if "localdir" in parm: + localdir = parm["localdir"] + else: + localdir = module + + cvs_rsh = None + if method == "ext": + if "rsh" in parm: + cvs_rsh = parm["rsh"] + + tarfn = data.expand('%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, tag, date), localdata) + data.setVar('TARFILES', dlfile, localdata) + data.setVar('TARFN', tarfn, localdata) + + dl = os.path.join(dldir, tarfn) + if os.access(dl, os.R_OK): + bb.debug(1, "%s already exists, skipping cvs checkout." % tarfn) + continue + + pn = data.getVar('PN', d, 1) + cvs_tarball_stash = None + if pn: + cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1) + if cvs_tarball_stash == None: + cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1) + if cvs_tarball_stash: + fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1) + uri = cvs_tarball_stash + tarfn + bb.note("fetch " + uri) + fetchcmd = fetchcmd.replace("${URI}", uri) + ret = os.system(fetchcmd) + if ret == 0: + bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn) + continue + + if date: + options.append("-D %s" % date) + if tag: + options.append("-r %s" % tag) + + olddir = os.path.abspath(os.getcwd()) + os.chdir(data.expand(dldir, localdata)) + +# setup cvsroot + if method == "dir": + cvsroot = path + else: + cvsroot = ":" + method + ":" + user + if pswd: + cvsroot += ":" + pswd + cvsroot += "@" + host + ":" + path + + data.setVar('CVSROOT', cvsroot, localdata) + data.setVar('CVSCOOPTS', " ".join(options), localdata) + data.setVar('CVSMODULE', module, localdata) + cvscmd = data.getVar('FETCHCOMMAND', localdata, 1) + cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1) + + if cvs_rsh: + cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd) + cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd) + +# create module directory + bb.debug(2, "Fetch: checking for module directory") + pkg=data.expand('${PN}', d) + pkgdir=os.path.join(data.expand('${CVSDIR}', localdata), pkg) + moddir=os.path.join(pkgdir,localdir) + if os.access(os.path.join(moddir,'CVS'), os.R_OK): + bb.note("Update " + loc) +# update sources there + os.chdir(moddir) + myret = os.system(cvsupdatecmd) + else: + bb.note("Fetch " + loc) +# check out sources there + bb.mkdirhier(pkgdir) + os.chdir(pkgdir) + bb.debug(1, "Running %s" % cvscmd) + myret = os.system(cvscmd) + + if myret != 0: + try: + os.rmdir(moddir) + except OSError: + pass + raise FetchError(module) + + os.chdir(moddir) + os.chdir('..') +# tar them up to a defined filename + myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(moddir))) + if myret != 0: + try: + os.unlink(tarfn) + except OSError: + pass + os.chdir(olddir) + del localdata diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py new file mode 100644 index 0000000..296b926 --- /dev/null +++ b/bitbake/lib/bb/fetch/git.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' git implementation + +Copyright (C) 2005 Richard Purdie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. +""" + +import os, re +import bb +from bb import data +from bb.fetch import Fetch +from bb.fetch import FetchError + +def prunedir(topdir): + # Delete everything reachable from the directory named in 'topdir'. + # CAUTION: This is dangerous! + for root, dirs, files in os.walk(topdir, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + +def rungitcmd(cmd,d): + + bb.debug(1, "Running %s" % cmd) + + # Need to export PATH as git is likely to be in metadata paths + # rather than host provided + pathcmd = 'export PATH=%s; %s' % (data.expand('${PATH}', d), cmd) + + myret = os.system(pathcmd) + + if myret != 0: + raise FetchError("Git: %s failed" % pathcmd) + +def gettag(parm): + if 'tag' in parm: + tag = parm['tag'] + else: + tag = "" + if not tag: + tag = "master" + + return tag + +class Git(Fetch): + """Class to fetch a module or modules from git repositories""" + def supports(url, d): + """Check to see if a given url can be fetched with cvs. + Expects supplied url in list form, as outputted by bb.decodeurl(). + """ + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) + return type in ['git'] + supports = staticmethod(supports) + + def localpath(url, d): + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) + + #if user sets localpath for file, use it instead. + if "localpath" in parm: + return parm["localpath"] + + tag = gettag(parm) + + localname = data.expand('git_%s%s_%s.tar.gz' % (host, path.replace('/', '.'), tag), d) + + return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s' % (localname), d)) + + localpath = staticmethod(localpath) + + def go(self, d, urls = []): + """Fetch urls""" + if not urls: + urls = self.urls + + for loc in urls: + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, d)) + + tag = gettag(parm) + + gitsrcname = '%s%s' % (host, path.replace('/', '.')) + + repofile = os.path.join(data.getVar("DL_DIR", d, 1), 'git_%s.tar.gz' % (gitsrcname)) + repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) + + coname = '%s' % (tag) + codir = os.path.join(repodir, coname) + + cofile = self.localpath(loc, d) + + # Always update to current if tag=="master" + #if os.access(cofile, os.R_OK) and (tag != "master"): + if os.access(cofile, os.R_OK): + bb.debug(1, "%s already exists, skipping git checkout." % cofile) + continue + +# Still Need to add GIT_TARBALL_STASH Support... +# pn = data.getVar('PN', d, 1) +# cvs_tarball_stash = None +# if pn: +# cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1) +# if cvs_tarball_stash == None: +# cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1) +# if cvs_tarball_stash: +# fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1) +# uri = cvs_tarball_stash + tarfn +# bb.note("fetch " + uri) +# fetchcmd = fetchcmd.replace("${URI}", uri) +# ret = os.system(fetchcmd) +# if ret == 0: +# bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn) +# continue + + #if os.path.exists(repodir): + #prunedir(repodir) + + bb.mkdirhier(repodir) + os.chdir(repodir) + + #print("Changing to %s" % repodir) + + if os.access(repofile, os.R_OK): + rungitcmd("tar -xzf %s" % (repofile),d) + else: + rungitcmd("git clone rsync://%s%s %s" % (host, path, repodir),d) + + rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d) + + #print("Changing to %s" % repodir) + os.chdir(repodir) + rungitcmd("git pull rsync://%s%s" % (host, path),d) + + #print("Changing to %s" % repodir) + os.chdir(repodir) + rungitcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ),d) + + if os.path.exists(codir): + prunedir(codir) + + #print("Changing to %s" % repodir) + bb.mkdirhier(codir) + os.chdir(repodir) + rungitcmd("git read-tree %s" % (tag),d) + + rungitcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")),d) + + #print("Changing to %s" % codir) + os.chdir(codir) + rungitcmd("tar -czf %s %s" % (cofile, os.path.join(".", "*") ),d) + diff --git a/bitbake/lib/bb/fetch/local.py b/bitbake/lib/bb/fetch/local.py new file mode 100644 index 0000000..51938f8 --- /dev/null +++ b/bitbake/lib/bb/fetch/local.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +Based on functions from the base bb module, Copyright 2003 Holger Schurig +""" + +import os, re +import bb +from bb import data +from bb.fetch import Fetch + +class Local(Fetch): + def supports(url, d): + """Check to see if a given url can be fetched in the local filesystem. + Expects supplied url in list form, as outputted by bb.decodeurl(). + """ + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) + return type in ['file','patch'] + supports = staticmethod(supports) + + def localpath(url, d): + """Return the local filename of a given url assuming a successful fetch. + """ + path = url.split("://")[1] + newpath = path + if path[0] != "/": + filespath = data.getVar('FILESPATH', d, 1) + if filespath: + newpath = bb.which(filespath, path) + if not newpath: + filesdir = data.getVar('FILESDIR', d, 1) + if filesdir: + newpath = os.path.join(filesdir, path) + return newpath + localpath = staticmethod(localpath) + + def go(self, urls = []): + """Fetch urls (no-op for Local method)""" +# no need to fetch local files, we'll deal with them in place. + return 1 diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py new file mode 100644 index 0000000..97672a3 --- /dev/null +++ b/bitbake/lib/bb/fetch/svn.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +Based on functions from the base bb module, Copyright 2003 Holger Schurig +""" + +import os, re +import bb +from bb import data +from bb.fetch import Fetch +from bb.fetch import FetchError +from bb.fetch import MissingParameterError + +class Svn(Fetch): + """Class to fetch a module or modules from svn repositories""" + def supports(url, d): + """Check to see if a given url can be fetched with svn. + Expects supplied url in list form, as outputted by bb.decodeurl(). + """ + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) + return type in ['svn'] + supports = staticmethod(supports) + + def localpath(url, d): + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) + if "localpath" in parm: +# if user overrides local path, use it. + return parm["localpath"] + + if not "module" in parm: + raise MissingParameterError("svn method needs a 'module' parameter") + else: + module = parm["module"] + if 'rev' in parm: + revision = parm['rev'] + else: + revision = "" + + date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1) + + return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, revision, date), d)) + localpath = staticmethod(localpath) + + def go(self, d, urls = []): + """Fetch urls""" + if not urls: + urls = self.urls + + localdata = data.createCopy(d) + data.setVar('OVERRIDES', "svn:%s" % data.getVar('OVERRIDES', localdata), localdata) + data.update_data(localdata) + + for loc in urls: + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, localdata)) + if not "module" in parm: + raise MissingParameterError("svn method needs a 'module' parameter") + else: + module = parm["module"] + + dlfile = self.localpath(loc, localdata) + dldir = data.getVar('DL_DIR', localdata, 1) +# if local path contains the svn +# module, consider the dir above it to be the +# download directory +# pos = dlfile.find(module) +# if pos: +# dldir = dlfile[:pos] +# else: +# dldir = os.path.dirname(dlfile) + +# setup svn options + options = [] + if 'rev' in parm: + revision = parm['rev'] + else: + revision = "" + + date = data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1) + + if "method" in parm: + method = parm["method"] + else: + method = "pserver" + + if "proto" in parm: + proto = parm["proto"] + else: + proto = "svn" + + svn_rsh = None + if method == "ext": + if "rsh" in parm: + svn_rsh = parm["rsh"] + + tarfn = data.expand('%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, revision, date), localdata) + data.setVar('TARFILES', dlfile, localdata) + data.setVar('TARFN', tarfn, localdata) + + dl = os.path.join(dldir, tarfn) + if os.access(dl, os.R_OK): + bb.debug(1, "%s already exists, skipping svn checkout." % tarfn) + continue + + svn_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1) + if svn_tarball_stash: + fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1) + uri = svn_tarball_stash + tarfn + bb.note("fetch " + uri) + fetchcmd = fetchcmd.replace("${URI}", uri) + ret = os.system(fetchcmd) + if ret == 0: + bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn) + continue + + olddir = os.path.abspath(os.getcwd()) + os.chdir(data.expand(dldir, localdata)) + +# setup svnroot +# svnroot = ":" + method + ":" + user +# if pswd: +# svnroot += ":" + pswd + svnroot = host + path + + data.setVar('SVNROOT', svnroot, localdata) + data.setVar('SVNCOOPTS', " ".join(options), localdata) + data.setVar('SVNMODULE', module, localdata) + svncmd = data.getVar('FETCHCOMMAND', localdata, 1) + svncmd = "svn co %s://%s/%s" % (proto, svnroot, module) + + if revision: + svncmd = "svn co -r %s %s://%s/%s" % (revision, proto, svnroot, module) + if svn_rsh: + svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd) + +# create temp directory + bb.debug(2, "Fetch: creating temporary directory") + bb.mkdirhier(data.expand('${WORKDIR}', localdata)) + data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvn.XXXXXX', localdata), localdata) + tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false") + tmpfile = tmppipe.readline().strip() + if not tmpfile: + bb.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.") + raise FetchError(module) + +# check out sources there + os.chdir(tmpfile) + bb.note("Fetch " + loc) + bb.debug(1, "Running %s" % svncmd) + myret = os.system(svncmd) + if myret != 0: + try: + os.rmdir(tmpfile) + except OSError: + pass + raise FetchError(module) + + os.chdir(os.path.join(tmpfile, os.path.dirname(module))) +# tar them up to a defined filename + myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(module))) + if myret != 0: + try: + os.unlink(tarfn) + except OSError: + pass +# cleanup + os.system('rm -rf %s' % tmpfile) + os.chdir(olddir) + del localdata diff --git a/bitbake/lib/bb/fetch/wget.py b/bitbake/lib/bb/fetch/wget.py new file mode 100644 index 0000000..d9bbdd4 --- /dev/null +++ b/bitbake/lib/bb/fetch/wget.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +Based on functions from the base bb module, Copyright 2003 Holger Schurig +""" + +import os, re +import bb +from bb import data +from bb.fetch import Fetch +from bb.fetch import FetchError +from bb.fetch import MD5SumError +from bb.fetch import uri_replace + +class Wget(Fetch): + """Class to fetch urls via 'wget'""" + def supports(url, d): + """Check to see if a given url can be fetched using wget. + Expects supplied url in list form, as outputted by bb.decodeurl(). + """ + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) + return type in ['http','https','ftp'] + supports = staticmethod(supports) + + def localpath(url, d): +# strip off parameters + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) + if "localpath" in parm: +# if user overrides local path, use it. + return parm["localpath"] + url = bb.encodeurl([type, host, path, user, pswd, {}]) + + return os.path.join(data.getVar("DL_DIR", d), os.path.basename(url)) + localpath = staticmethod(localpath) + + def go(self, d, urls = []): + """Fetch urls""" + + def md5_sum(basename, d): + """ + Fast and incomplete OVERRIDE implementation for MD5SUM handling + MD5SUM_basename = "SUM" and fallback to MD5SUM_basename + """ + var = "MD5SUM_%s" % basename + return data.getVar(var, d) or data.getVar("MD5SUM", d) + + def verify_md5sum(wanted_sum, got_sum): + """ + Verify the md5sum we wanted with the one we got + """ + if not wanted_sum: + return True + + return wanted_sum == got_sum + + def fetch_uri(uri, basename, dl, md5, parm, d): + # the MD5 sum we want to verify + wanted_md5sum = md5_sum(basename, d) + if os.path.exists(dl): +# file exists, but we didnt complete it.. trying again.. + fetchcmd = data.getVar("RESUMECOMMAND", d, 1) + else: + fetchcmd = data.getVar("FETCHCOMMAND", d, 1) + + bb.note("fetch " + uri) + fetchcmd = fetchcmd.replace("${URI}", uri) + fetchcmd = fetchcmd.replace("${FILE}", basename) + bb.debug(2, "executing " + fetchcmd) + ret = os.system(fetchcmd) + if ret != 0: + return False + + # check if sourceforge did send us to the mirror page + dl_dir = data.getVar("DL_DIR", d, True) + if not os.path.exists(dl): + os.system("rm %s*" % dl) # FIXME shell quote it + bb.debug(2,"sourceforge.net send us to the mirror on %s" % basename) + return False + +# supposedly complete.. write out md5sum + if bb.which(data.getVar('PATH', d), 'md5sum'): + try: + md5pipe = os.popen('md5sum ' + dl) + md5data = (md5pipe.readline().split() or [ "" ])[0] + md5pipe.close() + except OSError: + md5data = "" + + # verify the md5sum + if not verify_md5sum(wanted_md5sum, md5data): + raise MD5SumError(uri) + + md5out = file(md5, 'w') + md5out.write(md5data) + md5out.close() + return True + + if not urls: + urls = self.urls + + localdata = data.createCopy(d) + data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata) + data.update_data(localdata) + + for uri in urls: + completed = 0 + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(uri, localdata)) + basename = os.path.basename(path) + dl = self.localpath(uri, d) + dl = data.expand(dl, localdata) + md5 = dl + '.md5' + + if os.path.exists(md5): +# complete, nothing to see here.. + continue + + premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ] + for (find, replace) in premirrors: + newuri = uri_replace(uri, find, replace, d) + if newuri != uri: + if fetch_uri(newuri, basename, dl, md5, parm, localdata): + completed = 1 + break + + if completed: + continue + + if fetch_uri(uri, basename, dl, md5, parm, localdata): + continue + +# try mirrors + mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ] + for (find, replace) in mirrors: + newuri = uri_replace(uri, find, replace, d) + if newuri != uri: + if fetch_uri(newuri, basename, dl, md5, parm, localdata): + completed = 1 + break + + if not completed: + raise FetchError(uri) + + del localdata diff --git a/bitbake/lib/bb/manifest.py b/bitbake/lib/bb/manifest.py new file mode 100644 index 0000000..30bb454 --- /dev/null +++ b/bitbake/lib/bb/manifest.py @@ -0,0 +1,144 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2003, 2004 Chris Larson +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. + +import os, sys +import bb, bb.data + +def getfields(line): + fields = {} + fieldmap = ( "pkg", "src", "dest", "type", "mode", "uid", "gid", "major", "minor", "start", "inc", "count" ) + for f in xrange(len(fieldmap)): + fields[fieldmap[f]] = None + + if not line: + return None + + splitline = line.split() + if not len(splitline): + return None + + try: + for f in xrange(len(fieldmap)): + if splitline[f] == '-': + continue + fields[fieldmap[f]] = splitline[f] + except IndexError: + pass + return fields + +def parse (mfile, d): + manifest = [] + while 1: + line = mfile.readline() + if not line: + break + if line.startswith("#"): + continue + fields = getfields(line) + if not fields: + continue + manifest.append(fields) + return manifest + +def emit (func, manifest, d): +#str = "%s () {\n" % func + str = "" + for line in manifest: + emittedline = emit_line(func, line, d) + if not emittedline: + continue + str += emittedline + "\n" +# str += "}\n" + return str + +def mangle (func, line, d): + import copy + newline = copy.copy(line) + src = bb.data.expand(newline["src"], d) + + if src: + if not os.path.isabs(src): + src = "${WORKDIR}/" + src + + dest = newline["dest"] + if not dest: + return + + if dest.startswith("/"): + dest = dest[1:] + + if func is "do_install": + dest = "${D}/" + dest + + elif func is "do_populate": + dest = "${WORKDIR}/install/" + newline["pkg"] + "/" + dest + + elif func is "do_stage": + varmap = {} + varmap["${bindir}"] = "${STAGING_DIR}/${HOST_SYS}/bin" + varmap["${libdir}"] = "${STAGING_DIR}/${HOST_SYS}/lib" + varmap["${includedir}"] = "${STAGING_DIR}/${HOST_SYS}/include" + varmap["${datadir}"] = "${STAGING_DATADIR}" + + matched = 0 + for key in varmap.keys(): + if dest.startswith(key): + dest = varmap[key] + "/" + dest[len(key):] + matched = 1 + if not matched: + newline = None + return + else: + newline = None + return + + newline["src"] = src + newline["dest"] = dest + return newline + +def emit_line (func, line, d): + import copy + newline = copy.deepcopy(line) + newline = mangle(func, newline, d) + if not newline: + return None + + str = "" + type = newline["type"] + mode = newline["mode"] + src = newline["src"] + dest = newline["dest"] + if type is "d": + str = "install -d " + if mode: + str += "-m %s " % mode + str += dest + elif type is "f": + if not src: + return None + if dest.endswith("/"): + str = "install -d " + str += dest + "\n" + str += "install " + else: + str = "install -D " + if mode: + str += "-m %s " % mode + str += src + " " + dest + del newline + return str diff --git a/bitbake/lib/bb/parse/.gitignore b/bitbake/lib/bb/parse/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/lib/bb/parse/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py new file mode 100644 index 0000000..cb27416 --- /dev/null +++ b/bitbake/lib/bb/parse/__init__.py @@ -0,0 +1,73 @@ +""" +BitBake Parsers + +File parsers for the BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson +Copyright (C) 2003, 2004 Phil Blundell + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +Based on functions from the base bb module, Copyright 2003 Holger Schurig +""" + +__all__ = [ 'ParseError', 'SkipPackage', 'cached_mtime', 'mark_dependency', + 'supports', 'handle', 'init' ] +handlers = [] + +import bb, os + +class ParseError(Exception): + """Exception raised when parsing fails""" + +class SkipPackage(Exception): + """Exception raised to skip this package""" + +__mtime_cache = {} +def cached_mtime(f): + if not __mtime_cache.has_key(f): + update_mtime(f) + return __mtime_cache[f] + +def update_mtime(f): + __mtime_cache[f] = os.stat(f)[8] + +def mark_dependency(d, f): + if f.startswith('./'): + f = "%s/%s" % (os.getcwd(), f[2:]) + deps = (bb.data.getVar('__depends', d) or "").split() + deps.append("%s@%s" % (f, cached_mtime(f))) + bb.data.setVar('__depends', " ".join(deps), d) + +def supports(fn, data): + """Returns true if we have a handler for this file, false otherwise""" + for h in handlers: + if h['supports'](fn, data): + return 1 + return 0 + +def handle(fn, data, include = 0): + """Call the handler that is appropriate for this file""" + for h in handlers: + if h['supports'](fn, data): + return h['handle'](fn, data, include) + raise ParseError("%s is not a BitBake file" % fn) + +def init(fn, data): + for h in handlers: + if h['supports'](fn): + return h['init'](data) + + +from parse_py import __version__, ConfHandler, BBHandler diff --git a/bitbake/lib/bb/parse/parse_c/.gitignore b/bitbake/lib/bb/parse/parse_c/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/lib/bb/parse/parse_c/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/lib/bb/parse/parse_c/bitbakeparser.l b/bitbake/lib/bb/parse/parse_c/bitbakeparser.l new file mode 100644 index 0000000..ee4ce14 --- /dev/null +++ b/bitbake/lib/bb/parse/parse_c/bitbakeparser.l @@ -0,0 +1,288 @@ +/* bbf.flex + + written by Marc Singer + 6 January 2005 + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + USA. + + DESCRIPTION + ----------- + + flex lexer specification for a BitBake input file parser. + + Unfortunately, flex doesn't welcome comments within the rule sets. + I say unfortunately because this lexer is unreasonably complex and + comments would make the code much easier to comprehend. + + The BitBake grammar is not regular. In order to interpret all + of the available input files, the lexer maintains much state as it + parses. There are places where this lexer will emit tokens that + are invalid. The parser will tend to catch these. + + The lexer requires C++ at the moment. The only reason for this has + to do with a very small amount of managed state. Producing a C + lexer should be a reasonably easy task as long as the %reentrant + option is used. + + + NOTES + ----- + + o RVALUES. There are three kinds of RVALUES. There are unquoted + values, double quote enclosed strings, and single quote + strings. Quoted strings may contain unescaped quotes (of either + type), *and* any type may span more than one line by using a + continuation '\' at the end of the line. This requires us to + recognize all types of values with a single expression. + Moreover, the only reason to quote a value is to include + trailing or leading whitespace. Whitespace within a value is + preserved, ugh. + + o CLASSES. C_ patterns define classes. Classes ought not include + a repitition operator, instead letting the reference to the class + define the repitition count. + + C_SS - symbol start + C_SB - symbol body + C_SP - whitespace + +*/ + +%option never-interactive +%option yylineno +%option noyywrap +%option reentrant stack + + +%{ + +#include "token.h" +#include "lexer.h" +#include + +extern void *bbparseAlloc(void *(*mallocProc)(size_t)); +extern void bbparseFree(void *p, void (*freeProc)(void*)); +extern void *bbparseAlloc(void *(*mallocProc)(size_t)); +extern void *bbparse(void*, int, token_t, lex_t*); +extern void bbparseTrace(FILE *TraceFILE, char *zTracePrompt); + +//static const char* rgbInput; +//static size_t cbInput; + + +int lineError; +int errorParse; + +enum { + errorNone = 0, + errorUnexpectedInput, + errorUnsupportedFeature, +}; + +#define YY_EXTRA_TYPE lex_t* + + /* Read from buffer */ +#define YY_INPUT(buf,result,max_size) \ + { yyextra->input(buf, &result, max_size); } + +//#define YY_DECL static size_t yylex () + +#define ERROR(e) \ + do { lineError = yylineno; errorParse = e; yyterminate (); } while (0) + +static const char* fixup_escapes (const char* sz); + +%} + + +C_SP [ \t] +COMMENT #.*\n +OP_ASSIGN "=" +OP_IMMEDIATE ":=" +OP_PREPEND "=+" +OP_APPEND "+=" +OP_COND "?=" +B_OPEN "{" +B_CLOSE "}" + +K_ADDTASK "addtask" +K_ADDHANDLER "addhandler" +K_AFTER "after" +K_BEFORE "before" +K_DEF "def" +K_INCLUDE "include" +K_INHERIT "inherit" +K_PYTHON "python" +K_FAKEROOT "fakeroot" +K_EXPORT "export" +K_EXPORT_FUNC "EXPORT_FUNCTIONS" + +STRING \"([^\n\r]|"\\\n")*\" +SSTRING \'([^\n\r]|"\\\n")*\' +VALUE ([^'" \t\n])|([^'" \t\n]([^\n]|(\\\n))*[^'" \t\n]) + +C_SS [a-zA-Z_] +C_SB [a-zA-Z0-9_+-.] +REF $\{{C_SS}{C_SB}*\} +SYMBOL {C_SS}{C_SB}* +VARIABLE $?{C_SS}({C_SB}*|{REF})*(\[[a-zA-Z0-9_]*\])? +FILENAME ([a-zA-Z_./]|{REF})(([-+a-zA-Z0-9_./]*)|{REF})* + +PROC \({C_SP}*\) + +%s S_DEF +%s S_DEF_ARGS +%s S_DEF_BODY +%s S_FUNC +%s S_INCLUDE +%s S_INHERIT +%s S_PROC +%s S_RVALUE +%s S_TASK + +%% + +{OP_APPEND} { BEGIN S_RVALUE; + yyextra->accept (T_OP_APPEND); } +{OP_PREPEND} { BEGIN S_RVALUE; + yyextra->accept (T_OP_PREPEND); } +{OP_IMMEDIATE} { BEGIN S_RVALUE; + yyextra->accept (T_OP_IMMEDIATE); } +{OP_ASSIGN} { BEGIN S_RVALUE; + yyextra->accept (T_OP_ASSIGN); } +{OP_COND} { BEGIN S_RVALUE; + yyextra->accept (T_OP_COND); } + +\\\n{C_SP}* { } +{STRING} { BEGIN INITIAL; + size_t cb = yyleng; + while (cb && isspace (yytext[cb - 1])) + --cb; + yytext[cb - 1] = 0; + yyextra->accept (T_STRING, yytext + 1); } +{SSTRING} { BEGIN INITIAL; + size_t cb = yyleng; + while (cb && isspace (yytext[cb - 1])) + --cb; + yytext[cb - 1] = 0; + yyextra->accept (T_STRING, yytext + 1); } + +{VALUE} { ERROR (errorUnexpectedInput); } +{C_SP}*\n+ { BEGIN INITIAL; + yyextra->accept (T_STRING, NULL); } + +{K_INCLUDE} { BEGIN S_INCLUDE; + yyextra->accept (T_INCLUDE); } +{K_INHERIT} { BEGIN S_INHERIT; + yyextra->accept (T_INHERIT); } +{K_ADDTASK} { BEGIN S_TASK; + yyextra->accept (T_ADDTASK); } +{K_ADDHANDLER} { yyextra->accept (T_ADDHANDLER); } +{K_EXPORT_FUNC} { BEGIN S_FUNC; + yyextra->accept (T_EXPORT_FUNC); } +{K_BEFORE} { yyextra->accept (T_BEFORE); } +{K_AFTER} { yyextra->accept (T_AFTER); } +{K_EXPORT} { yyextra->accept (T_EXPORT); } + +{K_FAKEROOT} { yyextra->accept (T_FAKEROOT); } +{K_PYTHON} { yyextra->accept (T_PYTHON); } +{PROC}{C_SP}*{B_OPEN}{C_SP}*\n* { BEGIN S_PROC; + yyextra->accept (T_PROC_OPEN); } +{B_CLOSE}{C_SP}*\n* { BEGIN INITIAL; + yyextra->accept (T_PROC_CLOSE); } +([^}][^\n]*)?\n* { yyextra->accept (T_PROC_BODY, yytext); } + +{K_DEF} { BEGIN S_DEF; } +{SYMBOL} { BEGIN S_DEF_ARGS; + yyextra->accept (T_SYMBOL, yytext); } +[^\n:]*: { yyextra->accept (T_DEF_ARGS, yytext); } +{C_SP}*\n { BEGIN S_DEF_BODY; } +{C_SP}+[^\n]*\n { yyextra->accept (T_DEF_BODY, yytext); } +\n { yyextra->accept (T_DEF_BODY, yytext); } +. { BEGIN INITIAL; unput (yytext[0]); } + +{COMMENT} { } + +{SYMBOL} { yyextra->accept (T_SYMBOL, yytext); } +{VARIABLE} { yyextra->accept (T_VARIABLE, yytext); } + +{SYMBOL} { yyextra->accept (T_TSYMBOL, yytext); } +{SYMBOL} { yyextra->accept (T_FSYMBOL, yytext); } +{SYMBOL} { yyextra->accept (T_ISYMBOL, yytext); } +{FILENAME} { BEGIN INITIAL; + yyextra->accept (T_ISYMBOL, yytext); } + +\n { BEGIN INITIAL; } +\n { BEGIN INITIAL; } +\n { BEGIN INITIAL; } + +[ \t\r\n] /* Insignificant whitespace */ + +. { ERROR (errorUnexpectedInput); } + + /* Check for premature termination */ +<> { return T_EOF; } + +%% + +void lex_t::accept (int token, const char* sz) +{ + token_t t; + memset (&t, 0, sizeof (t)); + t.copyString(sz); + + /* tell lemon to parse the token */ + parse (parser, token, t, this); +} + +int lex_t::line ()const +{ + return yyget_lineno (scanner); +} + +const char* lex_t::filename ()const +{ + return m_fileName; +} + +void parse (MappedFile* mf) +{ + void* parser = bbparseAlloc (malloc); + yyscan_t scanner; + lex_t lex; + + yylex_init (&scanner); + + lex.parser = parser; + lex.scanner = scanner; + lex.mf = mf; + lex.rgbInput = mf->m_rgb; + lex.cbInput = mf->m_cb; + lex.parse = bbparse; + yyset_extra (&lex, scanner); + + + int result = yylex (scanner); + + lex.accept (0); + bbparseTrace (NULL, NULL); + + if (result != T_EOF) + WARNING ("premature end of file\n"); + + yylex_destroy (scanner); + bbparseFree (parser, free); +} diff --git a/bitbake/lib/bb/parse/parse_c/bitbakeparser.py b/bitbake/lib/bb/parse/parse_c/bitbakeparser.py new file mode 100644 index 0000000..ed7b13e --- /dev/null +++ b/bitbake/lib/bb/parse/parse_c/bitbakeparser.py @@ -0,0 +1,133 @@ +""" + +BitBake C Parser Python Code + +Copyright (C) 2005 Holger Hans Peter Freyther + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +__version__ = "0xdeadbeef" + +class CParser: + """ + The C-based Parser for Bitbake + """ + def __init__(self, data, type): + """ + Constructor + """ + self._data = data + + def _syntax_error(self, file, line): + """ + lemon/flex reports an syntax error to us and we will + raise an exception + """ + pass + + def _export(self, data): + """ + EXPORT VAR = "MOO" + we will now export VAR + """ + pass + + def _assign(self, key, value): + """ + VAR = "MOO" + we will assign moo to VAR + """ + pass + + def _assign(self, key, value): + """ + """ + pass + + def _append(self, key, value): + """ + VAR += "MOO" + we will append " MOO" to var + """ + pass + + def _prepend(self, key, value): + """ + VAR =+ "MOO" + we will prepend "MOO " to var + """ + pass + + def _immediate(self, key, value): + """ + VAR := "MOO ${CVSDATE}" + we will assign immediately and expand vars + """ + pass + + def _conditional(self, key, value): + """ + """ + pass + + def _add_task(self, task, before = None, after = None): + """ + """ + pass + + def _include(self, file): + """ + """ + pass + + def _inherit(self, file): + """ + """ + pass + + def _shell_procedure(self, name, body): + """ + """ + pass + + def _python_procedure(self, name, body): + """ + """ + pass + + def _fakeroot_procedure(self, name, body): + """ + """ + pass + + def _def_procedure(self, a, b, c): + """ + """ + pass + + def _export_func(self, name): + """ + """ + pass + + def _add_handler(self, handler): + """ + """ + pass diff --git a/bitbake/lib/bb/parse/parse_c/bitbakeparser.y b/bitbake/lib/bb/parse/parse_c/bitbakeparser.y new file mode 100644 index 0000000..4bc81a9 --- /dev/null +++ b/bitbake/lib/bb/parse/parse_c/bitbakeparser.y @@ -0,0 +1,161 @@ +/* bbp.lemon + + written by Marc Singer + 6 January 2005 + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + USA. + + DESCRIPTION + ----------- + + lemon parser specification file for a BitBake input file parser. + + Most of the interesting shenanigans are done in the lexer. The + BitBake grammar is not regular. In order to emit tokens that + the parser can properly interpret in LALR fashion, the lexer + manages the interpretation state. This is why there are ISYMBOLs, + SYMBOLS, and TSYMBOLS. + + This parser was developed by reading the limited available + documentation for BitBake and by analyzing the available BB files. + There is no assertion of correctness to be made about this parser. + +*/ + +%token_type {token_t} +%name bbparse +%token_prefix T_ +%extra_argument {lex_t* lex} + +%include { +#include "token.h" +} + + +%token_destructor { $$.release_this (); } + +%syntax_error { printf ("%s:%d: syntax error\n", + lex->filename (), lex->line ()); } + +program ::= statements. + +statements ::= statements statement. +statements ::= . + +variable(r) ::= SYMBOL(s). + { r.assignString( s.string() ); + s.assignString( 0 ); + s.release_this(); } +variable(r) ::= VARIABLE(v). + { + r.assignString( v.string() ); + v.assignString( 0 ); + v.release_this(); } + +statement ::= EXPORT variable(s) OP_ASSIGN STRING(v). + { e_assign( s.string(), v.string() ); + e_export( s.string() ); + s.release_this(); v.release_this(); } +statement ::= EXPORT variable(s) OP_IMMEDIATE STRING(v). + { e_immediate (s.string(), v.string() ); + e_export( s.string() ); + s.release_this(); v.release_this(); } +statement ::= EXPORT variable(s) OP_COND STRING(v). + { e_cond( s.string(), v.string() ); + s.release_this(); v.release_this(); } + +statement ::= variable(s) OP_ASSIGN STRING(v). + { e_assign( s.string(), v.string() ); + s.release_this(); v.release_this(); } +statement ::= variable(s) OP_PREPEND STRING(v). + { e_prepend( s.string(), v.string() ); + s.release_this(); v.release_this(); } +statement ::= variable(s) OP_APPEND STRING(v). + { e_append( s.string() , v.string() ); + s.release_this(); v.release_this(); } +statement ::= variable(s) OP_IMMEDIATE STRING(v). + { e_immediate( s.string(), v.string() ); + s.release_this(); v.release_this(); } +statement ::= variable(s) OP_COND STRING(v). + { e_cond( s.string(), v.string() ); + s.release_this(); v.release_this(); } + +task ::= TSYMBOL(t) BEFORE TSYMBOL(b) AFTER TSYMBOL(a). + { e_addtask( t.string(), b.string(), a.string() ); + t.release_this(); b.release_this(); a.release_this(); } +task ::= TSYMBOL(t) AFTER TSYMBOL(a) BEFORE TSYMBOL(b). + { e_addtask( t.string(), b.string(), a.string()); + t.release_this(); a.release_this(); b.release_this(); } +task ::= TSYMBOL(t). + { e_addtask( t.string(), NULL, NULL); + t.release_this();} +task ::= TSYMBOL(t) BEFORE TSYMBOL(b). + { e_addtask( t.string(), b.string(), NULL); + t.release_this(); b.release_this(); } +task ::= TSYMBOL(t) AFTER TSYMBOL(a). + { e_addtask( t.string(), NULL, a.string()); + t.release_this(); a.release_this(); } +tasks ::= tasks task. +tasks ::= task. +statement ::= ADDTASK tasks. + +statement ::= ADDHANDLER SYMBOL(s). + { e_addhandler( s.string()); s.release_this (); } + +func ::= FSYMBOL(f). { e_export_func(f.string()); f.release_this(); } +funcs ::= funcs func. +funcs ::= func. +statement ::= EXPORT_FUNC funcs. + +inherit ::= ISYMBOL(i). { e_inherit(i.string() ); i.release_this (); } +inherits ::= inherits inherit. +inherits ::= inherit. +statement ::= INHERIT inherits. + +statement ::= INCLUDE ISYMBOL(i). + { e_include(i.string() ); i.release_this(); } + +proc_body(r) ::= proc_body(l) PROC_BODY(b). + { /* concatenate body lines */ + r.assignString( token_t::concatString(l.string(), b.string()) ); + l.release_this (); + b.release_this (); + } +proc_body(b) ::= . { b.assignString(0); } +statement ::= variable(p) PROC_OPEN proc_body(b) PROC_CLOSE. + { e_proc( p.string(), b.string() ); + p.release_this(); b.release_this(); } +statement ::= PYTHON SYMBOL(p) PROC_OPEN proc_body(b) PROC_CLOSE. + { e_proc_python (p.string(), b.string() ); + p.release_this(); b.release_this(); } +statement ::= PYTHON PROC_OPEN proc_body(b) PROC_CLOSE. + { e_proc_python( NULL, b.string()); + b.release_this (); } + +statement ::= FAKEROOT SYMBOL(p) PROC_OPEN proc_body(b) PROC_CLOSE. + { e_proc_fakeroot(p.string(), b.string() ); + p.release_this (); b.release_this (); } + +def_body(r) ::= def_body(l) DEF_BODY(b). + { /* concatenate body lines */ + r.assignString( token_t::concatString(l.string(), b.string()); + l.release_this (); b.release_this (); + } +def_body(b) ::= . { b.sz = 0; } +statement ::= SYMBOL(p) DEF_ARGS(a) def_body(b). + { e_def( p.string(), a.string(), b.string()); + p.release_this(); a.release_this(); b.release_this(); } + diff --git a/bitbake/lib/bb/parse/parse_c/lexer.h b/bitbake/lib/bb/parse/parse_c/lexer.h new file mode 100644 index 0000000..1edf72d --- /dev/null +++ b/bitbake/lib/bb/parse/parse_c/lexer.h @@ -0,0 +1,41 @@ +/* +Copyright (C) 2005 Holger Hans Peter Freyther + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +*/ + +#ifndef LEXER_H +#define LEXER_H + +struct lex_t { + void *parser; + void *scanner; + void* (*parse)(void*, int, token_t, lex_t*); + + void accept(int token, const char* string = 0); + void input(char *buf, int *result, int_max_size); + int line()const; + const char* filename()const; +private: + const char* m_fileName; +}; + + +#endif diff --git a/bitbake/lib/bb/parse/parse_c/token.h b/bitbake/lib/bb/parse/parse_c/token.h new file mode 100644 index 0000000..2351fda --- /dev/null +++ b/bitbake/lib/bb/parse/parse_c/token.h @@ -0,0 +1,83 @@ +/* +Copyright (C) 2005 Holger Hans Peter Freyther + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +*/ + +#ifndef TOKEN_H +#define TOKEN_H + +#define PURE_METHOD + +struct token_t { + const char* string()const PURE_METHOD; + + static char* concatString(const char* l, const char* r); + void assignString(const char* str); + void copyString(const char* str); + + void release_this(); + +private: + char *m_string; + size_t m_stringLen; +}; + +inline const char* token_t::string()const +{ + return m_string; +} + +/* + * append str to the current string + */ +inline char* token_t::concatString(const char* l, const char* r) +{ + size_t cb = (l ? strlen (l) : 0) + strlen (r) + 1; + r_sz = new char[cb]; + *r_sz = 0; + if (l) strcat (r_sz, l); + strcat (r_sz, r); + + return r_sz; +} + +inline void token_t::assignString(const char* str) +{ + m_string = str; + m_stringLen = str ? strlen(str) : 0; +} + +inline void token_t::copyString(const char* str) +{ + if( str ) { + m_stringLen = strlen(str); + m_string = new char[m_stringLen+1]; + strcpy(m_string, str) + } +} + +inline void token_t::release_this() +{ + delete m_string; + m_string = 0; +} + +#endif diff --git a/bitbake/lib/bb/parse/parse_py/.gitignore b/bitbake/lib/bb/parse/parse_py/.gitignore new file mode 100644 index 0000000..90ec22b --- /dev/null +++ b/bitbake/lib/bb/parse/parse_py/.gitignore @@ -0,0 +1 @@ +.svn diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py new file mode 100644 index 0000000..fac3e85 --- /dev/null +++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +"""class for handling .bb files + + Reads a .bb file and obtains its metadata + + Copyright (C) 2003, 2004 Chris Larson + Copyright (C) 2003, 2004 Phil Blundell + + This program is free software; you can redistribute it and/or modify it under + the terms of the GNU General Public License as published by the Free Software + Foundation; either version 2 of the License, or (at your option) any later + version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + this program; if not, write to the Free Software Foundation, Inc., 59 Temple + Place, Suite 330, Boston, MA 02111-1307 USA.""" + +import re, bb, os, sys +import bb.fetch, bb.build +from bb import debug, data, fetch, fatal + +from ConfHandler import include, localpath, obtain, init +from bb.parse import ParseError + +__func_start_regexp__ = re.compile( r"(((?Ppython)|(?Pfakeroot))\s*)*(?P[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" ) +__inherit_regexp__ = re.compile( r"inherit\s+(.+)" ) +__export_func_regexp__ = re.compile( r"EXPORT_FUNCTIONS\s+(.+)" ) +__addtask_regexp__ = re.compile("addtask\s+(?P\w+)\s*((before\s*(?P((.*(?=after))|(.*))))|(after\s*(?P((.*(?=before))|(.*)))))*") +__addhandler_regexp__ = re.compile( r"addhandler\s+(.+)" ) +__def_regexp__ = re.compile( r"def\s+(\w+).*:" ) +__python_func_regexp__ = re.compile( r"(\s+.*)|(^$)" ) +__word__ = re.compile(r"\S+") + +__infunc__ = "" +__inpython__ = False +__body__ = [] +__bbpath_found__ = 0 +__classname__ = "" +classes = [ None, ] + +def supports(fn, d): + localfn = localpath(fn, d) + return localfn[-3:] == ".bb" or localfn[-8:] == ".bbclass" or localfn[-4:] == ".inc" + +def inherit(files, d): + __inherit_cache = data.getVar('__inherit_cache', d) or "" + fn = "" + lineno = 0 + for f in files: + file = data.expand(f, d) + if file[0] != "/" and file[-8:] != ".bbclass": + file = os.path.join('classes', '%s.bbclass' % file) + + if not file in __inherit_cache.split(): + debug(2, "BB %s:%d: inheriting %s" % (fn, lineno, file)) + __inherit_cache += " %s" % file + include(fn, file, d) + data.setVar('__inherit_cache', __inherit_cache, d) + + +def handle(fn, d, include = 0): + global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __infunc__, __body__, __bbpath_found__, __residue__ + __body__ = [] + __bbpath_found__ = 0 + __infunc__ = "" + __classname__ = "" + __residue__ = [] + + if include == 0: + debug(2, "BB " + fn + ": handle(data)") + else: + debug(2, "BB " + fn + ": handle(data, include)") + + (root, ext) = os.path.splitext(os.path.basename(fn)) + init(d) + + if ext == ".bbclass": + __classname__ = root + classes.append(__classname__) + + if include != 0: + oldfile = data.getVar('FILE', d) + else: + oldfile = None + + fn = obtain(fn, d) + bbpath = (data.getVar('BBPATH', d, 1) or '').split(':') + if not os.path.isabs(fn): + f = None + for p in bbpath: + p = data.expand(p, d) + j = os.path.join(p, fn) + if os.access(j, os.R_OK): + abs_fn = j + f = open(j, 'r') + break + if f is None: + raise IOError("file not found") + else: + f = open(fn,'r') + abs_fn = fn + + if ext != ".bbclass": + bbpath.insert(0, os.path.dirname(abs_fn)) + data.setVar('BBPATH', ":".join(bbpath), d) + + if include: + bb.parse.mark_dependency(d, abs_fn) + + if ext != ".bbclass": + data.setVar('FILE', fn, d) + i = (data.getVar("INHERIT", d, 1) or "").split() + if not "base" in i and __classname__ != "base": + i[0:0] = ["base"] + inherit(i, d) + + lineno = 0 + while 1: + lineno = lineno + 1 + s = f.readline() + if not s: break + s = s.rstrip() + feeder(lineno, s, fn, d) + if __inpython__: + # add a blank line to close out any python definition + feeder(lineno + 1, "", fn, d) + if ext == ".bbclass": + classes.remove(__classname__) + else: + if include == 0: + data.expandKeys(d) + data.update_data(d) + anonqueue = data.getVar("__anonqueue", d, 1) or [] + for anon in anonqueue: + data.setVar("__anonfunc", anon["content"], d) + data.setVarFlags("__anonfunc", anon["flags"], d) + from bb import build + try: + t = data.getVar('T', d) + data.setVar('T', '${TMPDIR}/', d) + build.exec_func("__anonfunc", d) + data.delVar('T', d) + if t: + data.setVar('T', t, d) + except Exception, e: + bb.debug(1, "executing anonymous function: %s" % e) + raise + data.delVar("__anonqueue", d) + data.delVar("__anonfunc", d) + set_additional_vars(fn, d, include) + data.update_data(d) + + for var in data.keys(d): + if data.getVarFlag(var, 'handler', d): + bb.event.register(data.getVar(var, d)) + continue + + if not data.getVarFlag(var, 'task', d): + continue + + deps = data.getVarFlag(var, 'deps', d) or [] + postdeps = data.getVarFlag(var, 'postdeps', d) or [] + bb.build.add_task(var, deps, d) + for p in postdeps: + pdeps = data.getVarFlag(p, 'deps', d) or [] + pdeps.append(var) + data.setVarFlag(p, 'deps', pdeps, d) + bb.build.add_task(p, pdeps, d) + bbpath.pop(0) + if oldfile: + bb.data.setVar("FILE", oldfile, d) + return d + +def feeder(lineno, s, fn, d): + global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__,__infunc__, __body__, __bbpath_found__, classes, bb, __residue__ + if __infunc__: + if s == '}': + __body__.append('') + data.setVar(__infunc__, '\n'.join(__body__), d) + data.setVarFlag(__infunc__, "func", 1, d) + if __infunc__ == "__anonymous": + anonqueue = bb.data.getVar("__anonqueue", d) or [] + anonitem = {} + anonitem["content"] = bb.data.getVar("__anonymous", d) + anonitem["flags"] = bb.data.getVarFlags("__anonymous", d) + anonqueue.append(anonitem) + bb.data.setVar("__anonqueue", anonqueue, d) + bb.data.delVarFlags("__anonymous", d) + bb.data.delVar("__anonymous", d) + __infunc__ = "" + __body__ = [] + else: + __body__.append(s) + return + + if __inpython__: + m = __python_func_regexp__.match(s) + if m: + __body__.append(s) + return + else: + text = '\n'.join(__body__) + comp = compile(text, "", "exec") + exec comp in __builtins__ + __body__ = [] + __inpython__ = False + funcs = data.getVar('__functions__', d) or "" + data.setVar('__functions__', "%s\n%s" % (funcs, text), d) +# fall through + + if s == '' or s[0] == '#': return # skip comments and empty lines + + if s[-1] == '\\': + __residue__.append(s[:-1]) + return + + s = "".join(__residue__) + s + __residue__ = [] + + m = __func_start_regexp__.match(s) + if m: + __infunc__ = m.group("func") or "__anonymous" + key = __infunc__ + if data.getVar(key, d): +# clean up old version of this piece of metadata, as its +# flags could cause problems + data.setVarFlag(key, 'python', None, d) + data.setVarFlag(key, 'fakeroot', None, d) + if m.group("py") is not None: + data.setVarFlag(key, "python", "1", d) + else: + data.delVarFlag(key, "python", d) + if m.group("fr") is not None: + data.setVarFlag(key, "fakeroot", "1", d) + else: + data.delVarFlag(key, "fakeroot", d) + return + + m = __def_regexp__.match(s) + if m: + __body__.append(s) + __inpython__ = True + return + + m = __export_func_regexp__.match(s) + if m: + fns = m.group(1) + n = __word__.findall(fns) + for f in n: + allvars = [] + allvars.append(f) + allvars.append(classes[-1] + "_" + f) + + vars = [[ allvars[0], allvars[1] ]] + if len(classes) > 1 and classes[-2] is not None: + allvars.append(classes[-2] + "_" + f) + vars = [] + vars.append([allvars[2], allvars[1]]) + vars.append([allvars[0], allvars[2]]) + + for (var, calledvar) in vars: + if data.getVar(var, d) and not data.getVarFlag(var, 'export_func', d): + continue + + if data.getVar(var, d): + data.setVarFlag(var, 'python', None, d) + data.setVarFlag(var, 'func', None, d) + + for flag in [ "func", "python" ]: + if data.getVarFlag(calledvar, flag, d): + data.setVarFlag(var, flag, data.getVarFlag(calledvar, flag, d), d) + for flag in [ "dirs" ]: + if data.getVarFlag(var, flag, d): + data.setVarFlag(calledvar, flag, data.getVarFlag(var, flag, d), d) + + if data.getVarFlag(calledvar, "python", d): + data.setVar(var, "\tbb.build.exec_func('" + calledvar + "', d)\n", d) + else: + data.setVar(var, "\t" + calledvar + "\n", d) + data.setVarFlag(var, 'export_func', '1', d) + + return + + m = __addtask_regexp__.match(s) + if m: + func = m.group("func") + before = m.group("before") + after = m.group("after") + if func is None: + return + var = "do_" + func + + data.setVarFlag(var, "task", 1, d) + + if after is not None: +# set up deps for function + data.setVarFlag(var, "deps", after.split(), d) + if before is not None: +# set up things that depend on this func + data.setVarFlag(var, "postdeps", before.split(), d) + return + + m = __addhandler_regexp__.match(s) + if m: + fns = m.group(1) + hs = __word__.findall(fns) + for h in hs: + data.setVarFlag(h, "handler", 1, d) + return + + m = __inherit_regexp__.match(s) + if m: + + files = m.group(1) + n = __word__.findall(files) + inherit(n, d) + return + + from bb.parse import ConfHandler + return ConfHandler.feeder(lineno, s, fn, d) + +__pkgsplit_cache__={} +def vars_from_file(mypkg, d): + if not mypkg: + return (None, None, None) + if mypkg in __pkgsplit_cache__: + return __pkgsplit_cache__[mypkg] + + myfile = os.path.splitext(os.path.basename(mypkg)) + parts = myfile[0].split('_') + __pkgsplit_cache__[mypkg] = parts + exp = 3 - len(parts) + tmplist = [] + while exp != 0: + exp -= 1 + tmplist.append(None) + parts.extend(tmplist) + return parts + +def set_additional_vars(file, d, include): + """Deduce rest of variables, e.g. ${A} out of ${SRC_URI}""" + + debug(2,"BB %s: set_additional_vars" % file) + + src_uri = data.getVar('SRC_URI', d) + if not src_uri: + return + src_uri = data.expand(src_uri, d) + + a = data.getVar('A', d) + if a: + a = data.expand(a, d).split() + else: + a = [] + + from bb import fetch + try: + fetch.init(src_uri.split(), d) + except fetch.NoMethodError: + pass + except bb.MalformedUrl,e: + raise ParseError("Unable to generate local paths for SRC_URI due to malformed uri: %s" % e) + + a += fetch.localpaths(d) + del fetch + data.setVar('A', " ".join(a), d) + + +# Add us to the handlers list +from bb.parse import handlers +handlers.append({'supports': supports, 'handle': handle, 'init': init}) +del handlers diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py new file mode 100644 index 0000000..ecae5d1 --- /dev/null +++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +"""class for handling configuration data files + + Reads a .conf file and obtains its metadata + + Copyright (C) 2003, 2004 Chris Larson + Copyright (C) 2003, 2004 Phil Blundell + + This program is free software; you can redistribute it and/or modify it under + the terms of the GNU General Public License as published by the Free Software + Foundation; either version 2 of the License, or (at your option) any later + version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + this program; if not, write to the Free Software Foundation, Inc., 59 Temple + Place, Suite 330, Boston, MA 02111-1307 USA.""" + +import re, bb.data, os, sys +from bb import debug, fatal +from bb.parse import ParseError + +#__config_regexp__ = re.compile( r"(?Pexport\s*)?(?P[a-zA-Z0-9\-_+.${}]+)\s*(?P:)?(?P\?)?=\s*(?P['\"]?)(?P.*)(?P=apo)$") +__config_regexp__ = re.compile( r"(?Pexport\s*)?(?P[a-zA-Z0-9\-_+.${}/]+)(\[(?P[a-zA-Z0-9\-_+.]+)\])?\s*((?P:=)|(?P\?=)|(?P\+=)|(?P=\+)|(?P=\.)|(?P\.=)|=)\s*(?P['\"]?)(?P.*)(?P=apo)$") +__include_regexp__ = re.compile( r"include\s+(.+)" ) + +def init(data): + if not bb.data.getVar('TOPDIR', data): + bb.data.setVar('TOPDIR', os.getcwd(), data) + if not bb.data.getVar('BBPATH', data): + bb.data.setVar('BBPATH', os.path.join(sys.prefix, 'share', 'bitbake'), data) + +def supports(fn, d): + return localpath(fn, d)[-5:] == ".conf" + +def localpath(fn, d): + if os.path.exists(fn): + return fn + + localfn = None + try: + localfn = bb.fetch.localpath(fn, d) + except bb.MalformedUrl: + pass + + if not localfn: + localfn = fn + return localfn + +def obtain(fn, data = bb.data.init()): + import sys, bb + fn = bb.data.expand(fn, data) + localfn = bb.data.expand(localpath(fn, data), data) + + if localfn != fn: + dldir = bb.data.getVar('DL_DIR', data, 1) + if not dldir: + debug(1, "obtain: DL_DIR not defined") + return localfn + bb.mkdirhier(dldir) + try: + bb.fetch.init([fn]) + except bb.fetch.NoMethodError: + (type, value, traceback) = sys.exc_info() + debug(1, "obtain: no method: %s" % value) + return localfn + + try: + bb.fetch.go(data) + except bb.fetch.MissingParameterError: + (type, value, traceback) = sys.exc_info() + debug(1, "obtain: missing parameters: %s" % value) + return localfn + except bb.fetch.FetchError: + (type, value, traceback) = sys.exc_info() + debug(1, "obtain: failed: %s" % value) + return localfn + return localfn + + +def include(oldfn, fn, data = bb.data.init()): + if oldfn == fn: # prevent infinate recursion + return None + + import bb + fn = bb.data.expand(fn, data) + oldfn = bb.data.expand(oldfn, data) + + from bb.parse import handle + try: + ret = handle(fn, data, 1) + except IOError: + debug(2, "CONF file '%s' not found" % fn) + +def handle(fn, data = bb.data.init(), include = 0): + if include: + inc_string = "including" + else: + inc_string = "reading" + init(data) + + if include == 0: + bb.data.inheritFromOS(data) + oldfile = None + else: + oldfile = bb.data.getVar('FILE', data) + + fn = obtain(fn, data) + bbpath = [] + if not os.path.isabs(fn): + f = None + vbbpath = bb.data.getVar("BBPATH", data) + if vbbpath: + bbpath += vbbpath.split(":") + for p in bbpath: + currname = os.path.join(bb.data.expand(p, data), fn) + if os.access(currname, os.R_OK): + f = open(currname, 'r') + abs_fn = currname + debug(1, "CONF %s %s" % (inc_string, currname)) + break + if f is None: + raise IOError("file '%s' not found" % fn) + else: + f = open(fn,'r') + debug(1, "CONF %s %s" % (inc_string,fn)) + abs_fn = fn + + if include: + bb.parse.mark_dependency(data, abs_fn) + + lineno = 0 + bb.data.setVar('FILE', fn, data) + while 1: + lineno = lineno + 1 + s = f.readline() + if not s: break + w = s.strip() + if not w: continue # skip empty lines + s = s.rstrip() + if s[0] == '#': continue # skip comments + while s[-1] == '\\': + s2 = f.readline()[:-1].strip() + lineno = lineno + 1 + s = s[:-1] + s2 + feeder(lineno, s, fn, data) + + if oldfile: + bb.data.setVar('FILE', oldfile, data) + return data + +def feeder(lineno, s, fn, data = bb.data.init()): + m = __config_regexp__.match(s) + if m: + groupd = m.groupdict() + key = groupd["var"] + if "exp" in groupd and groupd["exp"] != None: + bb.data.setVarFlag(key, "export", 1, data) + if "ques" in groupd and groupd["ques"] != None: + val = bb.data.getVar(key, data) + if val == None: + val = groupd["value"] + elif "colon" in groupd and groupd["colon"] != None: + val = bb.data.expand(groupd["value"], data) + elif "append" in groupd and groupd["append"] != None: + val = "%s %s" % ((bb.data.getVar(key, data) or ""), groupd["value"]) + elif "prepend" in groupd and groupd["prepend"] != None: + val = "%s %s" % (groupd["value"], (bb.data.getVar(key, data) or "")) + elif "postdot" in groupd and groupd["postdot"] != None: + val = "%s%s" % ((bb.data.getVar(key, data) or ""), groupd["value"]) + elif "predot" in groupd and groupd["predot"] != None: + val = "%s%s" % (groupd["value"], (bb.data.getVar(key, data) or "")) + else: + val = groupd["value"] + if 'flag' in groupd and groupd['flag'] != None: +# bb.note("setVarFlag(%s, %s, %s, data)" % (key, groupd['flag'], val)) + bb.data.setVarFlag(key, groupd['flag'], val, data) + else: + bb.data.setVar(key, val, data) + return + + m = __include_regexp__.match(s) + if m: + s = bb.data.expand(m.group(1), data) +# debug(2, "CONF %s:%d: including %s" % (fn, lineno, s)) + include(fn, s, data) + return + + raise ParseError("%s:%d: unparsed line: '%s'" % (fn, lineno, s)); + +# Add us to the handlers list +from bb.parse import handlers +handlers.append({'supports': supports, 'handle': handle, 'init': init}) +del handlers diff --git a/bitbake/lib/bb/parse/parse_py/__init__.py b/bitbake/lib/bb/parse/parse_py/__init__.py new file mode 100644 index 0000000..6a2ce40 --- /dev/null +++ b/bitbake/lib/bb/parse/parse_py/__init__.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake Parsers + +File parsers for the BitBake build tools. + +Copyright (C) 2003, 2004 Chris Larson +Copyright (C) 2003, 2004 Phil Blundell + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +Based on functions from the base bb module, Copyright 2003 Holger Schurig +""" +__version__ = '1.0' + +__all__ = [ 'ConfHandler', 'BBHandler'] + +import ConfHandler +import BBHandler diff --git a/bitbake/lib/bb/shell.py b/bitbake/lib/bb/shell.py new file mode 100644 index 0000000..24406bb --- /dev/null +++ b/bitbake/lib/bb/shell.py @@ -0,0 +1,808 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +########################################################################## +# +# Copyright (C) 2005 Michael 'Mickey' Lauer , Vanille Media +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# +########################################################################## +# +# Thanks to: +# * Holger Freyther +# * Justin Patrin +# +########################################################################## + +""" +BitBake Shell + +IDEAS: + * list defined tasks per package + * list classes + * toggle force + * command to reparse just one (or more) bbfile(s) + * automatic check if reparsing is necessary (inotify?) + * frontend for bb file manipulation + * more shell-like features: + - output control, i.e. pipe output into grep, sort, etc. + - job control, i.e. bring running commands into background and foreground + * start parsing in background right after startup + * ncurses interface + +PROBLEMS: + * force doesn't always work + * readline completion for commands with more than one parameters + +""" + +########################################################################## +# Import and setup global variables +########################################################################## + +try: + set +except NameError: + from sets import Set as set +import sys, os, imp, readline, socket, httplib, urllib, commands, popen2, copy, shlex, Queue, fnmatch +imp.load_source( "bitbake", os.path.dirname( sys.argv[0] )+"/bitbake" ) +from bb import data, parse, build, fatal + +__version__ = "0.5.3" +__credits__ = """BitBake Shell Version %s (C) 2005 Michael 'Mickey' Lauer +Type 'help' for more information, press CTRL-D to exit.""" % __version__ + +cmds = {} +leave_mainloop = False +last_exception = None +cooker = None +parsed = False +initdata = None +debug = os.environ.get( "BBSHELL_DEBUG", "" ) + +########################################################################## +# Class BitBakeShellCommands +########################################################################## + +class BitBakeShellCommands: + """This class contains the valid commands for the shell""" + + def __init__( self, shell ): + """Register all the commands""" + self._shell = shell + for attr in BitBakeShellCommands.__dict__: + if not attr.startswith( "_" ): + if attr.endswith( "_" ): + command = attr[:-1].lower() + else: + command = attr[:].lower() + method = getattr( BitBakeShellCommands, attr ) + debugOut( "registering command '%s'" % command ) + # scan number of arguments + usage = getattr( method, "usage", "" ) + if usage != "<...>": + numArgs = len( usage.split() ) + else: + numArgs = -1 + shell.registerCommand( command, method, numArgs, "%s %s" % ( command, usage ), method.__doc__ ) + + def _checkParsed( self ): + if not parsed: + print "SHELL: This command needs to parse bbfiles..." + self.parse( None ) + + def _findProvider( self, item ): + self._checkParsed() + preferred = data.getVar( "PREFERRED_PROVIDER_%s" % item, cooker.configuration.data, 1 ) + if not preferred: preferred = item + try: + lv, lf, pv, pf = cooker.findBestProvider( preferred ) + except KeyError: + if item in cooker.status.providers: + pf = cooker.status.providers[item][0] + else: + pf = None + return pf + + def alias( self, params ): + """Register a new name for a command""" + new, old = params + if not old in cmds: + print "ERROR: Command '%s' not known" % old + else: + cmds[new] = cmds[old] + print "OK" + alias.usage = " " + + def buffer( self, params ): + """Dump specified output buffer""" + index = params[0] + print self._shell.myout.buffer( int( index ) ) + buffer.usage = "" + + def buffers( self, params ): + """Show the available output buffers""" + commands = self._shell.myout.bufferedCommands() + if not commands: + print "SHELL: No buffered commands available yet. Start doing something." + else: + print "="*35, "Available Output Buffers", "="*27 + for index, cmd in enumerate( commands ): + print "| %s %s" % ( str( index ).ljust( 3 ), cmd ) + print "="*88 + + def build( self, params, cmd = "build" ): + """Build a providee""" + globexpr = params[0] + self._checkParsed() + names = globfilter( cooker.status.pkg_pn.keys(), globexpr ) + if len( names ) == 0: names = [ globexpr ] + print "SHELL: Building %s" % ' '.join( names ) + + oldcmd = cooker.configuration.cmd + cooker.configuration.cmd = cmd + cooker.build_cache = [] + cooker.build_cache_fail = [] + + for name in names: + try: + cooker.buildProvider( name ) + except build.EventException, e: + print "ERROR: Couldn't build '%s'" % name + global last_exception + last_exception = e + break + + cooker.configuration.cmd = oldcmd + + build.usage = "" + + def clean( self, params ): + """Clean a providee""" + self.build( params, "clean" ) + clean.usage = "" + + def compile( self, params ): + """Execute 'compile' on a providee""" + self.build( params, "compile" ) + compile.usage = "" + + def configure( self, params ): + """Execute 'configure' on a providee""" + self.build( params, "configure" ) + configure.usage = "" + + def edit( self, params ): + """Call $EDITOR on a providee""" + name = params[0] + bbfile = self._findProvider( name ) + if bbfile is not None: + os.system( "%s %s" % ( os.environ.get( "EDITOR", "vi" ), bbfile ) ) + else: + print "ERROR: Nothing provides '%s'" % name + edit.usage = "" + + def environment( self, params ): + """Dump out the outer BitBake environment (see bbread)""" + data.emit_env(sys.__stdout__, cooker.configuration.data, True) + + def exit_( self, params ): + """Leave the BitBake Shell""" + debugOut( "setting leave_mainloop to true" ) + global leave_mainloop + leave_mainloop = True + + def fetch( self, params ): + """Fetch a providee""" + self.build( params, "fetch" ) + fetch.usage = "" + + def fileBuild( self, params, cmd = "build" ): + """Parse and build a .bb file""" + name = params[0] + bf = completeFilePath( name ) + print "SHELL: Calling '%s' on '%s'" % ( cmd, bf ) + + oldcmd = cooker.configuration.cmd + cooker.configuration.cmd = cmd + cooker.build_cache = [] + cooker.build_cache_fail = [] + + thisdata = copy.deepcopy( initdata ) + # Caution: parse.handle modifies thisdata, hence it would + # lead to pollution cooker.configuration.data, which is + # why we use it on a safe copy we obtained from cooker right after + # parsing the initial *.conf files + try: + bbfile_data = parse.handle( bf, thisdata ) + except parse.ParseError: + print "ERROR: Unable to open or parse '%s'" % bf + else: + item = data.getVar('PN', bbfile_data, 1) + data.setVar( "_task_cache", [], bbfile_data ) # force + try: + cooker.tryBuildPackage( os.path.abspath( bf ), item, bbfile_data ) + except build.EventException, e: + print "ERROR: Couldn't build '%s'" % name + global last_exception + last_exception = e + + cooker.configuration.cmd = oldcmd + fileBuild.usage = "" + + def fileClean( self, params ): + """Clean a .bb file""" + self.fileBuild( params, "clean" ) + fileClean.usage = "" + + def fileEdit( self, params ): + """Call $EDITOR on a .bb file""" + name = params[0] + os.system( "%s %s" % ( os.environ.get( "EDITOR", "vi" ), completeFilePath( name ) ) ) + fileEdit.usage = "" + + def fileRebuild( self, params ): + """Rebuild (clean & build) a .bb file""" + self.fileClean( params ) + self.fileBuild( params ) + fileRebuild.usage = "" + + def fileReparse( self, params ): + """(re)Parse a bb file""" + bbfile = params[0] + print "SHELL: Parsing '%s'" % bbfile + parse.update_mtime( bbfile ) + bb_data, fromCache = cooker.load_bbfile( bbfile ) + cooker.pkgdata[bbfile] = bb_data + if fromCache: + print "SHELL: File has not been updated, not reparsing" + else: + print "SHELL: Parsed" + fileReparse.usage = "" + + def force( self, params ): + """Toggle force task execution flag (see bitbake -f)""" + cooker.configuration.force = not cooker.configuration.force + print "SHELL: Force Flag is now '%s'" % repr( cooker.configuration.force ) + + def help( self, params ): + """Show a comprehensive list of commands and their purpose""" + print "="*30, "Available Commands", "="*30 + allcmds = cmds.keys() + allcmds.sort() + for cmd in allcmds: + function,numparams,usage,helptext = cmds[cmd] + print "| %s | %s" % (usage.ljust(30), helptext) + print "="*78 + + def lastError( self, params ): + """Show the reason or log that was produced by the last BitBake event exception""" + if last_exception is None: + print "SHELL: No Errors yet (Phew)..." + else: + reason, event = last_exception.args + print "SHELL: Reason for the last error: '%s'" % reason + if ':' in reason: + msg, filename = reason.split( ':' ) + filename = filename.strip() + print "SHELL: Dumping log file for last error:" + try: + print open( filename ).read() + except IOError: + print "ERROR: Couldn't open '%s'" % filename + + def match( self, params ): + """Dump all files or providers matching a glob expression""" + what, globexpr = params + if what == "files": + self._checkParsed() + for key in globfilter( cooker.pkgdata.keys(), globexpr ): print key + elif what == "providers": + self._checkParsed() + for key in globfilter( cooker.status.pkg_pn.keys(), globexpr ): print key + else: + print "Usage: match %s" % self.print_.usage + match.usage = " " + + def new( self, params ): + """Create a new .bb file and open the editor""" + dirname, filename = params + packages = '/'.join( data.getVar( "BBFILES", cooker.configuration.data, 1 ).split('/')[:-2] ) + fulldirname = "%s/%s" % ( packages, dirname ) + + if not os.path.exists( fulldirname ): + print "SHELL: Creating '%s'" % fulldirname + os.mkdir( fulldirname ) + if os.path.exists( fulldirname ) and os.path.isdir( fulldirname ): + if os.path.exists( "%s/%s" % ( fulldirname, filename ) ): + print "SHELL: ERROR: %s/%s already exists" % ( fulldirname, filename ) + return False + print "SHELL: Creating '%s/%s'" % ( fulldirname, filename ) + newpackage = open( "%s/%s" % ( fulldirname, filename ), "w" ) + print >>newpackage,"""DESCRIPTION = "" +SECTION = "" +AUTHOR = "" +HOMEPAGE = "" +MAINTAINER = "" +LICENSE = "GPL" +PR = "r0" + +SRC_URI = "" + +#inherit base + +#do_configure() { +# +#} + +#do_compile() { +# +#} + +#do_stage() { +# +#} + +#do_install() { +# +#} +""" + newpackage.close() + os.system( "%s %s/%s" % ( os.environ.get( "EDITOR" ), fulldirname, filename ) ) + new.usage = " " + + def pasteBin( self, params ): + """Send a command + output buffer to http://pastebin.com""" + index = params[0] + contents = self._shell.myout.buffer( int( index ) ) + status, error, location = sendToPastebin( contents ) + if status == 302: + print "SHELL: Pasted to %s" % location + else: + print "ERROR: %s %s" % ( status, error ) + pasteBin.usage = "" + + def pasteLog( self, params ): + """Send the last event exception error log (if there is one) to http://pastebin.com""" + if last_exception is None: + print "SHELL: No Errors yet (Phew)..." + else: + reason, event = last_exception.args + print "SHELL: Reason for the last error: '%s'" % reason + if ':' in reason: + msg, filename = reason.split( ':' ) + filename = filename.strip() + print "SHELL: Pasting log file to pastebin..." + + status, error, location = sendToPastebin( open( filename ).read() ) + + if status == 302: + print "SHELL: Pasted to %s" % location + else: + print "ERROR: %s %s" % ( status, error ) + + def patch( self, params ): + """Execute 'patch' command on a providee""" + self.build( params, "patch" ) + patch.usage = "" + + def parse( self, params ): + """(Re-)parse .bb files and calculate the dependency graph""" + cooker.status = cooker.ParsingStatus() + ignore = data.getVar("ASSUME_PROVIDED", cooker.configuration.data, 1) or "" + cooker.status.ignored_dependencies = set( ignore.split() ) + cooker.handleCollections( data.getVar("BBFILE_COLLECTIONS", cooker.configuration.data, 1) ) + + cooker.collect_bbfiles( cooker.myProgressCallback ) + cooker.buildDepgraph() + global parsed + parsed = True + print + + def reparse( self, params ): + """(re)Parse a providee's bb file""" + bbfile = self._findProvider( params[0] ) + if bbfile is not None: + print "SHELL: Found bbfile '%s' for '%s'" % ( bbfile, params[0] ) + self.fileReparse( [ bbfile ] ) + else: + print "ERROR: Nothing provides '%s'" % params[0] + reparse.usage = "" + + def getvar( self, params ): + """Dump the contents of an outer BitBake environment variable""" + var = params[0] + value = data.getVar( var, cooker.configuration.data, 1 ) + print value + getvar.usage = "" + + def peek( self, params ): + """Dump contents of variable defined in providee's metadata""" + name, var = params + bbfile = self._findProvider( name ) + if bbfile is not None: + value = cooker.pkgdata[bbfile].getVar( var, 1 ) + print value + else: + print "ERROR: Nothing provides '%s'" % name + peek.usage = " " + + def poke( self, params ): + """Set contents of variable defined in providee's metadata""" + name, var, value = params + bbfile = self._findProvider( name ) + d = cooker.pkgdata[bbfile] + if bbfile is not None: + data.setVar( var, value, d ) + + # mark the change semi persistant + cooker.pkgdata.setDirty(bbfile, d) + print "OK" + else: + print "ERROR: Nothing provides '%s'" % name + poke.usage = " " + + def print_( self, params ): + """Dump all files or providers""" + what = params[0] + if what == "files": + self._checkParsed() + for key in cooker.pkgdata.keys(): print key + elif what == "providers": + self._checkParsed() + for key in cooker.status.providers.keys(): print key + else: + print "Usage: print %s" % self.print_.usage + print_.usage = "" + + def python( self, params ): + """Enter the expert mode - an interactive BitBake Python Interpreter""" + sys.ps1 = "EXPERT BB>>> " + sys.ps2 = "EXPERT BB... " + import code + interpreter = code.InteractiveConsole( dict( globals() ) ) + interpreter.interact( "SHELL: Expert Mode - BitBake Python %s\nType 'help' for more information, press CTRL-D to switch back to BBSHELL." % sys.version ) + + def showdata( self, params ): + """Execute 'showdata' on a providee""" + self.build( params, "showdata" ) + showdata.usage = "" + + def setVar( self, params ): + """Set an outer BitBake environment variable""" + var, value = params + data.setVar( var, value, cooker.configuration.data ) + print "OK" + setVar.usage = " " + + def rebuild( self, params ): + """Clean and rebuild a .bb file or a providee""" + self.build( params, "clean" ) + self.build( params, "build" ) + rebuild.usage = "" + + def shell( self, params ): + """Execute a shell command and dump the output""" + if params != "": + print commands.getoutput( " ".join( params ) ) + shell.usage = "<...>" + + def stage( self, params ): + """Execute 'stage' on a providee""" + self.build( params, "stage" ) + stage.usage = "" + + def status( self, params ): + """""" + print "-" * 78 + print "build cache = '%s'" % cooker.build_cache + print "build cache fail = '%s'" % cooker.build_cache_fail + print "building list = '%s'" % cooker.building_list + print "build path = '%s'" % cooker.build_path + print "consider_msgs_cache = '%s'" % cooker.consider_msgs_cache + print "build stats = '%s'" % cooker.stats + if last_exception is not None: print "last_exception = '%s'" % repr( last_exception.args ) + print "memory output contents = '%s'" % self._shell.myout._buffer + + def test( self, params ): + """""" + print "testCommand called with '%s'" % params + + def unpack( self, params ): + """Execute 'unpack' on a providee""" + self.build( params, "unpack" ) + unpack.usage = "" + + def which( self, params ): + """Computes the providers for a given providee""" + item = params[0] + + self._checkParsed() + + preferred = data.getVar( "PREFERRED_PROVIDER_%s" % item, cooker.configuration.data, 1 ) + if not preferred: preferred = item + + try: + lv, lf, pv, pf = cooker.findBestProvider( preferred ) + except KeyError: + lv, lf, pv, pf = (None,)*4 + + try: + providers = cooker.status.providers[item] + except KeyError: + print "SHELL: ERROR: Nothing provides", preferred + else: + for provider in providers: + if provider == pf: provider = " (***) %s" % provider + else: provider = " %s" % provider + print provider + which.usage = "" + +########################################################################## +# Common helper functions +########################################################################## + +def completeFilePath( bbfile ): + """Get the complete bbfile path""" + if not cooker.pkgdata: return bbfile + for key in cooker.pkgdata.keys(): + if key.endswith( bbfile ): + return key + return bbfile + +def sendToPastebin( content ): + """Send content to http://www.pastebin.com""" + mydata = {} + mydata["parent_pid"] = "" + mydata["format"] = "bash" + mydata["code2"] = content + mydata["paste"] = "Send" + mydata["poster"] = "%s@%s" % ( os.environ.get( "USER", "unknown" ), socket.gethostname() or "unknown" ) + params = urllib.urlencode( mydata ) + headers = {"Content-type": "application/x-www-form-urlencoded","Accept": "text/plain"} + + conn = httplib.HTTPConnection( "pastebin.com:80" ) + conn.request("POST", "/", params, headers ) + + response = conn.getresponse() + conn.close() + + return response.status, response.reason, response.getheader( "location" ) or "unknown" + +def completer( text, state ): + """Return a possible readline completion""" + debugOut( "completer called with text='%s', state='%d'" % ( text, state ) ) + + if state == 0: + line = readline.get_line_buffer() + if " " in line: + line = line.split() + # we are in second (or more) argument + if line[0] in cmds and hasattr( cmds[line[0]][0], "usage" ): # known command and usage + u = getattr( cmds[line[0]][0], "usage" ).split()[0] + if u == "": + allmatches = cooker.configuration.data.keys() + elif u == "": + if cooker.pkgdata is None: allmatches = [ "(No Matches Available. Parsed yet?)" ] + else: allmatches = [ x.split("/")[-1] for x in cooker.pkgdata.keys() ] + elif u == "": + if cooker.pkgdata is None: allmatches = [ "(No Matches Available. Parsed yet?)" ] + else: allmatches = cooker.status.providers.iterkeys() + else: allmatches = [ "(No tab completion available for this command)" ] + else: allmatches = [ "(No tab completion available for this command)" ] + else: + # we are in first argument + allmatches = cmds.iterkeys() + + completer.matches = [ x for x in allmatches if x[:len(text)] == text ] + #print "completer.matches = '%s'" % completer.matches + if len( completer.matches ) > state: + return completer.matches[state] + else: + return None + +def debugOut( text ): + if debug: + sys.stderr.write( "( %s )\n" % text ) + +def columnize( alist, width = 80 ): + """ + A word-wrap function that preserves existing line breaks + and most spaces in the text. Expects that existing line + breaks are posix newlines (\n). + """ + return reduce(lambda line, word, width=width: '%s%s%s' % + (line, + ' \n'[(len(line[line.rfind('\n')+1:]) + + len(word.split('\n',1)[0] + ) >= width)], + word), + alist + ) + +def globfilter( names, pattern ): + return fnmatch.filter( names, pattern ) + +########################################################################## +# Class MemoryOutput +########################################################################## + +class MemoryOutput: + """File-like output class buffering the output of the last 10 commands""" + def __init__( self, delegate ): + self.delegate = delegate + self._buffer = [] + self.text = [] + self._command = None + + def startCommand( self, command ): + self._command = command + self.text = [] + def endCommand( self ): + if self._command is not None: + if len( self._buffer ) == 10: del self._buffer[0] + self._buffer.append( ( self._command, self.text ) ) + def removeLast( self ): + if self._buffer: + del self._buffer[ len( self._buffer ) - 1 ] + self.text = [] + self._command = None + def lastBuffer( self ): + if self._buffer: + return self._buffer[ len( self._buffer ) -1 ][1] + def bufferedCommands( self ): + return [ cmd for cmd, output in self._buffer ] + def buffer( self, i ): + if i < len( self._buffer ): + return "BB>> %s\n%s" % ( self._buffer[i][0], "".join( self._buffer[i][1] ) ) + else: return "ERROR: Invalid buffer number. Buffer needs to be in (0, %d)" % ( len( self._buffer ) - 1 ) + def write( self, text ): + if self._command is not None and text != "BB>> ": self.text.append( text ) + if self.delegate is not None: self.delegate.write( text ) + def flush( self ): + return self.delegate.flush() + def fileno( self ): + return self.delegate.fileno() + def isatty( self ): + return self.delegate.isatty() + +########################################################################## +# Class BitBakeShell +########################################################################## + +class BitBakeShell: + + def __init__( self ): + """Register commands and set up readline""" + self.commandQ = Queue.Queue() + self.commands = BitBakeShellCommands( self ) + self.myout = MemoryOutput( sys.stdout ) + self.historyfilename = os.path.expanduser( "~/.bbsh_history" ) + self.startupfilename = os.path.expanduser( "~/.bbsh_startup" ) + + readline.set_completer( completer ) + readline.set_completer_delims( " " ) + readline.parse_and_bind("tab: complete") + + try: + readline.read_history_file( self.historyfilename ) + except IOError: + pass # It doesn't exist yet. + + print __credits__ + + # save initial cooker configuration (will be reused in file*** commands) + global initdata + initdata = copy.deepcopy( cooker.configuration.data ) + + def cleanup( self ): + """Write readline history and clean up resources""" + debugOut( "writing command history" ) + try: + readline.write_history_file( self.historyfilename ) + except: + print "SHELL: Unable to save command history" + + def registerCommand( self, command, function, numparams = 0, usage = "", helptext = "" ): + """Register a command""" + if usage == "": usage = command + if helptext == "": helptext = function.__doc__ or "" + cmds[command] = ( function, numparams, usage, helptext ) + + def processCommand( self, command, params ): + """Process a command. Check number of params and print a usage string, if appropriate""" + debugOut( "processing command '%s'..." % command ) + try: + function, numparams, usage, helptext = cmds[command] + except KeyError: + print "SHELL: ERROR: '%s' command is not a valid command." % command + self.myout.removeLast() + else: + if (numparams != -1) and (not len( params ) == numparams): + print "Usage: '%s'" % usage + return + + result = function( self.commands, params ) + debugOut( "result was '%s'" % result ) + + def processStartupFile( self ): + """Read and execute all commands found in $HOME/.bbsh_startup""" + if os.path.exists( self.startupfilename ): + startupfile = open( self.startupfilename, "r" ) + for cmdline in startupfile: + debugOut( "processing startup line '%s'" % cmdline ) + if not cmdline: + continue + if "|" in cmdline: + print "ERROR: '|' in startup file is not allowed. Ignoring line" + continue + self.commandQ.put( cmdline.strip() ) + + def main( self ): + """The main command loop""" + while not leave_mainloop: + try: + if self.commandQ.empty(): + sys.stdout = self.myout.delegate + cmdline = raw_input( "BB>> " ) + sys.stdout = self.myout + else: + cmdline = self.commandQ.get() + if cmdline: + allCommands = cmdline.split( ';' ) + for command in allCommands: + pipecmd = None + # + # special case for expert mode + if command == 'python': + sys.stdout = self.myout.delegate + self.processCommand( command, "" ) + sys.stdout = self.myout + else: + self.myout.startCommand( command ) + if '|' in command: # disable output + command, pipecmd = command.split( '|' ) + delegate = self.myout.delegate + self.myout.delegate = None + tokens = shlex.split( command, True ) + self.processCommand( tokens[0], tokens[1:] or "" ) + self.myout.endCommand() + if pipecmd is not None: # restore output + self.myout.delegate = delegate + + pipe = popen2.Popen4( pipecmd ) + pipe.tochild.write( "\n".join( self.myout.lastBuffer() ) ) + pipe.tochild.close() + sys.stdout.write( pipe.fromchild.read() ) + # + except EOFError: + print + return + except KeyboardInterrupt: + print + +########################################################################## +# Start function - called from the BitBake command line utility +########################################################################## + +def start( aCooker ): + global cooker + cooker = aCooker + bbshell = BitBakeShell() + bbshell.processStartupFile() + bbshell.main() + bbshell.cleanup() + +if __name__ == "__main__": + print "SHELL: Sorry, this program should only be called by BitBake." diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py new file mode 100644 index 0000000..ee8713a --- /dev/null +++ b/bitbake/lib/bb/utils.py @@ -0,0 +1,71 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake Utility Functions + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +This file is part of the BitBake build tools. +""" + +digits = "0123456789" +ascii_letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +import re + +def explode_version(s): + r = [] + alpha_regexp = re.compile('^([a-zA-Z]+)(.*)$') + numeric_regexp = re.compile('^(\d+)(.*)$') + while (s != ''): + if s[0] in digits: + m = numeric_regexp.match(s) + r.append(int(m.group(1))) + s = m.group(2) + continue + if s[0] in ascii_letters: + m = alpha_regexp.match(s) + r.append(m.group(1)) + s = m.group(2) + continue + s = s[1:] + return r + +def vercmp_part(a, b): + va = explode_version(a) + vb = explode_version(b) + while True: + if va == []: + ca = None + else: + ca = va.pop(0) + if vb == []: + cb = None + else: + cb = vb.pop(0) + if ca == None and cb == None: + return 0 + if ca > cb: + return 1 + if ca < cb: + return -1 + +def vercmp(ta, tb): + (va, ra) = ta + (vb, rb) = tb + + r = vercmp_part(va, vb) + if (r == 0): + r = vercmp_part(ra, rb) + return r diff --git a/bitbake/setup.py b/bitbake/setup.py new file mode 100755 index 0000000..adb14a7 --- /dev/null +++ b/bitbake/setup.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Copyright (C) 2003 Chris Larson +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. + +from distutils.core import setup +import os, sys + +# bbdir = os.path.join(sys.prefix, 'share', 'bitbake') +# docdir = os.path.join(sys.prefix, 'share', 'doc') +bbdir = os.path.join('bitbake') +docdir = os.path.join('doc') + +def clean_doc(type): + origpath = os.path.abspath(os.curdir) + os.chdir(os.path.join(origpath, 'doc', 'manual')) + make = os.environ.get('MAKE') or 'make' + os.system('%s clean-%s' % (make, type)) + +def generate_doc(type): + origpath = os.path.abspath(os.curdir) + os.chdir(os.path.join(origpath, 'doc', 'manual')) + make = os.environ.get('MAKE') or 'make' + ret = os.system('%s %s' % (make, type)) + if ret != 0: + print "ERROR: Unable to generate html documentation." + sys.exit(ret) + os.chdir(origpath) + +if 'bdist' in sys.argv[1:]: + generate_doc('html') + +sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), 'lib')) +import bb +import glob +setup(name='bitbake', + version=bb.__version__, + license='GPL', + url='http://developer.berlios.de/projects/bitbake/', + description='BitBake build tool', + long_description='BitBake is a simple tool for the execution of tasks. It is derived from Portage, which is the package management system used by the Gentoo Linux distribution. It is most commonly used to build packages, as it can easily use its rudamentary inheritence to abstract common operations, such as fetching sources, unpacking them, patching them, compiling them, and so on. It is the basis of the OpenEmbedded project, which is being used for OpenZaurus, Familiar, and a number of other Linux distributions.', + author='Chris Larson', + author_email='clarson@elinux.org', + packages=['bb', 'bb.fetch', 'bb.parse', 'bb.parse.parse_py'], + package_dir={'bb': os.path.join('lib', 'bb')}, + scripts=[os.path.join('bin', 'bitbake'), + os.path.join('bin', 'bbimage')], + data_files=[(os.path.join(bbdir, 'conf'), [os.path.join('conf', 'bitbake.conf')]), + (os.path.join(bbdir, 'classes'), [os.path.join('classes', 'base.bbclass')]), + (os.path.join(docdir, 'bitbake-%s' % bb.__version__, 'html'), glob.glob(os.path.join('doc', 'manual', 'html', '*.html'))), + (os.path.join(docdir, 'bitbake-%s' % bb.__version__, 'pdf'), glob.glob(os.path.join('doc', 'manual', 'pdf', '*.pdf'))),], + ) + +if 'bdist' in sys.argv[1:]: + clean_doc('html') -- 2.41.1