From eb2013609f78db60a4384c25c4b9074a0c450487 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Wed, 3 Jun 2026 09:29:47 +1000 Subject: [PATCH 1/4] Convert old leo files - still need cleaning up --- docs/isolation_info.rst | 196 +++++------- docs/pool_info.rst | 667 +++++++++++++++------------------------- 2 files changed, 327 insertions(+), 536 deletions(-) diff --git a/docs/isolation_info.rst b/docs/isolation_info.rst index f5762f7..1a3731e 100644 --- a/docs/isolation_info.rst +++ b/docs/isolation_info.rst @@ -1,42 +1,4 @@ - - - - - - - - - - - -Isolation research -explore fakechroot -research chroot (why only root?) -break out of chroot exploit -break out of chroot exploit #2 - -security analysis of debian buildd infrastructure -explore openvz -explore vmware -explore xen -explore ubuntu support for xen - -explore uml -explore rootstrap -explore pbuilder-uml -explore tun/tap - -explore virtualbox -explore dchroot / schroot -time chroot tarball extraction - -explore vserver -explore linux capabilities -explore grsecurity chroot security - - - -@nocolor +@nocolor * SUMMARY roadmap @@ -166,7 +128,7 @@ use UML to create fake root-like environment keep offshoot of UML builder around like a daemon invocation of UML transparent? - uml <uml> [ command ] + uml [ command ] first boots uml if it doesn't exist rsync result back to fakeroot system @@ -193,7 +155,7 @@ pbuilder in fakeroot / fakechroot environment? use uml? thats what rootstrap does -RESOURCE: http://slashdot.org/comments.pl?threshold=5&mode=thread&commentsort=0&op=Change&sid=234745 +RESOURCE: http://slashdot.org/comments.pl?threshold=5&mode=thread&commentsort=0&op=Change&sid=234745 performance evaluation of xen vs openVZ @@ -215,8 +177,8 @@ openstandard proposed describing the protocol that guest OSes communicates with paravirtualization - doesn't simulate hardware - offers a pecial API hypervisor / hypercalls - -SUMMARY + +SUMMARY won't work with statiatically linked programs (neither does fakeroot) it allows chroot() to work without root privileges @@ -362,8 +324,8 @@ LIMITATIONS o Your real uid should exist in /etc/passwd. Create it with adduser --uid realuid realuser. - o debuild(1) cleans environment. Use --preserve-env option to prevent this behaviour. -SUMMARY + o debuild(1) cleans environment. Use --preserve-env option to prevent this behaviour. +SUMMARY chroot is a dangerous privilege to grant normal users chroot wasn't designed as a security mechanism but as a software testing tool @@ -378,7 +340,7 @@ A: 1) supposedly to prevent users from changing the contents of trusted files and tricking suid programs into giving permissions - I.e., after chroot /tmp, /etc/passwd -> /tmp/etc/passwd + I.e., after chroot /tmp, /etc/passwd -> /tmp/etc/passwd Q: how do you sneak the suid program into the chroot? A: with a hardlink DISCOVERY: its possible for a regular user to create hardlinks to root suid programs @@ -389,7 +351,7 @@ A: mkdir("testdir"); chroot("testdir"); // make nested jail -- key to escape! fchdir(fd); // back to first jail's / - for (int i=0; i<10; i++) { + for (int i=0; i<10; i++) { chdir(".."); // successively higher ("/../"=="/") } chroot("."); // final "jail" is real / @@ -429,14 +391,14 @@ chroot(2) has no effect on process's current directory you could hide a hardlink to the setuid program there RESOURCE: http://lists.debian.org/debian-security/2001/10/msg00033.html - -001 #include <stdio.h> -002 #include <errno.h> -003 #include <fcntl.h> -004 #include <string.h> -005 #include <unistd.h> -006 #include <sys/stat.h> -007 #include <sys/types.h> + +001 #include +002 #include +003 #include +004 #include +005 #include +006 #include +007 #include 008 009 /* 010 ** You should set NEED_FCHDIR to 1 if the chroot() on your @@ -465,9 +427,9 @@ RESOURCE: http://lists.debian.org/debian-security/2001/10/msg00033.html 033 /* 034 ** First we create the temporary directory if it doesn't exist 035 */ -036 if (stat(TEMP_DIR,&sbuf)<0) { +036 if (stat(TEMP_DIR,&sbuf)<0) { 037 if (errno==ENOENT) { -038 if (mkdir(TEMP_DIR,0755)<0) { +038 if (mkdir(TEMP_DIR,0755)<0) { 039 fprintf(stderr,"Failed to create %s - %s\n", TEMP_DIR, 040 strerror(errno)); 041 exit(1); @@ -490,7 +452,7 @@ RESOURCE: http://lists.debian.org/debian-security/2001/10/msg00033.html 058 ** working directory to the directory given to chroot(). 059 ** 060 */ -061 if ((dir_fd=open(".",O_RDONLY))<0) { +061 if ((dir_fd=open(".",O_RDONLY))<0) { 062 fprintf(stderr,"Failed to open "." for reading - %s\n", 063 strerror(errno)); 064 exit(1); @@ -500,7 +462,7 @@ RESOURCE: http://lists.debian.org/debian-security/2001/10/msg00033.html 068 /* 069 ** Next we chroot() to the temporary directory 070 */ -071 if (chroot(TEMP_DIR)<0) { +071 if (chroot(TEMP_DIR)<0) { 072 fprintf(stderr,"Failed to chroot to %s - %s\n",TEMP_DIR, 073 strerror(errno)); 074 exit(1); @@ -519,7 +481,7 @@ RESOURCE: http://lists.debian.org/debian-security/2001/10/msg00033.html 087 ** working directory to the directory given to chroot(). 088 ** 089 */ -090 if (fchdir(dir_fd)<0) { +090 if (fchdir(dir_fd)<0) { 091 fprintf(stderr,"Failed to fchdir - %s\n", 092 strerror(errno)); 093 exit(1); @@ -539,7 +501,7 @@ RESOURCE: http://lists.debian.org/debian-security/2001/10/msg00033.html 107 ** directory to the current working directory - at this point the real 108 ** root directory. 109 */ -110 for(x=0;x<1024;x++) { +110 for(x=0;x<1024;x++) { 111 chdir(".."); 112 } 113 chroot("."); @@ -547,30 +509,30 @@ RESOURCE: http://lists.debian.org/debian-security/2001/10/msg00033.html 115 /* 116 ** We're finally out - so exec a shell in interactive mode 117 */ -118 if (execl("/bin/sh","-i",NULL)<0) { +118 if (execl("/bin/sh","-i",NULL)<0) { 119 fprintf(stderr,"Failed to exec - %s\n",strerror(errno)); 120 exit(1); 121 } -122 } -#include <stdlib.h> -#include <stdio.h> -#include <errno.h> -#include <unistd.h> -#include <sys/stat.h> -#include <sys/types.h> +122 } +#include +#include +#include +#include +#include +#include int main(void) { int i; mkdir("breakout", 0777); - if (chroot("breakout") < 0) + if (chroot("breakout") < 0) perror("chroot failed"); - for (i = 0; i < 100; i++) - if (chdir("..") < 0) + for (i = 0; i < 100; i++) + if (chdir("..") < 0) perror("chdir failed"); - if (chroot(".") < 0) + if (chroot(".") < 0) perror("chroot2 failed"); execl("/bin/bash", "/bin/bash", (char *)NULL); @@ -578,8 +540,8 @@ int main(void) exit(0); } - -buildd uses sbuild + +buildd uses sbuild sbuild installs build-depends as source? some of the buildds are run by non-DDs that can not be trusted @@ -614,8 +576,8 @@ chroots designed to produce clean builds - -advantage + +advantage files can be seen (not in an image) RESOURCE: http://kerneltrap.org/node/6492 @@ -643,8 +605,8 @@ git.openviz.org RESOURCE: http://community.livejournal.com/openvz/tag/openvz has been merged into SLES - -RESOURCE: http://en.wikipedia.org/wiki/VMware + +RESOURCE: http://en.wikipedia.org/wiki/VMware virtual processing based on dynamic recompilation @@ -660,7 +622,7 @@ how good is the quality of the isolation? NSA considers vm isolation almost as good as air gap? -RESOURCE: http://www.thisishull.net/showthread.php?t=271743&page=2 +RESOURCE: http://www.thisishull.net/showthread.php?t=271743&page=2 vmware guests broken into from the outside @@ -668,8 +630,8 @@ RESOURCE: http://www.eweek.com/article2/0,1759,1904647,00.asp at least one known exploit - nat code in vmware - -RESOURCE: http://www.cl.cam.ac.uk/research/srg/netos/xen/ + +RESOURCE: http://www.cl.cam.ac.uk/research/srg/netos/xen/ official site of the research group that created xen @@ -719,8 +681,8 @@ RESOURCE: http://wiki.xensource.com/xenwiki/XenFaq#head-0a539c0b540b1e563d5b0f39 no support for ACPI/APM reduced battery support and no suspend/resume - -RESOURCE: https://wiki.ubuntu.com/Xen + +RESOURCE: https://wiki.ubuntu.com/Xen ideal goal - ship with xen enabled kernel by default @@ -776,8 +738,8 @@ same kernel for dom0 and the other domains - -TODO: + +TODO: run debian inside UML? there's a tutorial on running debian inside UML setup networking so that uml has access to my apt-proxy @@ -941,8 +903,8 @@ is skas running on my Gentoo - -* SUMMARY + +* SUMMARY create a filesystem image available as a Gentoo ebuild depends on vanilla sources @@ -965,20 +927,20 @@ doesn't need root access? * RESOURCE: rootstrap(1) * RESOURCE: http://people.debian.org/~torsten/rootstrap.html - using rootstrap for package checking -uses rootstrap to create UML image + using rootstrap for package checking +uses rootstrap to create UML image IDEAS: keep UML environment around - -http://en.wikipedia.org/wiki/TUN/TAP + +http://en.wikipedia.org/wiki/TUN/TAP In computer networking, TUN and TAP are virtual network kernel drivers. They implement network devices that are supported entirely in software, which is different from ordinary network devices that are backed up by hardware network adapters. TAP (as in network TAP) simulates an Ethernet device and it operates with Layer 2 packets such as Ethernet frames. TUN (as in network TUNnel) simulates a network layer device and it operates with Layer 3 packets such as IP packets. TAP is used to create a Network bridge, while TUN is used with Routing. -Packets sent by an operating system via a TUN/TAP device are delivered to a user-space program that attaches itself to the device. A user-space program may also pass packets into a TUN/TAP device. In this case TUN/TAP device delivers (or "injects") these packets to the operating system network stack thus emulating their reception from an external source. -/etc/init.d/vboxdrv +Packets sent by an operating system via a TUN/TAP device are delivered to a user-space program that attaches itself to the device. A user-space program may also pass packets into a TUN/TAP device. In this case TUN/TAP device delivers (or "injects") these packets to the operating system network stack thus emulating their reception from an external source. +/etc/init.d/vboxdrv /etc/init.d/vboxnet Q: does it run without root privileges? @@ -1048,7 +1010,7 @@ SUMMARY write-through disks state not saved in suspension - vboxmanage convertdd <thefile>.img <thefile>.vdi + vboxmanage convertdd .img .vdi IDEAS we could just install debian/ubuntu to the system (normal way via ISO) @@ -1080,9 +1042,9 @@ default debian settings debconf-set-selections vboxconf NAT port forwarding - VBoxManage setextradata "Guest" VBoxInternal/Devices/pcnet/0/LUN#0/Config/<service>/Protocol TCP - VBoxManage setextradata "Guest" VBoxInternal/Devices/pcnet/0/LUN#0/Config/<service>/GuestPort 22 - VBoxManage setextradata "Guest" VBoxInternal/Devices/pcnet/0/LUN#0/Config/<service>/HostPort 2222 + VBoxManage setextradata "Guest" VBoxInternal/Devices/pcnet/0/LUN#0/Config//Protocol TCP + VBoxManage setextradata "Guest" VBoxInternal/Devices/pcnet/0/LUN#0/Config//GuestPort 22 + VBoxManage setextradata "Guest" VBoxInternal/Devices/pcnet/0/LUN#0/Config//HostPort 2222 HIF (Host Interface Networking) setup a new network card (e.g., vbox0) on host, to which guests are connected @@ -1136,11 +1098,11 @@ VBoxSDL -vm "ubuntu edgy" extended RDP (graphics and audio) -modifyvm <vmname> +modifyvm -vrdp on -vrdpport -vrdpauthtype -VBoxVRDP -startvm <uuid|name> +VBoxVRDP -startvm 8.17 VBoxManage getextradata/setextradata @@ -1155,7 +1117,7 @@ labeling is a bit tricky due to adjustment of non-standard resolutions 9.6 multiple monitors for guest -modifyvm <vmname> -monitor 3 +modifyvm -monitor 3 specify which screen you want to conect to with @1, @2 in domain logon @@ -1226,8 +1188,8 @@ partners co-developing security infrastructure for government use - -IDEAS + +IDEAS hack schroot to support VM chroots? hack schroot to support deck? just use LVM snapshot type schroot instead of deck? @@ -1274,7 +1236,7 @@ USAGE SUMMARY tar -C jaunty/ -zcvf jaunty.tar.gz . example session - SESSION_ID=$(schroot -b -c <session-chroot>) + SESSION_ID=$(schroot -b -c ) schroot -r -c $SESSION_ID schroot -e -c $SESSION_ID @@ -1337,8 +1299,8 @@ RESOURCE: schroot.conf man page RESOURCE: schroot-setup(5) describes schroot setup.d scripts - sets up mounts, networking -after caching + sets up mounts, networking +after caching create 2 seconds (uncompressed) @@ -1346,8 +1308,8 @@ create extract 6 seconds compressed - 4-6 seconds uncompressed -SUMMARY + 4-6 seconds uncompressed +SUMMARY very mature project, under active development first public release 2001 more efficient than virtualization @@ -1363,7 +1325,7 @@ Q: does chroot work inside the vserver? RESOURCE: https://lists.linux-foundation.org/pipermail/containers/ containers are under very active development -RESOURCE: http://forums.grsecurity.net/viewtopic.php?t=1801&highlight=&sid=a31a551220aac5586ddae62d55836973 +RESOURCE: http://forums.grsecurity.net/viewtopic.php?t=1801&highlight=&sid=a31a551220aac5586ddae62d55836973 report that grsecurity and vserver apply together @@ -1407,15 +1369,15 @@ other appraoches paravirtualization native virtualization operating system-leve virtualization - -Q: can block devices be accessed without capabilities? + +Q: can block devices be accessed without capabilities? SUMMARY: POSIX capabilities exist per process? inheritable (I), permitted (P), effective(E) see the capabilities of a process - cat /proc/<pid>/status + cat /proc//status CapInh CapPrm CapEff @@ -1473,8 +1435,8 @@ RESOURCE: http://lwn.net/Articles/179361/ containers and lightweight virtualization - -SUMMARY + +SUMMARY we can't mknod without preventing some packages which need to create device nodes this is also a problem with vserver @@ -1559,7 +1521,7 @@ IDEAS run experiments with grsec chroot restrictions is breakout prevented? -RESOURCE: kernel sec options->grsec->filesystem protections->chroot jail restrictions +RESOURCE: kernel sec options->grsec->filesystem protections->chroot jail restrictions deny mounts Q: what are the chroot hardening mechanisms? @@ -1612,6 +1574,4 @@ R: operation not permitted IDEA: we could restrict privileges in the chroot using an inheritable policy - - - + diff --git a/docs/pool_info.rst b/docs/pool_info.rst index 33a6a80..4fcbbdb 100644 --- a/docs/pool_info.rst +++ b/docs/pool_info.rst @@ -1,165 +1,12 @@ - - - - - - - - - - - -Project -design -brainstorming -first draft - -terminology -file/data structure -cli syntax -usage examples - -implementation -v0.9 -implement full command set for binaries in regular directories -init /path/to/build-chroot -register / unregister # regular directories -info -exists -refactor away stock - -list -get -pool contains pool -BUG: sync not importing binaries? -support regular sources in regular directories -setup a local caching proxy -fix squid -configure jaunty to use apt-proxy -apt-proxy README -test debootstrap with caching proxy - -research existing solutions -explore cowdancer -explore fl-cow -exlore apt-move -explore pbuilder -alternatives - -explore sbuild -explore build dependencies -research suid - -exists -list -get - -sort debian packages according to dpkg algorithm -fix getpath to get newest packages? -pool.Pool().list() can not be invoked twice - -support complex stock types -test/develop lightweight branch cloning -support Git branches -register git branches -fix info -unregister git branches -exists -Stock.sync -light update of light clone -copy all tags - - -list -get - -fix subpools -optimize Stock attributes -support Sumo branches - -extend pool-info -pool-info-build package -add build logs to pool-info -support binary names - -fix pool-info handling of --stocks and --subpools? -bug: debversion.compare mis-calculating integers -optimize -normalize bug - - -unreleased -bug: deb_cmp_versions doesn't work correctly -bug: lexical comparison in debversion is wrong -pool-gc -remove cached packages when we unregister a stock -optimize pool-get'ing a cached binary -add internal cache to PackageCache -make syncing optional - -bug: pool-info-build broken -bug: pool doesn't reproduce thin branch -setup regression testing suite -setup pyhello - -clean up pool code? -bug: permission problems when root runs the pool -bug: pool-list delayed - -bug: pool ignores epoch in versions -optimize extraction of control file from Debian package -optimize parse_control -implement a generic key/value filesystem cache -profile optimization -use md5sum instead of sha1? -calculate key just by the hash of the control -extract control.tar.gz from python -optimize? - - -optimization: implement extraction of control file fully in Python - -bug: mishandling udebs? -add pool-info command that shows the source of binary packages -add --binary-versions option to pool-info command - -patch unregister and gc to take advantage of binary-versions -optimize pool privilege dropping -implement POC master/slave RPC -benchmark forked_constructor vs forked_func -hack pool -control initialization of Pool with __new__ -develop attribute access -work out bugs -stress test -communicate exception backtrace? -bug: some operations are executed as root - -refactor cli interface into Pool -list -get - - - -bugs/wishlist -pool info command that shows the source of a package -adapter that allows pool to be accessed by apt-get -bug: pool assumes that packages have always existed in all versions -accept multi-arguments in register/unregister? -auto less output in pool-info? / pool-info-build - - - - -@nocolor - -The pool is a system thats maintains a pool of binary packages which may be either imported as-is from a registered collection of binaries, or built on-demand from a registered source or collection of sources. +@nocolor + +The pool is a system thats maintains a pool of binary packages which may be either imported as-is from a registered collection of binaries, or built on-demand from a registered source or collection of sources. The dominant abstraction for the pool is a virtual filesystem folder that you can get get specific versions of specific packages from. Behind the scenes, the pool will build those binary packages for from source if required and cache the built binaries for future access. -In this sense calling it a pool is in line with software terminology conventions [http://en.wikipedia.org/wiki/Object_pool]. -* IDEAS +In this sense calling it a pool is in line with software terminology conventions [http://en.wikipedia.org/wiki/Object_pool]. +* IDEAS change untagged auto-versioning to always be smaller than tagged autoversioning auto-versioning a separate program? @@ -168,7 +15,7 @@ auto-versioning a separate program? leave failed builds in temporary directory so we can diagnose problem manually - $TMPDIR/pool/<poolname>/<package> + $TMPDIR/pool// figure out how to allow packages to be built manually unversioned - for testing? @@ -236,32 +83,32 @@ Q: do we support different versioning for single-debian package and multi-debian Q: can we support object caching safely? A: if its content addressable then yes - -Notes: + +Notes: * The pools tucks away all of its internals neatly out of site in a `hidden' directory. The reason for doing this was to make it easy to treat the pool as a `special' directory, which may contain registered stocks or subpools as sub-directories. * The pool has no data formats/configuration files. All of its `data' is stored directly as filesystem constructs. .pool/ build/ - root -> /path/to/buildroot # symbolic link + root -> /path/to/buildroot # symbolic link logs/ - <package>-<version>.build + -.build log of the build process pkgcache/ - <package>-<version>.<arch>.deb # maybe in a pool-like tree + -..deb # maybe in a pool-like tree stock/ - <name>#<branch>/ - link -> /path/to/stock # symbolic link to the stock + #/ + link -> /path/to/stock # symbolic link to the stock index-sources/ - <relative-path>/<package> + / contains cached list of versions index-binaries/ - <relative-path>/<package-filename> + / zero size SYNC_HEAD # contains the last checkout/.git/HEAD we synced against @@ -274,8 +121,8 @@ A: - -=== ENVIRONMENT + +=== ENVIRONMENT POOL_DIR defaults to cwd @@ -313,22 +160,22 @@ A: or no information if it wasn't built or an error that the package doesn't exist -* get [-options] <output-dir> [ package[=version] ... ] +* get [-options] [ package[=version] ... ] Get packages from pool If a package is specified without a version, get the newest package. If no packages are specified as arguments, get all the newest packages. Options: - -i --input <file> file from which we read package list (- for stdin) + -i --input file from which we read package list (- for stdin) -s --strict fatal error on missing packages -q --quiet suppress warnings about missing packages -t --tree output dir is in a package tree format (like a repository) -* list [ <package-glob> ] - if <package-glob> is provided, print only those packages whose names match the glob +* list [ ] + if is provided, print only those packages whose names match the glob otherwise, by default, print a list of the newest packages -a --all-versions @@ -341,7 +188,7 @@ Options: * exists package[=version] Check if package exists in pool -Prints true/false if <package> exists in the pool. +Prints true/false if exists in the pool. If true exitcode = 0, else exitcode = 1 === FUTURE CLI? @@ -357,8 +204,8 @@ open open the FUSE-based filesystem representation of the pool close - close the FUSE-based filesystem representation of the pool -cd pools + close the FUSE-based filesystem representation of the pool +cd pools mkdir private cd private @@ -422,11 +269,11 @@ for package in $(cat /path/to/versioned-product-manifest); do if pool-exists -q $package; then pool-get /tmp/product-repo --tree -s $package fi -done -ROADMAP +done +ROADMAP support Sumo arenas - -pool + +pool a system for which maintains a pool of binary packages built on-demand from registered stock stock @@ -462,8 +309,8 @@ build log package cache a cache of binary packages either built from source or imported as-is - -Q: how do i name this project? + +Q: how do i name this project? alternatives pms (package management system), and you know what else... @@ -521,10 +368,10 @@ latest? current -A: the action is to generate, the outcome is an instance. - - -initialize a new pool +A: the action is to generate, the outcome is an instance. + + +initialize a new pool * init /path/to/build-chroot initialize a new pool @@ -534,8 +381,8 @@ A: the action is to generate, the outcome is an instance. mkdir .pool/build/logs link .pool/build to buildroot - -* register /path/to/stock + +* register /path/to/stock register a package stock with the pool stock type can be: another pool @@ -552,15 +399,15 @@ no auto-versioning * register logic raise exception if stock is already registered if it isn't, register stock by - creating stock/<stock_name> + creating stock/ creating symbolic link - ln -s <dir> stock/<stock_name>/path + ln -s stock//path * unregister logic raise "stock not registered" exception if dir doesn't match stock/path rmdir .pool/stock/name - -TESTS: + +TESTS: register a directory with one binary package register another directory with two more packages @@ -579,9 +426,9 @@ sync logic hardlink/copy into cache - -* list [ <package-glob> ] - if <package-glob> is provided, print only those packages whose names match the glob + +* list [ ] + if is provided, print only those packages whose names match the glob otherwise, by default, print a list of the newest packages -a --all-versions @@ -589,8 +436,8 @@ sync logic -n --name-only print only the names of packages in the pool (without the version) - incompatible with -a option -logic + incompatible with -a option +logic if no arguments, list the newest packages check if package exists @@ -605,18 +452,18 @@ sync logic ask the pool to output that package to the outputdir - -* logic + +* logic list .pool/stock subdirectories for each stock: print its path - -class PoolStocks: + +class PoolStocks: def __init__(self): pass - -ROADMAP + +ROADMAP solve typing issue proxy all pool functions handle circular dependencies @@ -629,8 +476,8 @@ A: Q: how do I prevent cyclic subpools? A: track recursed_paths. Prevent - -* ROADMAP + +* ROADMAP exists list get @@ -641,7 +488,7 @@ get * IDEAS get_sources - path/to/dir/package-name: <version> + path/to/dir/package-name: add all legally listed binary packages files we build to cache (so we don't have to rebuild the same package multiple times) @@ -650,16 +497,16 @@ extract package to package build directory we need to install build dependencies to the buildroot with apt-get - - + + * TODO explore building packages with pbuilder Q: does pbuilder need root privileges? Q: can we use fakeroot? Q: how does pbuilder work? - -SUMMARY + +SUMMARY how it works LD_PRELOAD trick @@ -669,7 +516,7 @@ how it works usage cp -al src dst - cd dst && cow-shell + cd dst && cow-shell file hardlinked - wil be copied to a new file before write usage with chroot @@ -685,9 +532,8 @@ cowdancer is copy-on-write upon directory trees copied with cp -la pdebuild --pbuilder cowbuilder - - -Q: does the pbuilder environment setup mounts? + +Q: does the pbuilder environment setup mounts? TODO read manual @@ -742,7 +588,7 @@ INSIGHT: cowdancer is not secure for isolation purposes hardlinks could be modified -RESOURE: http://apsy.gse.uni-magdeburg.de/main/index.psp?page=hanke/pbuilder-ssh&lang=en +RESOURE: http://apsy.gse.uni-magdeburg.de/main/index.psp?page=hanke/pbuilder-ssh&lang=en pbuilder-ssh: remote front-end for pbuilder not part of the distribution @@ -756,13 +602,13 @@ pbuilder doesn't try to guess what the package needs pbuilder create create base chroot image with debootstrap --distribution - -same principle as cowdancer, slightly different implementation + +same principle as cowdancer, slightly different implementation fl-cow will apply to any directory inside FLCOW_PATH - -* SUMMARY + +* SUMMARY what is this good for? we don't want to retrieve build dependencies over the network over and over again IDEA: patch apt-proxy so it doesn't hang if it can't retrieve a new Packages/Sources @@ -961,15 +807,15 @@ adjust refresh patterns with * RESOURCE: http://apt-proxy.sourceforge.net/ - -SUMMARY + +SUMMARY to do chroot we need suid privileges how do we do this correctly? RESOURCE: http://www.cs.berkeley.edu/~daw/papers/setuid-usenix02.pdf - -* SUMMARY: + +* SUMMARY: giving up on squid for now - Gentoo package is broken * GOTCHA: @@ -1006,10 +852,10 @@ recommended configuration http_access allow my_network #allow access for our network http_access deny all #and deny from all others - -debootstrap jaunty jaunty http://apt:9999/ubuntu - -emacs /etc/hosts + +debootstrap jaunty jaunty http://apt:9999/ubuntu + +emacs /etc/hosts 127.0.0.1 apt ping apt @@ -1023,13 +869,13 @@ import packages from apt-cache? no it doesn't work - broken on ubuntu - -maintains a local mirror from various sources + +maintains a local mirror from various sources configured by default to create a mirror from apt's cache caches the control file of packages - .index/<package> + .index/ configuration file LOCALDIR absolute path to your mirror @@ -1054,8 +900,8 @@ apt-move packages GOTCHA: apt-move packages aren't authenticated - http://wiki.debian.org/SecureApt -A2: debootstrap, which uses wget. This means you can easily install new + http://wiki.debian.org/SecureApt +A2: debootstrap, which uses wget. This means you can easily install new machine using the packages out of your apt-proxy cache. In boot floppies, specify http://APTPROXY:9999/main as your debian mirror (replacing APTPROXY with the name or IP address of the machine where apt-proxy is running). @@ -1075,8 +921,8 @@ A4: pbuilder, which also uses debootstrap. Add this to /etc/pbuilderrc: MIRRORSITE=http://APTPROXT:9999/main NONUSMIRRORSITE=http://APTPROXT:9999/non-US - -SUMMARY + +SUMMARY location of Packages/Sources /var/lib/apt/lists @@ -1101,10 +947,10 @@ INSIGHT: the lack of build dependencies is a QA feature * analysis of build-dependencies in Ubuntu -cat /var/lib/apt/lists/*Sources | grep-dctrl -n -s Build-Depends ""|sed 's/, /\n/g'|sort -u > build-depends +cat /var/lib/apt/lists/*Sources | grep-dctrl -n -s Build-Depends ""|sed 's/, /\n/g'|sort -u > build-depends 7238 different packages with different versions -cat build-depends | perl -pe 's/\s*[,\|]\s*/\n/g' | perl -pe 's/\s*\(.*//; s/^\s*//; s/\[.*//' | sort -u > build-depends-pkgs +cat build-depends | perl -pe 's/\s*[,\|]\s*/\n/g' | perl -pe 's/\s*\(.*//; s/^\s*//; s/\[.*//' | sort -u > build-depends-pkgs 3078 different packages (multiple versions counted) apt-cache show $(cat build-depends-pkgs) | grep-dctrl -n -s Installed-Size "" | awk '{ sum += $1 } END { print sum }' @@ -1113,29 +959,29 @@ apt-cache show $(cat build-depends-pkgs) | grep-dctrl -n -s Installed-Size "" | IDEA: extract build dependencies for the packages in world -cat /turnkey/repository/*/*/Packages > sphinx -apt-cache showsrc $(cat sphinx | grep-dctrl -n -s Package "" |sort -u) > sphinx-sources +cat /turnkey/repository/*/*/Packages > sphinx +apt-cache showsrc $(cat sphinx | grep-dctrl -n -s Package "" |sort -u) > sphinx-sources 567 binaries 546 sources -cat sphinx-sources |grep-dctrl -n -s Build-Depends "" | perl -pe 's/\s*[,\|]\s*/\n/g' |sort -u > sphinx-sources-builddeps +cat sphinx-sources |grep-dctrl -n -s Build-Depends "" | perl -pe 's/\s*[,\|]\s*/\n/g' |sort -u > sphinx-sources-builddeps 742 build depends (with different versions) -cat sphinx-sources-builddeps | perl -pe 's/\s*\(.*//; s/^\s*//; s/\[.*//' |sort -u > sphinx-sources-builddeps-pkgs +cat sphinx-sources-builddeps | perl -pe 's/\s*\(.*//; s/^\s*//; s/\[.*//' |sort -u > sphinx-sources-builddeps-pkgs 438 build depends (different versions counted once) apt-cache show $(cat sphinx-sources-builddeps-pkgs) | grep-dctrl -n -s Installed-Size "" | awk '{ sum += $1 } END { print sum }' 771MB - -apt-build + +apt-build debian-builder - -assume get_sources returns: + +assume get_sources returns: /path/to/package: version get_sources logic @@ -1153,9 +999,8 @@ parse output from dpkg-parsechangelog ^(\w[-+0-9a-z.]*) \(([^\(\) \t]+)\)((\s+[-+0-9a-z.]+)+)\;/i parse the changelog ourselves - - -getpath logic + +getpath logic try to get the package from the cache, but if we can't: search stocks to find source path matching package/version deckdebuild the package with the provided build root @@ -1166,8 +1011,8 @@ parse the changelog ourselves POOL_TMPDIR defaults to TMPDIR or /var/tmp -$POOL_TMPDIR/<package>-<version>/ - <package>-<version> +$POOL_TMPDIR/-/ + - *.debs @@ -1189,8 +1034,8 @@ A: not really the source is a full filesystem path (unless we want to report an * TODO: debsrc.get_name pool.tmpdir - -TODO: fixed untagged auto-versioning + +TODO: fixed untagged auto-versioning features support branches in Git repositories @@ -1227,8 +1072,8 @@ tests/milestones get that stable versions built register a Sumo repository - -SUMMARY + +SUMMARY sbuild mails build logs to maintainer builds from dsc @@ -1236,8 +1081,8 @@ tests/milestones Q: how is sbuild different from pbuilder? A: doesn't build the package directly from source (pdebuild) (though generating source package is not a problem) - -SUMMARY + +SUMMARY goal: checkout a specific branch to a separate working directory as efficiently as possible insight: we may to modify this script somewhat to work with Sumo arenas use sumo-checkout instead of git-checkout @@ -1265,36 +1110,36 @@ logic rm -f .git/refs/remotes/origin/HEAD mv .git/refs/remotes/origin/* .git/refs/heads - git-checkout -q <branch> + git-checkout -q test cd /turnkey/tmp/testgit rm -rf test git-clone-branch ../pyproject/ master test - -trailing zeros not considered + +trailing zeros not considered -2010 == 0-2010 - \d-2010 > 0-2010 + \d-2010 > 0-2010 2010 == 2010-0000 1.01 == 1.1 - 1.02 > 1.1 + 1.02 > 1.1 - 2010.1 > 2010-1 + 2010.1 > 2010-1 - 2010:1 > 2010.1 + 2010:1 > 2010.1 logic remove trailing zeros after any non-number IDEA: translate sort algorithm directly to python? just remove trailing zeros? - -TEST + +TEST get pool to give you an older package than is available put newer package in subpool older package in cache will be returned @@ -1320,10 +1165,9 @@ cli level solution - -problem: modification of default array value - - + +problem: modification of default array value + test: register /turnkey/tmp/verseek/git-single should register checked out branch (e.g., master) @@ -1341,8 +1185,8 @@ register logic check that the directory exists try to create a git instance at that directory if stock is a git repo and branch is none: - branch = currently checked out branch -roadmap + branch = currently checked out branch +roadmap specific branches unregistered require explicit branch specification @@ -1356,9 +1200,8 @@ implicit branch unregistration - - -test + +test pool-register /turnkey/tmp/verseek/git-single#devel pool-exists pyproject=1.0 @@ -1423,17 +1266,16 @@ Stock interface versions (relative/path, versions) - - -logic + +logic verseek to desired version - chdir to <workdir>/relative_path + chdir to /relative_path deckdebuild import built package into cache return the path to the built package - -psuedo-logic + +psuedo-logic HEAD is a special file @@ -1460,14 +1302,14 @@ checkout logic set atlernates to point to original repo update/create the desired branch on checkout - git-update-ref refs/heads/<branch> <commit> + git-update-ref refs/heads/ checkout the desired branch - git-checkout -q -f <branch> + git-checkout -q -f create HEAD - -IDEA: + +IDEA: IDEAS: @@ -1495,7 +1337,7 @@ TODO * test direct alternates creation git-init -echo /path/to/orig/.git/objects > .git/objects/info/alternates +echo /path/to/orig/.git/objects > .git/objects/info/alternates git-update-ref refs/heads/devel 0fb016541ae7608a137c9bdf1d047e87e7a6991b git-checkout -q -f devel @@ -1505,11 +1347,11 @@ git-checkout -q -f devel - -GOTCHA: rev-parse tag^0 != rev-parse tag + +GOTCHA: rev-parse tag^0 != rev-parse tag the first returns the commit pointed to by the tag, the second returns the tag object - -detect logic + +detect logic arena.internals? sumo-base would work anywhere in the arena (union, internals, etc) @@ -1518,8 +1360,8 @@ logic we need to open the arena in the checkout directory we to adjust the directory we scan - -Q: how do we handle subpools? + +Q: how do we handle subpools? A: we'll need to show the full stock path, not just the name package=version stock_path[#branch] relative_path @@ -1537,27 +1379,27 @@ extend pool-info --subpools show sub-poolsl --buildroot show buildroot --pkgcache show package cache - --source-versions show source versions -StockSubpool should detect circular dependencies + --source-versions show source versions +StockSubpool should detect circular dependencies if its link is in the recursed_paths register/unregister Q: can a new-class object inherit from an old-style object? - -slowing everything down (even pool-info) + +slowing everything down (even pool-info) magical object with cache or set it to None on initialisation and init on demand - ---registered (default) + +--registered (default) shows stocks and subpools --stocks only shows stocks --subpools - only shows subpools -cli syntax + only shows subpools +cli syntax pool-info-build source[=version] will return info on built package @@ -1601,30 +1443,26 @@ A: E: try changing the Package: to be different from Source: in hello-debhelper dpkg -f package.deb - R: Source -logic + R: Source +logic lookup the binary if the binary has a source, lookup the source else the source is the binary name -TEST: with hello-debhelper -> foo - - - - - -SUMMARY: +TEST: with hello-debhelper -> foo + +SUMMARY: version format [epoch:]upstream_version[-debian_revision convention to start deban_revision at 1 each time upstream version increases the following don't work - 0turnkey-<revision> - 0-turnkey-<revision> + 0turnkey- + 0-turnkey- only the ubuntu way works - 0turnkey<revision> + 0turnkey examples @@ -1651,21 +1489,21 @@ TEST: with hello-debhelper -> foo dpkg --compare-versions 1-1- lt 1-2; echo $? 1 explanation - 1-1 > 1 + 1-1 > 1 IDEA: recreate comparison algorithm - -dpkg --compare-versions 0-2010.10.5-d6cbb928 lt 0-2010.10.10-a9ee521c; echo $? + +dpkg --compare-versions 0-2010.10.5-d6cbb928 lt 0-2010.10.10-a9ee521c; echo $? isolated bug: - "10" < "5" + "10" < "5" -compare("10", 5") == -1 -IDEAS: +compare("10", 5") == -1 +IDEAS: compare with dpkg --compare-versions based implementation 272 per/sec with subprocess.call 227 per/sec with os.system @@ -1707,17 +1545,15 @@ optimization bottom line before: 3K/sec after: 44K/sec (x14 improvement) -python /usr/lib/python2.4/profile.py -s cumulative ./debversion.py -INSIGHT: we don't need normalize at all! +python /usr/lib/python2.4/profile.py -s cumulative ./debversion.py +INSIGHT: we don't need normalize at all! + - -. > ~ +. > ~ non-letters should sort earlier than letters - 'a' < '@' - - - -IDEAS + 'a' < '@' + +IDEAS PackageCache should precache contents of .pool/pkgcache PackageCache.getpath is where most of the time is being spent @@ -1728,8 +1564,8 @@ non-letters should sort earlier than letters Q: why is providing a list so much slower? A: because the list is verified through pool.exists - -logic + +logic clear everything out of the cache except binary packages/versions that we have sources for create a whitelist of package versions, delete from the cache everything not on the whitelist @@ -1753,12 +1589,12 @@ alternative names - -auto-cleaning + +auto-cleaning won't work for binary imports you'll need to invoke pool-gc to clean up those - -logic + +logic create cache at init append cache on add remove from cache on remove @@ -1766,10 +1602,10 @@ alternative names data structure map_filenames - (name, version) => filename + (name, version) => filename map_versions - name => [version1, version2...] + name => [version1, version2...] * before @@ -1802,9 +1638,8 @@ data structure real 0m0.156s user 0m0.112s sys 0m0.008s - - -test cases + +test cases pool-info-build source pool-info-build source=1.2.3 @@ -1837,8 +1672,8 @@ rewritten logic - -if you add a package to the control file: + +if you add a package to the control file: pool-list will show versions of the package that never existed if you remove a package from the control file: @@ -1847,9 +1682,8 @@ if you remove a package from the control file: workaround: don't add/remove packages in an existing project - change the name? so it shows autoversions for packages that never existed -and if you change the packag - -logic +and if you change the packag +logic if running as root drop privileges to user that owns the pool directory @@ -1891,8 +1725,8 @@ to drop privileges permanently os.setreuid(1000, 1000) to reset effective uid = 0, real uid != 0 (simulate suid): - os.setreuid(1000, 0) -IDEAS + os.setreuid(1000, 0) +IDEAS perform all operations via subpool test stock sources should be part of the suite buildroot should also be part of the test suite @@ -1921,7 +1755,7 @@ unregister info all options -exists package && exists package=version +exists package && exists package=version binary that exists source version source version that doesn't exist @@ -1952,8 +1786,8 @@ garblage collection commit to all stocks and see if the new version is registered - -SUMMARY + +SUMMARY not important enough to warrant the extra complexity IDEAS @@ -1979,18 +1813,18 @@ Stock.unregister - -maybe start with plain test and move up? - -PROBLEM: the data structures aren't being updated by sync because they're in a subprocess + +maybe start with plain test and move up? + +PROBLEM: the data structures aren't being updated by sync because they're in a subprocess SOLUTION: we need to reload them in the parent process test pool-unregister /turnkey/projects/pyproject/; pool-register /turnkey/projects/pyproject/; pool-info --source-versions - -INSIGHT: implementing this would allow us to eliminate a use case for garbage collection + +INSIGHT: implementing this would allow us to eliminate a use case for garbage collection when we unregister a stock it would clear any binaries from that stock don't need to run gc @@ -2004,8 +1838,8 @@ A: yes they are IDEAS: same principle as source versions but only for binaries? - -* IDEAS + +* IDEAS experiment with a package that has a version in its epoch when we import a package into the cache, set the filename to includes epoch appropriately @@ -2050,8 +1884,8 @@ TEST: - -SOLUTION: + +SOLUTION: import them as udeb into the pool when we export them, export them as udeb @@ -2066,8 +1900,8 @@ they have exactly the same structure as regular debs TEST: pool-get . lsb-release-udeb - -SUMMARY: + +SUMMARY: previously using debinfo, the first time is more expensive @@ -2189,8 +2023,8 @@ R: sys 0m3.224s - -* SCRATCH + +* SCRATCH support generic keys identify when the key is already a hash @@ -2217,10 +2051,10 @@ file structure contians value -with - ncalls tottime percall cumtime percall filename:lineno(function) +with + ncalls tottime percall cumtime percall filename:lineno(function) 1 0.000 0.000 5.670 5.670 profile:0(command.module.main()) - 1 0.000 0.000 5.670 5.670 <string>:1(?) + 1 0.000 0.000 5.670 5.670 :1(?) 1 0.040 0.040 5.670 5.670 cmd_list.py:51(main) 1 0.000 0.000 5.550 5.550 cmd_list.py:44(list_packages) 1 0.000 0.000 5.350 5.350 pool.py:529(wrapper) @@ -2244,12 +2078,12 @@ with 9101 0.130 0.000 0.130 0.000 :0(split) - -before optimization: 3.5K/sec + +before optimization: 3.5K/sec after: 23K/sec - -should save us quite a bit of IO time + +should save us quite a bit of IO time but we need to extract it from Python @@ -2264,13 +2098,13 @@ firefox E: does extracting the control file from a large deb take longer than extracting it from a small deb? R: no -ar -p linux-hardened-image-2.6.19-7-grsec-386_2.6.19-7.10.2_i386.deb control.tar.gz > control.tar.gz +ar -p linux-hardened-image-2.6.19-7-grsec-386_2.6.19-7.10.2_i386.deb control.tar.gz > control.tar.gz RESOURCE: ar(5) - -md5 is much faster -SUMMARY + +md5 is much faster +SUMMARY Python implementation of extraction: 4257 per/sec (non-optimized) 20X faster than executing ar @@ -2279,9 +2113,9 @@ RESOURCE: ar(5) starts with - !<arch>\n + !\n - filename/ <space> <space> <space> timestamp <space> <space> + filename/ timestamp Field Offset from Field Offset to Field Name Field Format 0 15 File name ASCII @@ -2337,21 +2171,21 @@ A: 0x8 0x44 Q: is extracting the control file from a large deb file any slower than extracting the control file from a small deb file? A: no - -IDEAS + +IDEAS do everything as a function instead of a class inline no error checking only parse the fields we need - -Python implementation shorter than command execution + +Python implementation shorter than command execution 4 lines of code after writing ar.py module performance improvement, depends on size of extracted control.tar.gz: with a 22K control.tar.gz: 2X improvement - with a >1K control.tar.gz: 7X improvement + with a >1K control.tar.gz: 7X improvement DISCOVERY: tarfile is implemented in Python! may be implemented in C in the future @@ -2364,14 +2198,14 @@ E: how much of an improvement if we turn on psyco? - -use case: Alon wants to figure out from which stock the aufs-modules package came from, how does he do this? - -SUMMARY: +use case: Alon wants to figure out from which stock the aufs-modules package came from, how does he do this? + + +SUMMARY: on unregister, add binaries to package list - on gc, add binaries in stocks in whitelist -SUMMARY + on gc, add binaries in stocks in whitelist +SUMMARY the overhead is due to extensive forking process creation overhead for each method invocation @@ -2420,8 +2254,8 @@ Q: what dropped pool methods does cmd_get call? shouldn't be because it doesn't sync getpath_deb - forked each time -Q: should we use XMLRPC? + forked each time +Q: should we use XMLRPC? protocol: @@ -2487,8 +2321,8 @@ forks child process creates instance, runs server parent process return dummy proxy instance - -LOGIC + +LOGIC add get/set calls if we get a method back, then bind a proxy attribute otherwise pass the value back @@ -2509,12 +2343,12 @@ protocol op, args (attr, val) - -rational: would make it much easier to debug issues inside forked code if we had a full backtrace + +rational: would make it much easier to debug issues inside forked code if we had a full backtrace GOTCHA: tracebacks can not be serialized. damn - we can print them out though -optimize if needed + we can print them out though +optimize if needed cpickle instead of pickle benchmarks for forked_constructor @@ -2527,8 +2361,8 @@ benchmarks for forked_func runs 236/sec (33X slower than forked_constructor) IDEA: - add examples -LOGIC + add examples +LOGIC create a pool instance if we're running as root and the pool instance is not owned by root wrap a dummy constructor which: @@ -2546,11 +2380,11 @@ ROADMAP fix cmd_get, then fix cmd_info at the end, pool has to pass regression tests - -LOGIC + +LOGIC create paths -RESOURCE: Python reference->index->__new__ +RESOURCE: Python reference->index->__new__ __new__( cls[, ...]) Called to create a new instance of class cls. __new__() is a static method (special-cased so you need not declare it as such) that takes the class of which an instance was requested as its first argument. The remaining arguments are those passed to the object constructor expression (the call to the class). The return value of __new__() should be the new object instance (usually an instance of cls). @@ -2561,15 +2395,15 @@ __new__( cls[, ...]) If __new__() does not return an instance of cls, then the new instance's __init__() method will not be invoked. - __new__() is intended mainly to allow subclasses of immutable types (like int, str, or tuple) to customize instance creation. -GOTCHA: yield doesn't work + __new__() is intended mainly to allow subclasses of immutable types (like int, str, or tuple) to customize instance creation. +GOTCHA: yield doesn't work pool-info --build-logs - -hack regression suite to support a pre-setup test pool - -PROBLEM: +hack regression suite to support a pre-setup test pool + + +PROBLEM: pool.subpools is returning live Pool objects from the forked process to the root process so when we sync a subpool, its being synced with root privileges @@ -2595,21 +2429,21 @@ IDEAS how? wrap proxy client's subpools? - -usage scenario + +usage scenario I want to resolve build dependencies from a pool I want to install packages live during development from the pool - -IDEA: instead of using info --stock-sources and --stock-binaries to list all sources, + +IDEA: instead of using info --stock-sources and --stock-binaries to list all sources, have a command that shows the stock for a specific package (latest version if no version is specified) pool-info-origin package[=version] - -PoolKernel + +PoolKernel Pool register unregister @@ -2628,8 +2462,8 @@ get * IDEAS -cmd_info.py: pool -> kernel -drop_privileges -> os_drop_privileges +cmd_info.py: pool -> kernel +drop_privileges -> os_drop_privileges create option NOT to retain privileges so we can drop privileges globally @@ -2642,8 +2476,8 @@ makerelative from Paths - -GOTCHA: how do we support non-match warnings? + +GOTCHA: how do we support non-match warnings? we could just print the warning from filter_packages @@ -2656,7 +2490,4 @@ returns a data structure inherited from list with .missing attribute -raises - - - +raises From 4313283f7c4b329ddbfc2bae4b925c58750f1071 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 4 Jun 2026 12:16:46 +1000 Subject: [PATCH 2/4] Update readme - old doc files still need cleaning/removing --- README.rst | 252 ++++++++++++++++++++++++++++------------------------- 1 file changed, 131 insertions(+), 121 deletions(-) diff --git a/README.rst b/README.rst index 634a010..494fa7d 100644 --- a/README.rst +++ b/README.rst @@ -4,93 +4,88 @@ Maintain a pool of packages from source and binary stocks Introduction ------------ -Pool is a system that maintains a pool of binary Debian packages which may -be either imported as-is from a registered collection of binaries, or -built on-demand from a registered source or collection of sources. +Pool is a system that maintains a pool of binary Debian packages which may be +either imported as-is from a registered collection of binaries, or built +on-demand from a registered source or collection of sources. -The dominant abstraction for the pool is a virtual filesystem folder -that you can get get specific versions of specific packages from. Behind -the scenes, the pool will build those binary packages from source if -required and cache the built binaries for future access. +The dominant abstraction for the pool is a virtual filesystem folder that +supports getting specific versions of specific packages. Behind the scenes, the +pool will build those binary packages from source if required (and source is +registered) and cache the built binaries for future access. Terminology / architecture -------------------------- -pool - A pool of packages from source and binary "stocks". +Pool: + A pool of packages from source and binary "stocks". -stock - a container for packages in binary or source form +Stock: + A directory containing the source of package/s (not necessarily package + source code). -stock types - git single - git repository containing one single package +Stock types: + Git single: + Git repository containing one single package. + Git: + Git repository containing multiple packages. + Plain: + Regular directory containing multiple packages. + Sub-pool: + Another pool. - git - git repository containing multiple packages +Binary: + A pre-built binary Debian package (aka deb file) inside a stock. - plain - regular directory containing multiple packages +Source: + Debian package source code - i.e. directory containing source code and a + debian/ subdirectory - inside a stock. - subpool - another pool - -binary - a binary Debian package inside a stock - -source - Debian package source code - inside a stock - -build root - a chroot environment with an suitable tool-chain setup for building +Buildroot: + A chroot environment with an suitable tool-chain set up for building packages; e.g. https://github.com/turnkeylinux/buildroot -build logs - the output from the build process for a package built from source +Build logs: + The output from the build process for a package built from source. -package cache - a cache of binary packages either built from source or imported as-is +Package cache: + A cache of binary packages either built from source or imported as-is. File/data structure ------------------- -All of pool's internals are neatly tucked out of sight in a special "hidden" -directory. The pool has no data formats/configuration files. All of its "data" -is stored directly as filesystem constructs:: +The pool has no data formats/configuration files. All of its "data" is stored +directly as filesystem constructs within a "hidden" directory:: .pool/ - build/ - buildinfo/ - __.buildinfo - # info about the build env - root -> /path/to/buildroot - # symbolic link - logs/ - _.build - # log of the build process - - pkgcache/ - __.deb - # maybe in a pool-like tree - - srcpkgcache/ - __.tar.gz - # package source archive; requires --source switch - - stocks/ - #/ - # if stock is git repo of package source - link -> /path/to/stock - # symbolic link to the stock - CHECKOUT - # local clone of git source branch - SYNC_HEAD - # text file containing commit id of HEAD (full SHA) - index-sources - - # text file containing available package version/s (one - # version per line) - version/s calculated using - # 'autoversion' + |-- build/ + | |-- buildinfo/ + | | `-- __.buildinfo + | | # info about the build env + | |-- root -> /path/to/buildroot + | | # symbolic link + | `-- logs/ + | `-- _.build + | # log of the build process + |-- pkgcache/ + | `-- __.deb + | # maybe in a pool-like tree + |-- srcpkgcache/ + | `-- __.tar.gz + | # package source archive; requires --source switch + `-- stocks/ + `-- #/ + | # if stock is git repo of package source + |-- link -> /path/to/stock + | # symbolic link to the stock + |-- CHECKOUT + | # local clone of git source branch + |-- SYNC_HEAD + | # text file containing commit id of HEAD (full SHA) + `-- index-sources + `-- + # text file containing available package version/s (one + # version per line) - version/s calculated using + # 'autoversion' Usage ----- @@ -99,11 +94,7 @@ Syntax: pool [options] Maintain a pool of Debian packages from source -Commands: - see below - - note: commands can be either separated with a space or a dash (as shown - below) +Note that commands can be either separated with a space or a dash. Global options: -h, --help show global/general help message and exit @@ -118,32 +109,43 @@ Environment variables:: Initialize a new pool ''''''''''''''''''''' -pool-init /path/to/buildroot +Create a pool capable of building packages from source:: + + pool-init /path/to/buildroot + +Create a pool that only supports plain and sub-pool stocks:: + + pool-init --no-buildroot Register a package stock into the pool '''''''''''''''''''''''''''''''''''''' -pool-register /path/to/stock[#branch] +Make package stocks available:: -Stock type can be: + pool-register /path/to/stock[#branch] -* another pool (warning - watch out for circular dependencies) -* /path/to/git_repository[#branch] -* /path/to/regular_directory +A stock type can be: +- /path/to/git_repository[#branch] +- /path/to/regular_directory +- another pool (warning - watch out for circular dependencies) Unregister a package stock from the pool '''''''''''''''''''''''''''''''''''''''' -pool-unregister /path/to/stock[#branch] +Remove stock:: -* only relevant content of .pool/stock is removed; cached packages in - .pool/pkgcache are NOT removed -* cached packages can be removed by running a garbage collect - see 'pool-gc' + pool-unregister /path/to/stock[#branch] + +Note: Only the relevant content of .pool/stock is removed; cached packages in + .pool/pkgcache are NOT removed. Cached packages can be removed by running + a garbage collect - see 'pool-gc' Print pool info ''''''''''''''' -pool-info [options] +Get info about package stocks and other pool config:: + + pool-info [options] Options:: @@ -163,27 +165,31 @@ Options:: Print binary package build log '''''''''''''''''''''''''''''' -pool-info-get +Get build log of specific package:: -* no info if package hasn't been built -* error if package doesn't exist + pool-info-get + +No info if package hasn't been built. Error if package doesn't exist. Check if package exists in pool ''''''''''''''''''''''''''''''' -pool-exists package[=version] +Simple check of package availability:: + + pool-exists package[=version] -Prints true/false if exists in the pool. -If true exitcode = 0, else exitcode = 1 +Prints true/false if exists in the pool. The exitcode matches the +status; 0 if True, else 1. List packages in pool ''''''''''''''''''''' -pool-list [ "" ] +List some/all packages - and their versions:: + + pool-list [ "" ] -If is provided, print only those packages whose names -match the glob otherwise, by default, print a list of the newest -packages. +By default all packages and their latest version are displayed. If + is provided, only the matching packages are returned. Options:: @@ -194,17 +200,22 @@ Options:: print only the names of packages in the pool (without the list) incompatible with -a option -* quoting is important to ensure that it is not expanded by the - shell +Quoting is important to ensure that it is not expanded by the +shell. + +Use --all-versions with caution when numerous git stocks are registered. Pool +considered every commit as a virtual version so the output may be very large. Get packages from pool '''''''''''''''''''''' -pool-get [-options] [ package[=version] ... ] +Get package/s from registered stock/s:: + + pool-get [-options] [ package[=version] ... ] -If a package is specified without a version, get the newest package. -If no packages are specified as arguments, get all the newest packages. -Summary of success/failure of package/s is shown on completion. +If a package is specified without a version, get the newest package. If no +packages are specified as arguments, get all the newest packages. Summary of +success/failure of package/s is shown on completion. Options:: @@ -227,12 +238,14 @@ Options:: Garbage collect stale cached data ''''''''''''''''''''''''''''''''' -pool-gc [ -options ] +Clean pool's package cache:: + + pool-gc [ -options ] Stale data includes: -A) A binary in the package cache that does not belong in any of the - registered stocks. +A) A binary in the package cache that does not belong in any of the registered + stocks. This includes binary packages which have since been removed from a registered stock. @@ -254,19 +267,19 @@ Example usage session cd private # initialize a new pool - pool-init /chroots/rocky-build + pool-init /chroots/my_buildroot - for p in /turnkey/projects/*; do + for pkg in /turnkey/projects/*; do # auto identifies the type of the stock we register - pool-register $p + pool-register $pkg done - + + # show pool information including registered stocks, etc. pool-info - show pool information (registered containers, etc.) - # woops, noticed I registered the wrong branch - # added #devel branch for emphasis - unregister would work without it - # since there is only one branch registered for that path + # woops, acidentally registered the wrong branch - added #devel branch for + # emphasis - unregister would work without it since there is only one + # branch registered for that path pool-unregister /turnkey/projects/pool#devel # print a list of all packages in the pool (by name only) @@ -276,29 +289,26 @@ Example usage session pool-list # print a list of all packagse that match this glob - pool-list turnkey-* + pool-list "turnkey-*" - # print a list of all package versions for neverland + # print a list of all package versions for package named neverland pool-list --all neverland # print a loooong list of all package versions, old and new, for all # packages - # watch out, every git commit in an autoversioned project is a new virtual - # version! pool-list --all for name in $(pool-list -n); do - if ! exists -q $name; then + if ! pool-exists -q "$name"; then echo "insane: package $name was just here a second ago" fi done - mkdir /tmp/newest - # get all the newest packages in the pool to /tmp/newest + mkdir /tmp/newest pool-get /tmp/newest - # get the newest neverland to /tmp/newest + # get the latest neverland package into /tmp/newest pool-get /tmp/newest neverland # get neverland 1.2.3 specifically to /tmp/newest @@ -309,10 +319,10 @@ Example usage session # (unsafe) pool-get /tmp/newest -q -i /path/to/product-manifest - # creates a Debian apt repository like filesystem tree + # create a Debian apt repository like filesystem tree mkdir /tmp/product-repo for package in $(cat /path/to/versioned-product-manifest); do - if pool-exists -q $package; then + if pool-exists -q "$package"; then pool-get /tmp/product-repo --tree -s $package fi done From c7615ea3e5b7c283d75743d45af619decb03dd0c Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Fri, 5 Jun 2026 13:45:53 +1000 Subject: [PATCH 3/4] Move release notes to their own dedicated dir --- {docs => release-notes}/RelNotes-0.9.1.txt | 0 {docs => release-notes}/RelNotes-0.9.2.txt | 0 {docs => release-notes}/RelNotes-0.9.txt | 0 {docs => release-notes}/RelNotes-1.0.1.txt | 0 {docs => release-notes}/RelNotes-1.0.txt | 0 {docs => release-notes}/RelNotes-1.2.0.txt | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename {docs => release-notes}/RelNotes-0.9.1.txt (100%) rename {docs => release-notes}/RelNotes-0.9.2.txt (100%) rename {docs => release-notes}/RelNotes-0.9.txt (100%) rename {docs => release-notes}/RelNotes-1.0.1.txt (100%) rename {docs => release-notes}/RelNotes-1.0.txt (100%) rename {docs => release-notes}/RelNotes-1.2.0.txt (100%) diff --git a/docs/RelNotes-0.9.1.txt b/release-notes/RelNotes-0.9.1.txt similarity index 100% rename from docs/RelNotes-0.9.1.txt rename to release-notes/RelNotes-0.9.1.txt diff --git a/docs/RelNotes-0.9.2.txt b/release-notes/RelNotes-0.9.2.txt similarity index 100% rename from docs/RelNotes-0.9.2.txt rename to release-notes/RelNotes-0.9.2.txt diff --git a/docs/RelNotes-0.9.txt b/release-notes/RelNotes-0.9.txt similarity index 100% rename from docs/RelNotes-0.9.txt rename to release-notes/RelNotes-0.9.txt diff --git a/docs/RelNotes-1.0.1.txt b/release-notes/RelNotes-1.0.1.txt similarity index 100% rename from docs/RelNotes-1.0.1.txt rename to release-notes/RelNotes-1.0.1.txt diff --git a/docs/RelNotes-1.0.txt b/release-notes/RelNotes-1.0.txt similarity index 100% rename from docs/RelNotes-1.0.txt rename to release-notes/RelNotes-1.0.txt diff --git a/docs/RelNotes-1.2.0.txt b/release-notes/RelNotes-1.2.0.txt similarity index 100% rename from docs/RelNotes-1.2.0.txt rename to release-notes/RelNotes-1.2.0.txt From 9f62bb54d6cf882cada65e5cd7f2329ab7a79463 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Fri, 5 Jun 2026 13:48:26 +1000 Subject: [PATCH 4/4] Updated v1.2.0 release notes, ready to publish stable release :) --- release-notes/RelNotes-1.2.0.txt | 61 ++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/release-notes/RelNotes-1.2.0.txt b/release-notes/RelNotes-1.2.0.txt index 8b9c227..ebd9040 100644 --- a/release-notes/RelNotes-1.2.0.txt +++ b/release-notes/RelNotes-1.2.0.txt @@ -1,21 +1,52 @@ v1.2.0 Release Notes ==================== -* new options for improved "preserve buildroot" behavior: +* Bugfixes: + - pool-get: + - Fix building multiple packages; continue with other packages on + failure. + - Ensure always non-zero exit if errors and '--strict'. + - Bugfix stock sub-path regression. + - Various minor bugfixes. - --preserve-buildroot-on-error (default) - --preserve-buildroot-always - --preserve-buildroot-never +* Features: + - Support for a pool without a buildroot (e.g. if only using stocks of + pre-built packages and/or other pools). + - Integrate pool-refresh into the main pool app. + - pool-get: + - New options for improved "preserve buildroot" behavior: + '--preserve-buildroot-on-error' (default) + '--preserve-buildroot-always' + '--preserve-buildroot-never' + - Exit with build summary noting success/failure and cause i.e. + either "missing stock" or "build failure". + - pool-register/pool-unregister/pool-refresh + - Show path to stock source dir (with # suffix if relevant) when + working with stocks. + - "Pretty" error messages - instead of stacktrace. + - Provide useful error message if a buildroot is regitered and it's not + actually a buildroot (default behavior on initial install of pool). + - Logging/debugging output. + - Build summary output. -* display build summary on exit - - show package status - success or failure - - if failure; note cause (failed build or missing stock) +* Code quality & cleanup: + - Update readme - remove redundant/outdated info & document new/updated + features. + - Linting fixes. + - Typing increased coverage, updates & improvements. + - Special thanks to Marcos for your input! :) + https://github.com/marcos-mendez + - Standardised formatting. + - Update tests to python3 - Special thanks to Marcos! :) + https://github.com/marcos-mendez + - Updated gitignore to ignore common build artifacts. + - Add Stock.__repr__() dunder method (useful for debugging). + - Replace c style string formating with f-strings. -* 'pool-get' bugfix when "getting" multiple packages: - - if package fails, continue with other packages - - was exiting immediately if a package failed - -* code tidying and improvement: - - linting: fixes - - formatting: improved consistency and better readability - - typing: fixes, modernization and additional coverage +* Packaging: + - Update package arch to 'all'. + - Update package dependencies. + - Linting debian package scripts. + - Migrate from 'setup.py' to 'pyproject.toml' install. + - Include 'py.typed' in package. + - Support reproducable builds.