Do not add a new Inotify watchers on timer
[ganeti-github.git] / devel / build_chroot
1 #!/bin/bash
2
3 #Requirements for this script to work:
4 #* Make sure that the user who uses the chroot is in group 'src', or change
5 #  the ${GROUP} variable to a group that contains the user.
6 #* Add any path of the host system that you want to access inside the chroot
7 #  to the /etc/schroot/default/fstab file. This is important in particular if
8 #  your homedir is not in /home.
9 #* Add this to your /etc/fstab:
10 #  tmpfs /var/lib/schroot/mount tmpfs defaults,size=3G 0 0
11 #  tmpfs /var/lib/schroot/unpack tmpfs defaults,size=3G 0 0
12
13 #Configuration
14 : ${ARCH:=amd64}
15 : ${DIST_RELEASE:=wheezy}
16 : ${CONF_DIR:=/etc/schroot/chroot.d}
17 : ${CHROOT_DIR:=/srv/chroot}
18 : ${ALTERNATIVE_EDITOR:=/usr/bin/vim.basic}
19 : ${CHROOT_FINAL_HOOK:=/bin/true}
20 : ${GROUP:=src}
21 # Additional Variables taken from the environmen
22 # DATA_DIR
23 # CHROOT_EXTRA_DEBIAN_PACKAGES
24
25 #Automatically generated variables
26 CHROOTNAME=$DIST_RELEASE-$ARCH
27 CHNAME=building_$CHROOTNAME
28 TEMP_CHROOT_CONF=$CONF_DIR/$CHNAME.conf
29 FINAL_CHROOT_CONF=$CHROOTNAME.conf
30 ROOT=`pwd`
31 CHDIR=$ROOT/$CHNAME
32 USER=`whoami`
33 COMP_FILENAME=$CHROOTNAME.tar.gz
34 COMP_FILEPATH=$ROOT/$COMP_FILENAME
35 TEMP_DATA_DIR=`mktemp -d`
36 ACTUAL_DATA_DIR=$DATA_DIR
37 ACTUAL_DATA_DIR=${ACTUAL_DATA_DIR:-$TEMP_DATA_DIR}
38 GHC_VERSION="7.6.3"
39 CABAL_INSTALL_VERSION="1.18.0.2"
40 SHA1_LIST='
41 cabal-install-1.18.0.2.tar.gz 2d1f7a48d17b1e02a1e67584a889b2ff4176a773
42 ghc-7.6.3-i386-unknown-linux.tar.bz2 f042b4171a2d4745137f2e425e6949c185f8ea14
43 ghc-7.6.3-x86_64-unknown-linux.tar.bz2 46ec3f3352ff57fba0dcbc8d9c20f7bcb6924b77
44 '
45
46 # export all variables needed in the schroot
47 export ARCH GHC_VERSION CABAL_INSTALL_VERSION SHA1_LIST
48
49 # Use gzip --rsyncable if available, to speed up transfers of generated files
50 # The environment variable GZIP is read automatically by 'gzip',
51 # see ENVIRONMENT in gzip(1).
52 gzip --rsyncable </dev/null >/dev/null 2>&1 && export GZIP="--rsyncable"
53
54 #Runnability checks
55 if [ $USER != 'root' ]
56 then
57   echo "This script requires root permissions to run"
58   exit
59 fi
60
61 if [ -f $TEMP_CHROOT_CONF ]
62 then
63   echo "The configuration file name for the temporary chroot"
64   echo "  $TEMP_CHROOT_CONF"
65   echo "already exists."
66   echo "Remove it or change the CHNAME value in the script."
67   exit
68 fi
69
70 #Create configuration dir and files if they do not exist
71 if [ ! -d $ACTUAL_DATA_DIR ]
72 then
73   mkdir $ACTUAL_DATA_DIR
74   echo "The data directory"
75   echo "  $ACTUAL_DATA_DIR"
76   echo "has been created."
77 fi
78
79 if [ ! -f $ACTUAL_DATA_DIR/final.schroot.conf.in ]
80 then
81   cat <<END >$ACTUAL_DATA_DIR/final.schroot.conf.in
82 [${CHROOTNAME}]
83 description=Debian ${DIST_RELEASE} ${ARCH}
84 groups=${GROUP}
85 source-root-groups=root
86 type=file
87 file=${CHROOT_DIR}/${COMP_FILENAME}
88 END
89   echo "The file"
90   echo " $ACTUAL_DATA_DIR/final.schroot.conf.in"
91   echo "has been created with default configurations."
92 fi
93
94 if [ ! -f $ACTUAL_DATA_DIR/temp.schroot.conf.in ]
95 then
96   cat <<END >$ACTUAL_DATA_DIR/temp.schroot.conf.in
97 [${CHNAME}]
98 description=Debian ${DIST_RELEASE} ${ARCH}
99 directory=${CHDIR}
100 groups=${GROUP}
101 users=root
102 type=directory
103 END
104   echo "The file"
105   echo " $ACTUAL_DATA_DIR/temp.schroot.conf.in"
106   echo "has been created with default configurations."
107 fi
108
109 #Stop on errors
110 set -e
111
112 #Cleanup
113 rm -rf $CHDIR
114 mkdir $CHDIR
115
116 #Install tools for building chroots
117 apt-get install -y schroot debootstrap
118
119 shopt -s expand_aliases
120 alias in_chroot='schroot -c $CHNAME -d / '
121 function subst_variables {
122   sed \
123     -e "s/\${ARCH}/$ARCH/" \
124     -e "s*\${CHDIR}*$CHDIR*" \
125     -e "s/\${CHNAME}/$CHNAME/" \
126     -e "s/\${CHROOTNAME}/$CHROOTNAME/" \
127     -e "s*\${CHROOT_DIR}*$CHROOT_DIR*" \
128     -e "s/\${COMP_FILENAME}/$COMP_FILENAME/" \
129     -e "s/\${DIST_RELEASE}/$DIST_RELEASE/" $@
130 }
131
132 #Generate chroot configurations
133 cat $ACTUAL_DATA_DIR/temp.schroot.conf.in | subst_variables > $TEMP_CHROOT_CONF
134 cat $ACTUAL_DATA_DIR/final.schroot.conf.in | subst_variables > $FINAL_CHROOT_CONF
135
136 #Install the base system
137 debootstrap --arch $ARCH $DIST_RELEASE $CHDIR
138
139 APT_INSTALL="apt-get install -y --no-install-recommends"
140
141 if [ $DIST_RELEASE = squeeze ]
142 then
143   echo "deb http://backports.debian.org/debian-backports" \
144        "$DIST_RELEASE-backports main contrib non-free" \
145        > $CHDIR/etc/apt/sources.list.d/backports.list
146 fi
147
148 #Install all the packages
149 in_chroot -- \
150   apt-get update
151
152
153 # Functions for downloading and checking Haskell core components.
154 # The functions run commands within the schroot.
155
156 # arguments : file_name expected_sha1
157 function verify_sha1 {
158   local SUM="$( in_chroot -- sha1sum "$1" | awk '{print $1;exit}' )"
159   if [ "$SUM" != "$2" ] ; then
160     echo "ERROR: The SHA1 sum $SUM of $1 doesn't match $2." >&2
161     return 1
162   else
163     echo "SHA1 of $1 verified correct."
164   fi
165 }
166
167 # arguments: URL
168 function lookup_sha1 {
169   grep -o "${1##*/}"'\s\+[0-9a-fA-F]*' <<<"$SHA1_LIST" | awk '{print $2;exit}'
170 }
171
172 # arguments : file_name URL
173 function download {
174   local FNAME="$1"
175   local URL="$2"
176   in_chroot -- wget --no-check-certificate --output-document="$FNAME" "$URL"
177   verify_sha1 "$FNAME" "$( lookup_sha1 "$URL" )"
178 }
179
180 function install_ghc {
181   local GHC_ARCH=$ARCH
182   local TDIR=$( schroot -c $CHNAME -d / -- mktemp -d )
183   [ -n "$TDIR" ]
184   if [ "$ARCH" == "amd64" ] ; then
185     download "$TDIR"/ghc.tar.bz2 \
186       http://www.haskell.org/ghc/dist/${GHC_VERSION}/ghc-${GHC_VERSION}-x86_64-unknown-linux.tar.bz2
187   elif [ "$ARCH" == "i386" ] ; then
188     download "$TDIR"/ghc.tar.bz2 \
189       http://www.haskell.org/ghc/dist/${GHC_VERSION}/ghc-${GHC_VERSION}-i386-unknown-linux.tar.bz2
190   else
191     echo "Don't know what GHC to download for architecture $ARCH" >&2
192     return 1
193   fi
194   schroot -c $CHNAME -d "$TDIR" -- \
195     tar xjf ghc.tar.bz2
196   schroot -c $CHNAME -d "$TDIR/ghc-${GHC_VERSION}" -- \
197     ./configure --prefix=/usr/local
198   schroot -c $CHNAME -d "$TDIR/ghc-${GHC_VERSION}" -- \
199      make install
200   schroot -c $CHNAME -d "/" -- \
201     rm -rf "$TDIR"
202 }
203
204 function install_cabal {
205   local TDIR=$( schroot -c $CHNAME -d / -- mktemp -d )
206   [ -n "$TDIR" ]
207   download "$TDIR"/cabal-install.tar.gz \
208     http://www.haskell.org/cabal/release/cabal-install-${CABAL_INSTALL_VERSION}/cabal-install-${CABAL_INSTALL_VERSION}.tar.gz
209   schroot -c $CHNAME -d "$TDIR" -- \
210     tar xzf cabal-install.tar.gz
211   schroot -c $CHNAME -d "$TDIR/cabal-install-${CABAL_INSTALL_VERSION}" -- \
212     bash -c 'EXTRA_CONFIGURE_OPTS="--enable-library-profiling" ./bootstrap.sh --global'
213   schroot -c $CHNAME -d "/" -- \
214     rm -rf "$TDIR"
215 }
216
217
218 case $DIST_RELEASE in
219
220   squeeze)
221
222     # do not install libghc6-network-dev, since it's too old, and just
223     # confuses the dependencies
224     in_chroot -- \
225       $APT_INSTALL \
226         autoconf automake \
227         zlib1g-dev \
228         libgmp3-dev \
229         libcurl4-gnutls-dev \
230         libpcre3-dev \
231         happy \
232         hscolour pandoc \
233         graphviz qemu-utils \
234         python-docutils \
235         python-simplejson \
236         python-pyparsing \
237         python-pyinotify \
238         python-pycurl \
239         python-ipaddr \
240         python-yaml \
241         python-paramiko
242
243     in_chroot -- \
244       $APT_INSTALL python-setuptools python-dev build-essential
245
246     in_chroot -- \
247       easy_install \
248         logilab-astng==0.24.1 \
249         logilab-common==0.58.3 \
250         mock==1.0.1 \
251         pylint==0.26.0
252
253     in_chroot -- \
254       easy_install \
255         sphinx==1.1.3 \
256         pep8==1.3.3 \
257         coverage==3.4 \
258         bitarray==0.8.0
259
260     install_ghc
261
262     install_cabal
263
264     in_chroot -- \
265       cabal update
266
267     # sinec we're using Cabal >=1.16, we can use the parallel install option
268     in_chroot -- \
269       cabal install --global -j --enable-library-profiling \
270         attoparsec-0.11.1.0 \
271         base64-bytestring-1.0.0.1 \
272         blaze-builder-0.3.3.2 \
273         case-insensitive-1.1.0.3 \
274         Crypto-4.2.5.1 \
275         curl-1.3.8 \
276         happy \
277         hashable-1.2.1.0 \
278         hinotify-0.3.6 \
279         hscolour-1.20.3 \
280         hslogger-1.2.3 \
281         json-0.7 \
282         lifted-base-0.2.2.0 \
283         lens-4.0.4 \
284         MonadCatchIO-transformers-0.3.0.0 \
285         network-2.4.1.2 \
286         parallel-3.2.0.4 \
287         parsec-3.1.3 \
288         regex-pcre-0.94.4 \
289         temporary-1.2.0.1 \
290         vector-0.10.9.1 \
291         zlib-0.5.4.1 \
292         \
293         'hlint>=1.9.12' \
294         HUnit-1.2.5.2 \
295         QuickCheck-2.6 \
296         test-framework-0.8.0.3 \
297         test-framework-hunit-0.3.0.1 \
298         test-framework-quickcheck2-0.3.0.2 \
299         \
300         snap-server-0.9.4.0 \
301         PSQueue-1.1 \
302         \
303         cabal-file-th-0.2.3 \
304         shelltestrunner
305
306     #Install selected packages from backports
307     in_chroot -- \
308       $APT_INSTALL -t squeeze-backports \
309         git \
310         git-email \
311         vim
312
313 ;;
314
315   wheezy)
316
317     in_chroot -- \
318       $APT_INSTALL \
319       autoconf automake ghc ghc-haddock libghc-network-dev \
320       libghc-test-framework{,-hunit,-quickcheck2}-dev \
321       libghc-json-dev libghc-curl-dev libghc-hinotify-dev \
322       libghc-parallel-dev libghc-utf8-string-dev \
323       libghc-hslogger-dev libghc-crypto-dev \
324       libghc-regex-pcre-dev libghc-attoparsec-dev \
325       libghc-vector-dev libghc-temporary-dev \
326       libghc-snap-server-dev libpcre3 libpcre3-dev happy hscolour pandoc \
327       libghc-zlib-dev libghc-psqueue-dev \
328       cabal-install \
329       python-setuptools python-sphinx python-epydoc graphviz python-pyparsing \
330       python-simplejson python-pycurl python-paramiko \
331       python-bitarray python-ipaddr python-yaml qemu-utils python-coverage pep8 \
332       shelltestrunner python-dev openssh-client vim git git-email
333
334     # We need version 0.9.4 of pyinotify because the packaged version, 0.9.3, is
335     # incompatibile with the packaged version of python-epydoc 3.0.1.
336     # Reason: a logger class in pyinotify calculates its superclasses at
337     # runtime, which clashes with python-epydoc's static analysis phase.
338     #
339     # Problem introduced in:
340     #   https://github.com/seb-m/pyinotify/commit/2c7e8f8959d2f8528e0d90847df360
341     # and "fixed" in:
342     #   https://github.com/seb-m/pyinotify/commit/98c5f41a6e2e90827a63ff1b878596
343
344     in_chroot -- \
345       easy_install \
346         logilab-astng==0.24.1 \
347         logilab-common==0.58.3 \
348         mock==1.0.1 \
349         pylint==0.26.0 \
350         pep8==1.3.3
351
352     in_chroot -- \
353       easy_install pyinotify==0.9.4
354
355      in_chroot -- \
356        cabal update
357
358      in_chroot -- \
359        cabal install --global \
360         'base64-bytestring>=1' \
361         lens-3.10.2 \
362         'lifted-base>=0.1.2' \
363         'hlint>=1.9.12'
364 ;;
365
366   testing)
367
368     in_chroot -- \
369       $APT_INSTALL \
370       autoconf automake ghc ghc-haddock libghc-network-dev \
371       libghc-test-framework{,-hunit,-quickcheck2}-dev \
372       libghc-json-dev libghc-curl-dev libghc-hinotify-dev \
373       libghc-parallel-dev libghc-utf8-string-dev \
374       libghc-hslogger-dev libghc-crypto-dev \
375       libghc-regex-pcre-dev libghc-attoparsec-dev \
376       libghc-vector-dev libghc-temporary-dev \
377       libghc-snap-server-dev libpcre3 libpcre3-dev happy hscolour pandoc \
378       libghc-zlib-dev libghc-psqueue-dev \
379       libghc-base64-bytestring-dev libghc-lens-dev libghc-lifted-base-dev \
380       libghc-cabal-dev \
381       cabal-install \
382       python-setuptools python-sphinx python-epydoc graphviz python-pyparsing \
383       python-simplejson python-pycurl python-pyinotify python-paramiko \
384       python-bitarray python-ipaddr python-yaml qemu-utils python-coverage pep8 \
385       shelltestrunner python-dev pylint openssh-client vim git git-email
386
387     in_chroot -- \
388       cabal update
389
390     in_chroot -- \
391       cabal install --global \
392        'hlint>=1.9.12'
393 ;;
394
395   precise)
396     # ghc, git-email and other dependencies are hosted in the universe
397     # repository, which is not enabled by default.
398     echo "Adding universe repository..."
399     cat > $CHDIR/etc/apt/sources.list.d/universe.list <<EOF
400 deb http://archive.ubuntu.com/ubuntu precise universe
401 EOF
402     in_chroot -- \
403       apt-get update
404
405     echo "Installing packages"
406     in_chroot -- \
407       $APT_INSTALL \
408       autoconf automake ghc ghc-haddock \
409       libghc-curl-dev libghc-hinotify-dev \
410       libghc-parallel-dev libghc-utf8-string-dev \
411       libghc-crypto-dev \
412       libghc-attoparsec-dev \
413       libghc-vector-dev libghc-temporary-dev libghc-psqueue-dev \
414       libghc-cabal-dev \
415       cabal-install \
416       libpcre3 libpcre3-dev happy hscolour pandoc \
417       python-setuptools python-sphinx python-epydoc graphviz python-pyparsing \
418       python-simplejson python-pyinotify python-pycurl python-paramiko \
419       python-bitarray python-ipaddr python-yaml qemu-utils python-coverage pep8 \
420       python-dev pylint openssh-client vim git git-email \
421       build-essential
422
423     in_chroot -- \
424       easy_install \
425         logilab-astng==0.24.1 \
426         logilab-common==0.58.3 \
427         mock==1.0.1 \
428         pylint==0.26.0 \
429         pep8==1.3.3
430
431     in_chroot -- \
432       cabal update
433
434      # Precise has network-2.4.0.0, which breaks, see
435      #   https://github.com/haskell/network/issues/60
436      in_chroot -- \
437        cabal install --global \
438         'base64-bytestring>=1' \
439         hslogger-1.2.3 \
440         'hlint>=1.9.12' \
441         json-0.7 \
442         lens-3.10.2 \
443         'lifted-base>=0.1.2' \
444         'network>=2.4.0.1' \
445         'regex-pcre>=0.94.4' \
446         parsec-3.1.3 \
447         shelltestrunner \
448         'snap-server>=0.8.1' \
449         test-framework-0.8.0.3 \
450         test-framework-hunit-0.3.0.1 \
451         test-framework-quickcheck2-0.3.0.2 \
452         'transformers>=0.3.0.0'
453     ;;
454
455   *)
456     in_chroot -- \
457       $APT_INSTALL \
458       autoconf automake ghc ghc-haddock libghc-network-dev \
459       libghc-test-framework{,-hunit,-quickcheck2}-dev \
460       libghc-json-dev libghc-curl-dev libghc-hinotify-dev \
461       libghc-parallel-dev libghc-utf8-string-dev \
462       libghc-hslogger-dev libghc-crypto-dev \
463       libghc-regex-pcre-dev libghc-attoparsec-dev \
464       libghc-vector-dev libghc-temporary-dev libghc-psqueue-dev \
465       libghc-snap-server-dev libpcre3 libpcre3-dev happy hscolour pandoc \
466       libghc-lens-dev libghc-lifted-base-dev \
467       libghc-cabal-dev \
468       cabal-install \
469       libghc-base64-bytestring-dev \
470       python-setuptools python-sphinx python-epydoc graphviz python-pyparsing \
471       python-simplejson python-pyinotify python-pycurl python-paramiko \
472       python-bitarray python-ipaddr python-yaml qemu-utils python-coverage pep8 \
473       shelltestrunner python-dev pylint openssh-client vim git git-email \
474       build-essential
475
476     in_chroot -- \
477       cabal update
478
479      in_chroot -- \
480        cabal install --global \
481         'hlint>=1.9.12'
482 ;;
483 esac
484
485 # print what packages and versions are installed:
486 in_chroot -- \
487   cabal list --installed --simple-output
488
489 in_chroot -- \
490   $APT_INSTALL sudo fakeroot rsync locales less socat
491
492 # Configure the locale
493 case $DIST_RELEASE in
494   precise)
495     in_chroot -- \
496       $APT_INSTALL language-pack-en
497     ;;
498   *)
499     echo "en_US.UTF-8 UTF-8" >> $CHDIR/etc/locale.gen
500
501     in_chroot -- \
502       locale-gen
503     ;;
504 esac
505
506 in_chroot -- \
507   $APT_INSTALL lvm2 ssh bridge-utils iproute iputils-arping \
508                ndisc6 python-openssl openssl \
509                python-mock fping qemu-utils
510
511 in_chroot -- \
512   easy_install psutil
513
514 in_chroot -- \
515   easy_install jsonpointer \
516     jsonpointer \
517     jsonpatch
518
519 in_chroot -- \
520   $APT_INSTALL \
521   python-epydoc debhelper quilt
522
523 # extra debian packages
524
525 for package in $CHROOT_EXTRA_DEBIAN_PACKAGES
526 do in_chroot -- \
527   $APT_INSTALL $package
528 done
529
530 #Set default editor
531 in_chroot -- \
532   update-alternatives --set editor $ALTERNATIVE_EDITOR
533
534 # Final user hook
535
536 in_chroot -- $CHROOT_FINAL_HOOK
537
538 rm -f $COMP_FILEPATH
539 echo "Creating compressed schroot image..."
540 cd $CHDIR
541 tar czf $COMP_FILEPATH ./*
542 cd $ROOT
543
544 rm -rf $CHDIR
545 rm -f $TEMP_CHROOT_CONF
546 rm -rf $TEMP_DATA_DIR
547
548 echo "Chroot created. In order to run it:"
549 echo " * sudo cp $FINAL_CHROOT_CONF $CONF_DIR/$FINAL_CHROOT_CONF"
550 echo " * sudo mkdir -p $CHROOT_DIR"
551 echo " * sudo cp $COMP_FILEPATH $CHROOT_DIR/$COMP_FILENAME"
552 echo "Then run \"schroot -c $CHROOTNAME\""