- initial import of revision 374 from cnc
[apt.git] / tools / genbasedir
1 #!/bin/sh
2 #
3 # $Id: genbasedir,v 1.6 2002/09/04 22:48:37 niemeyer Exp $
4 #
5 # This script generates the contents of the base/ directory, by creating
6 # the pkglists and the hash file. Update the components directory to contain
7 # the components of your repository.
8 #
9  
10 usage="\
11 Usage: genbasedir [<options>] <topdir> [<comp1> [<comp2> ...]]\n\
12
13 Options:\n\
14    -s, --sign         Generate and sign hashfile\n\
15    --default-key=ID   Use ID as gnupg secret key\n\
16    --hashonly         Do hash stuff only\n\
17    --listonly         Generate pkglists/srclists and quit\n\
18    --partial          Update just some of the already existent components\n\
19    --oldhashfile      Enable generation of old hashfile\n\
20    --bz2only          Generate only compressed lists\n\
21    --progress         Show progress bars for genpkglist/gensrclist\n\
22    --updateinfo=FILE  Update information file\n\
23    --flat             Use flat repository, where SRPMS and RPMS are in\n\
24                       the topdir (SRPMS are usually in 'topdir/..')\n\
25    --bloat            Do not strip the package file list, needed for some\n\
26                       distributions that use non-automatically generated\n\
27                       file dependencies\n\
28    --meta=NAME        Create a meta repository named NAME\n\
29    --compresslevel=n  Set bzip2 compress level (1-9)\n\
30    --cachedir=DIR     Use a custom md5sum cache directory for package list\n\
31                       generation (useful for non-root users).\n\
32    -h, --help         Display this help\n\
33 \n\
34 Examples:\n\
35 \n\
36    genbasedir /home/ftp/pub/conectiva\n\
37    genbasedir /home/ftp/pub/conectiva main extra devel\n\
38 "
39
40 getsize()
41 {
42     tmp=`wc -c $1`
43     echo $tmp|cut -f1 -d\  
44 }
45
46 phashstuff()
47 {
48     size=`getsize $1`
49     md5=`md5sum $1|cut -f1 -d\  `
50     echo " $md5 $size $2"
51 }
52
53 basedir=.
54 signature=0
55 listonly=0
56 hashonly=0
57 partial=0
58 oldhashfile=0
59 mapi=0
60 bz2only=0
61 meta=
62 meta_opts=
63 updateinfo=
64 progress=
65 flat=
66 defaultkey=
67 srcidxdir=
68
69
70 # bloat is necessary for non-Conectiva distros, at least RH,
71 # because they use file dependencies with a non-predictable
72 # heuristic. So we can't strip-off paths that will probably
73 # never appear in dependencies.
74 bloat=""
75
76 while test $# -gt 0 ; do
77     case "$1" in
78     -h | --help)
79         echo -e "$usage"
80         exit 0
81     ;;
82     --mapi)
83         # hee hee hee..
84         mapi=1
85         ;;
86     --listonly)
87         listonly=1
88         ;;
89     --hashonly)
90         hashonly=1
91         ;;
92         --partial)
93                 partial=1
94         ;;
95     --oldhashfile)
96         oldhashfile=1
97         ;;
98     --bz2only)
99         bz2only=1
100     ;;
101     --append)
102         append=1
103     ;;
104     --meta=*)
105         meta=`echo $1 | sed 's/^--meta=//'`
106         meta_opts="--append --meta $meta"
107     ;;
108     --updateinfo=*)
109                 updateinfo=$1
110         ;;
111     --bloat)
112                 bloat="--bloat"
113         ;;
114     --topdir=*)
115                 # Legacy
116                 _topdir=$1
117     ;;
118     --progress)
119         progress="--progress"
120     ;;
121     --flat)
122         flat="--flat"
123     ;;
124     -s | --sign)
125                 signature=1
126         ;;
127         --default-key=*)
128                 defaultkey="\"$1\""
129         ;;
130         --compresslevel=*)
131                 bz2opts="-`echo $1 | sed 's/^--compresslevel=//'`"
132         ;;
133         --cachedir=*)
134                 cachedir="`echo $1 | sed 's/^--cachedir=//'`"
135                 cacheopts="--cachedir $cachedir"
136                 [ -d $cachedir/genpkglist ] || mkdir -p $cachedir/genpkglist
137                 [ -d $cachedir/gensrclist ] || mkdir -p $cachedir/gensrclist
138         ;;
139     -*)
140         echo -e "$usage" 1>&2
141         exit 1
142     ;;
143     *)
144         break
145     ;;
146     esac
147     shift
148 done
149
150 if [ -z "$*" ]; then
151         echo -e "$usage" 1>&2
152         exit 1
153 fi
154
155 topdir=`echo $topdir_/$1|tr -s /`
156 shift
157
158 components=$*
159
160 basedir_=base
161 basedir=`echo $topdir/$basedir_|tr -s /`
162
163 pkglist_=$basedir_/pkglist
164 srclist_=$basedir_/srclist
165 release_=$basedir_/release
166 pkglist=$basedir/pkglist
167 srclist=$basedir/srclist
168 release=$basedir/release
169
170 if [ ! -d "$topdir" ]; then
171         echo "error: invalid topdir: $topdir" 1>&2
172         exit 1
173 fi
174
175 if [ ! -d "$basedir" ]; then
176         echo -n "Creating base directory... "
177         mkdir $basedir
178         echo "done"
179 fi
180
181 if [ -z "$components" ]; then
182         comps=$basedir/.components.tmp
183         rm -f $comps; touch $comps
184         for dir in $topdir/RPMS.* $topdir/SRPMS.* $topdir/../SRPMS.*; do
185                 if [ -d $dir ]; then
186                         echo $dir | sed 's/.*\.//' >> $comps
187                 fi
188         done
189         components=`cat $comps|sort|uniq`
190         components=`echo $components` # eat newlines
191         rm -f $comps
192 fi
193
194 if [ -z "$components" ]; then
195         echo "error: no components found" 2>&1
196     exit 1
197 else
198         echo "Components: $components"
199 fi
200
201 if [ $hashonly -ne 1 ]; then
202
203 # package lists
204 # -------------
205
206 test -z "$TMPDIR" && TMPDIR=/tmp;
207 srcidxdir=`mktemp -d $TMPDIR/genbasedir.XXXXXX 2>/dev/null`
208 if [ -z "$srcidxdir" -o ! -d "$srcidxdir" ]; then
209         srcidxdir=$basedir/.srcidxtmp.$$
210         mkdir -p $srcidxdir
211         if [ ! -d "$srcidxdir" ]; then
212                 echo "error: can't create temp dir" 1>&2
213                 exit 1
214         fi
215 fi
216
217 echo -n "Processing pkglists..."
218
219 # Save older pkglist outside loop, if creating a meta repository
220 if [ -n "$meta" ]; then 
221         if [ -f $basedir/pkglist.$meta ]; then
222                 mv -f $basedir/pkglist.$meta $basedir/pkglist.$meta.old
223         fi
224 fi
225
226 for comp in $components; do
227         if [ ! -d $topdir/RPMS.$comp ]; then
228                 continue
229         fi
230
231         echo -n " $comp"
232
233     # Save older pkglist inside loop, if creating a normal repository
234         if [ -z "$meta" ]; then
235                 if [ -f $basedir/pkglist.$comp ]; then
236                         mv -f $basedir/pkglist.$comp $basedir/pkglist.$comp.old
237                 fi
238         fi
239
240     if test x$updateinfo = x; then
241         (cd $basedir; genpkglist $progress $bloat $meta_opts $cacheopts --index $srcidxdir/srcidx.$comp $topdir $comp)
242     else
243         (cd $basedir; genpkglist $progress $bloat $meta_opts $cacheopts --index $srcidxdir/srcidx.$comp --info $updateinfo $topdir $comp)
244     fi
245
246     if [ -z "$meta" -a -f $basedir/pkglist.$comp ]; then
247
248         # Compare with older pkglist.
249         if [ -f $basedir/pkglist.$comp.old ]; then
250             if cmp -s $basedir/pkglist.$comp.old $basedir/pkglist.$comp; then
251                 mv -f $basedir/pkglist.$comp.old $basedir/pkglist.$comp
252             fi
253         fi
254
255         # Save older compressed pkglist
256         if [ -f $basedir/pkglist.$comp.bz2 ]; then
257             mv -f $basedir/pkglist.$comp.bz2 $basedir/pkglist.$comp.bz2.old
258         fi
259
260         bzip2 $bz2opts -c $basedir/pkglist.$comp > $basedir/pkglist.$comp.bz2
261
262         # Compare with older compressed pkglist.
263         if [ -f $basedir/pkglist.$comp.bz2.old ]; then
264             if cmp -s $basedir/pkglist.$comp.bz2.old $basedir/pkglist.$comp.bz2; then
265                 mv -f $basedir/pkglist.$comp.bz2.old $basedir/pkglist.$comp.bz2
266             fi
267         fi
268
269         rm -f $basedir/pkglist.$comp.old
270         rm -f $basedir/pkglist.$comp.bz2.old
271     fi
272
273     cat $srcidxdir/srcidx.$comp >> $srcidxdir/srcidx
274
275 done
276
277 if [ -n "$meta" -a -f $basedir/pkglist.$meta ]; then
278
279         # Compare with older pkglist.
280         if [ -f $basedir/pkglist.$meta.old ]; then
281                 if cmp -s $basedir/pkglist.$meta.old $basedir/pkglist.$meta; then
282                         mv -f $basedir/pkglist.$meta.old $basedir/pkglist.$meta
283                 fi
284         fi
285
286         # Save older compressed pkglist
287         if [ -f $basedir/pkglist.$meta.bz2 ]; then
288                 mv -f $basedir/pkglist.$meta.bz2 $basedir/pkglist.$meta.bz2.old
289         fi
290
291         bzip2 $bz2opts -c $basedir/pkglist.$meta > $basedir/pkglist.$meta.bz2
292
293         # Compare with older compressed pkglist.
294         if [ -f $basedir/pkglist.$meta.bz2.old ]; then
295                 if cmp -s $basedir/pkglist.$meta.bz2.old $basedir/pkglist.$meta.bz2; then
296                         mv -f $basedir/pkglist.$meta.bz2.old $basedir/pkglist.$meta.bz2
297                 fi
298         fi
299
300         rm -f $basedir/pkglist.$meta.old
301         rm -f $basedir/pkglist.$meta.bz2.old
302 fi
303
304 echo " [done]"
305
306 echo -n "Processing srclists..."
307
308 if [ -n "$meta" ]; then 
309         if [ -f $basedir/srclist.$meta ]; then
310                 mv -f $basedir/srclist.$meta $basedir/srclist.$meta.old
311         fi
312 fi
313
314 for comp in $components; do
315         if [ -z "$flat" ]; then
316                 srctopdir=`cd $topdir/..; pwd`
317         else
318                 srctopdir=`cd $topdir; pwd`
319         fi
320
321         if [ ! -d $srctopdir/SRPMS.$comp ]; then
322                 continue
323         fi
324
325         echo -n " $comp"
326
327     # Save older srclist
328     if [ -z "$meta" -a -f $basedir/srclist.$comp ]; then
329         mv -f $basedir/srclist.$comp $basedir/srclist.$comp.old
330     fi
331
332     if [ $mapi -ne 0 ]; then
333         (cd $basedir; gensrclist $progress $flat $meta_opts --mapi $srctopdir $comp $srcidxdir/srcidx.$comp)
334     else
335         (cd $basedir; gensrclist $progress $flat $meta_opts $srctopdir $comp $srcidxdir/srcidx)
336     fi
337
338     if [ -z "$meta" -a -f $basedir/srclist.$comp ]; then
339
340         # Compare with older srclist.
341         if [ -f $basedir/srclist.$comp.old ]; then
342             if cmp -s $basedir/srclist.$comp.old $basedir/srclist.$comp; then
343                 mv -f $basedir/srclist.$comp.old $basedir/srclist.$comp
344             fi
345         fi
346
347         # Save older compressed srclist
348         if [ -f $basedir/srclist.$comp.bz2 ]; then
349             mv -f $basedir/srclist.$comp.bz2 $basedir/srclist.$comp.bz2.old
350         fi
351
352         bzip2 $bz2opts -c $basedir/srclist.$comp > $basedir/srclist.$comp.bz2
353
354         # Compare with older compressed srclist.
355         if [ -f $basedir/srclist.$comp.bz2.old ]; then
356             if cmp -s $basedir/srclist.$comp.bz2.old $basedir/srclist.$comp.bz2; then
357                 mv -f $basedir/srclist.$comp.bz2.old $basedir/srclist.$comp.bz2
358             fi
359         fi
360
361         rm -f $basedir/srclist.$comp.old
362         rm -f $basedir/srclist.$comp.bz2.old
363     fi
364
365     rm -f $srcidxdir/srcidx.$comp
366 done
367
368 if [ -n "$meta" -a -f $basedir/srclist.$meta ]; then
369
370         # Compare with older srclist.
371         if [ -f $basedir/srclist.$meta.old ]; then
372                 if cmp -s $basedir/srclist.$meta.old $basedir/srclist.$meta; then
373                         mv -f $basedir/srclist.$meta.old $basedir/srclist.$meta
374                 fi
375         fi
376
377         # Save older compressed srclist
378         if [ -f $basedir/srclist.$meta.bz2 ]; then
379                 mv -f $basedir/srclist.$meta.bz2 $basedir/srclist.$meta.bz2.old
380         fi
381
382         bzip2 $bz2opts -c $basedir/srclist.$meta > $basedir/srclist.$meta.bz2
383
384         # Compare with older compressed srclist.
385         if [ -f $basedir/srclist.$meta.bz2.old ]; then
386                 if cmp -s $basedir/srclist.$meta.bz2.old $basedir/srclist.$meta.bz2; then
387                         mv -f $basedir/srclist.$meta.bz2.old $basedir/srclist.$meta.bz2
388                 fi
389         fi
390
391         rm -f $basedir/srclist.$meta.old
392         rm -f $basedir/srclist.$meta.bz2.old
393 fi
394
395 echo " [done]"
396
397 fi
398
399 rm -rf $srcidxdir
400
401 if [ -n "$meta" ]; then
402         components=$meta
403 fi
404
405 if [ $listonly -ne 0 ]; then
406         if [ $bz2only -eq 1 -a -f $pkglist.$comp ]; then
407                 rm -f $pkglist.$comp
408         fi
409         if [ $bz2only -eq 1 -a -f $srclist.$comp ]; then
410                 rm -f $srclist.$comp
411         fi
412 else
413         # component release file
414         # ----------------------
415
416         echo -n "Updating component releases..."
417         for comp in $components; do
418                 echo -n " $comp"
419                 if [ -f $release.$comp ]; then
420                         mv -f $release.$comp $release.$comp.old
421                         sed -e "/\(Package\|Source\)Count:/d" \
422                                 $release.$comp.old > $release.$comp
423                         rm -f $release.$comp.old
424                 else
425                         echo "Archive: Unknown"      >> $release.$comp
426                         echo "Component: $comp"      >> $release.$comp
427                         echo "Version: Unknown"      >> $release.$comp
428                         echo "Origin: Unknown"       >> $release.$comp
429                         echo "Label: Unknown"        >> $release.$comp
430                         echo "Architecture: unknown" >> $release.$comp
431                         echo "NotAutomatic: false"   >> $release.$comp
432                 fi
433                 if [ -f $pkglist.$comp ]; then
434                         echo "PackageCount: `countpkglist $pkglist.$comp`" \
435                                 >> $release.$comp
436                 fi
437                 if [ -f $srclist.$comp ]; then
438                         echo "SourceCount: `countpkglist $srclist.$comp`" \
439                                 >> $release.$comp
440                 fi
441         done
442         echo " [done]"
443
444         # release file
445         # ------------
446
447         if [ -f $release ]; then
448                 mv -f $release $release.old
449                 if [ $partial -eq 0 ]; then
450                         echo -n "Updating global release file... "
451                         sed -n -e "/^MD5Sum:/q" \
452                                    -e "s/^Date:.*\$/Date: `date -R`/" \
453                                    -e "s/^Components:.*\$/Components: $components/" \
454                                    -e "p" $release.old > $release
455                         echo "MD5Sum:" >> $release
456                 else
457                         echo -n "Partially updating global release file... "
458                         sed -n -e "/^\$/q" \
459                                    -e "s/^Date:.*\$/Date: `date -R`/" \
460                                    -e "p" $release.old > $release.pre
461                         for comp in $components; do
462                                 sed -e "\#^ .* $pkglist_.$comp\(.bz2\)\?\$#d" \
463                                     -e "\#^ .* $srclist_.$comp\(.bz2\)\?\$#d" \
464                                     -e "\#^ .* $release_.$comp\(.bz2\)\?\$#d" \
465                                     -e "s/^\(Components:.*\) $comp\(.*\)\$/\1\2/" \
466                                         $release.pre > $release.tmp
467                                 mv -f $release.tmp $release.pre
468                         done
469                         sed -e "s/^\(Components:.*\)\$/\1 $components/" \
470                                 $release.pre > $release
471                         rm -f $release.pre
472                 fi
473                 echo "[done]"
474         else
475                 echo -n "Creating global release file... "
476                 echo "Origin: Unknown"            >> $release
477                 echo "Label: Unknown"             >> $release
478                 echo "Suite: Unknown"             >> $release
479                 echo "Codename: Unknown"          >> $release
480                 echo "Date: `date -R`"            >> $release
481                 echo "Architectures: Unknown"     >> $release
482                 echo "Components: $components"    >> $release
483                 echo "Description: Not available" >> $release
484                 echo "MD5Sum:"                    >> $release
485                 echo "[done]"
486         fi
487
488         echo -n "Appending MD5Sum..."
489
490         for comp in $components; do
491                 echo -n " $comp"
492                 if [ -f $pkglist.$comp ]; then
493                         phashstuff $pkglist.$comp $pkglist_.$comp >> $release
494                         if [ $bz2only -eq 1 ]; then
495                                 rm -f $pkglist.$comp
496                         fi
497                 fi
498                 if [ -f $srclist.$comp ]; then
499                         phashstuff $srclist.$comp $srclist_.$comp >> $release
500                         if [ $bz2only -eq 1 ]; then
501                                 rm -f $srclist.$comp
502                         fi
503                 fi
504                 if [ -f $pkglist.$comp.bz2 ]; then
505                         phashstuff $pkglist.$comp.bz2 $pkglist_.$comp.bz2 >> $release
506                 fi
507                 if [ -f $srclist.$comp.bz2 ]; then
508                         phashstuff $srclist.$comp.bz2 $srclist_.$comp.bz2 >> $release
509                 fi
510                 if [ -f $release.$comp ]; then
511                         phashstuff $release.$comp $release_.$comp >> $release
512                 fi
513         done
514         echo " [done]"
515
516         echo >> $release
517
518         if [ $signature -ne 0 ]; then
519                 # Use eval because $defaultkey may contain spaces
520                 eval gpg --sign -armour --detach-sign --yes --quiet $defaultkey $release
521                 cat $release.asc >> $release
522                 rm -f $release.asc
523         fi
524
525         # Compare with older release
526         if [ -f $release.old ]; then
527                 if cmp -s $release.old $release; then
528                         mv -f $release.old $release
529                 fi
530                 rm -f $release.old
531         fi
532 fi
533
534
535 # old hashfile
536 # ------------
537
538 if [ $oldhashfile -ne 0 -a $listonly -eq 0 ]; then
539         # Save older hashfile
540         hf=$basedir/hashfile
541         if [ -f $hf ]; then
542                 mv -f $hf $hf.old
543                 if [ $partial -eq 0 ]; then
544                         echo -n "Updating legacy hashfile... "
545                         echo "MD5SUM:" > $hf
546                 else
547                         echo -n "Partially updating legacy hashfile file... "
548                         sed -n -e "/^\$/q;p" $hf.old > $hf.pre
549                         for comp in $components; do
550                                 sed -e "\#^ .* $pkglist_.$comp\(.bz2\)\?\$#d" \
551                                     -e "\#^ .* $srclist_.$comp\(.bz2\)\?\$#d" \
552                                     -e "\#^ .* $release_.$comp\(.bz2\)\?\$#d" \
553                                         $hf.pre > $hf.tmp
554                                 mv -f $hf.tmp $hf.pre
555                         done
556                         mv -f $hf.pre $hf
557                 fi
558         else
559                 echo -n "Creating legacy hashfile... "
560                 echo "MD5SUM:" > $hf
561         fi
562         echo "[done]"
563
564         echo -n "Appending MD5Sum..."
565
566         for comp in $components; do
567                 echo -n " $comp"
568
569                 if [ -f $pkglist.$comp ]; then
570                         phashstuff $pkglist.$comp $pkglist_.$comp >> $hf
571                 fi
572                 if [ -f $srclist.$comp ]; then
573                         phashstuff $srclist.$comp $srclist_.$comp >> $hf
574                 fi
575                 if [ -f $pkglist.$comp.bz2 ]; then
576                         phashstuff $pkglist.$comp.bz2 $pkglist_.$comp.bz2 >> $hf
577                 fi
578                 if [ -f $srclist.$comp.bz2 ]; then
579                         phashstuff $srclist.$comp.bz2 $srclist_.$comp.bz2 >> $hf
580                 fi
581                 if [ -f $release.$comp ]; then
582                         phashstuff $release.$comp $release_.$comp >> $hf
583                 fi
584         done
585         echo " [done]"
586
587         echo >> $hf
588
589         # Compare with older hashfile.
590         if [ -f $basedir/hashfile.old ]; then
591                 if cmp -s $basedir/hashfile.old $basedir/hashfile; then
592                         mv -f $basedir/hashfile.old $basedir/hashfile
593                 fi
594         fi
595
596         if [ $signature -ne 0 ]; then
597
598                 # Save older hashfile.gpg
599                 if [ -f $basedir/hashfile.gpg ]; then
600                         mv -f $basedir/hashfile.gpg $basedir/hashfile.gpg.old
601                 fi
602
603                 # Use eval because $defaultkey may contain spaces
604                 eval gpg --sign --armour --yes --quiet $defaultkey $basedir/hashfile
605                 mv -f $basedir/hashfile.asc $basedir/hashfile.gpg
606
607                 # Compare with older hashfile.gpg
608                 if [ -f $basedir/hashfile.gpg.old ]; then
609                         if cmp -s $basedir/hashfile.gpg.old $basedir/hashfile.gpg; then
610                                 mv -f $basedir/hashfile.gpg.old $basedir/hashfile.gpg
611                         fi
612                 fi
613         fi
614
615         rm -f $basedir/hashfile.old
616         rm -f $basedir/hashfile.gpg.old
617 fi
618
619 echo "All your base are belong to us!!!"
620
621 # vim:ts=4:sw=4