Date:2010-10-17 03:14:59 (8 years 1 month ago)
Author:Werner Almesberger
Commit:57fc0bd84e98ccfc0fb82c4f7719758ab78b69df
Message:Improve scalability by splitting the distributor cache. Additional cleanup and generalizations.

- boom/Makefile, dist/dk/Makefile: the "files with everything" are now
called dist/db/all.* instead of dist/db/digi-key.*
- boom/manu/panasonic/Makefile, boom/manu/stackpole/Makefile,
boom/manu/yageo/Makefile: changed digi-key.equ to all.equ
- boom/Makefile: there can now be a list of distributors
- boom/Makefile (tar): new target to generate a snapshot of the database
- boom/Makefile (PHONY): added "spotless"
- boom/dist/dk/Makefile: do not merge catalog listings so that we don't have
to search a huge cache with O(n^2)
- boom/dist/dk/Makefile (clean, rebuild, update, spotless): targets for
different degrees of rebuilding
- boom/dist/dk/dk-db.pl: added progress report output
Files: boom/Makefile (2 diffs)
boom/dist/dk/Makefile (1 diff)
boom/dist/dk/dk-db.pl (1 diff)
boom/manu/panasonic/Makefile (1 diff)
boom/manu/stackpole/Makefile (1 diff)
boom/manu/yageo/Makefile (1 diff)

Change Details

boom/Makefile
22
33KITS = 1
44
5INV = dist/dk/db/digi-key.inv
6EQU = dist/dk/db/digi-key.equ
7DSC = dist/dk/db/digi-key.dsc
5DIST = dk
6EQU = $(DIST:%=dist/%/db/all.equ)
7INV = $(DIST:%=dist/%/db/all.inv)
8DSC = $(DIST:%=dist/%/db/all.dsc)
89CHR = manu/panasonic/panasonic.chr manu/stackpole/stackpole.chr \
910      manu/yageo/yageo.chr
1011
11.PHONY: all show again
12DB = $(CHR) \
13    $(DIST:%=dist/%/db/*.equ) \
14    $(DIST:%=dist/%/db/*.inv) \
15    $(DIST:%=dist/%/db/*.dsc)
16
17
18.PHONY: all show again spotless tar
1219
1320all: show
1421
...... 
3037
3138spotless:
3239        rm -f test.lst test.par test.ord
40
41tar:
42        tar cfj boom-db.tar.bz2 $(DB)
43
boom/dist/dk/Makefile
1CACHE = query.data
2EQUS = panasonic-erj stackpole-rmcf yageo-rc
1MASTER = db/all.equ db/all.dsc db/all.inv
2PARTS = panasonic-erj stackpole-rmcf yageo-rc
33
4.PHONY: all
4.SECONDARY: # prevent *.cache from being automatically deleted
55
6all: db/digi-key.dsc db/digi-key.inv
6.PHONY: all clean rebuild update spotless
77
8$(CACHE): db/digi-key.equ
8all: $(MASTER) \
9              $(PARTS:%=db/%.equ) \
10              $(PARTS:%=db/%.inv) \
11              $(PARTS:%=db/%.dsc)
12
13db/%.cache: db/%.equ
914            awk '/^#END/ { exit } /^DIGI-KEY / { print $$2 }' $< | \
1015              perl ./dk-db.pl query \
11              `[ -r $(CACHE) ] && echo '' -i $(CACHE)` >_$@ || \
12              { rm -f $@ _$@; exit 1; }
13            mv _$@ $@
16              `[ -r $@ ] && echo '' -i $@` >$(@)_ || \
17              { rm -f $@ $(@)_; exit 1; }
18            mv $(@)_ $@
19
20db:
21            mkdir -p db
22
23#
24# We need "db" to create the directory. However, this also means that any
25# change in the db/ directory triggers a rebuild of all.equ. This is cheap, so
26# this is nothing but a small cosmetic issue.
27#
1428
15db/digi-key.equ: $(EQUS:%=db/%.equ)
16            cat $^ >$@ || { rm -rf $@; exit 1; }
29db/all.equ: db Makefile
30            for n in $(PARTS); do echo "include $$n.equ"; done >$@
1731
18db/digi-key.dsc: $(CACHE)
19            perl ./dk-db.pl dsc $< >$@ || \
20              { rm -f $@; exit 1; }
32db/all.inv: Makefile
33            for n in $(PARTS); do echo "include $$n.inv"; done >$@
2134
22db/digi-key.inv: $(CACHE)
23            perl ./dk-db.pl inv $< >$@ || \
24              { rm -f $@; exit 1; }
35db/all.dsc: Makefile
36            for n in $(PARTS); do echo "include $$n.dsc"; done >$@
2537
2638db/%.equ: %.catq
2739            perl ./dk-catq.pl $< >$@ || { rm -rf $@; exit 1; }
40
41db/%.inv: db/%.cache
42            perl ./dk-db.pl inv $< >$@ || { rm -f $@; exit 1; }
43
44db/%.dsc: db/%.cache
45            perl ./dk-db.pl dsc $< >$@ || { rm -f $@; exit 1; }
46
47
48# --- Cleanup -----------------------------------------------------------------
49
50#
51# We have various levels of cleanup:
52#
53# - clean: remove left-over temporary files
54# - rebuild: remove everything that can be rebuilt from cached data
55# - update: remove the short-lived inventory and pricing cache too
56# - again: remove also the long-lived equivalences cache
57#
58# "clean" does not disturb use of the database. The run-time cost of recovering
59# from "rebuild" is small. What takes a lot of time is "update". The additional
60# cost of "spotless" is small again.
61#
62
63clean:
64            rm -f $(PARTS:%=db/%.cache_)
65
66rebuild: clean
67            rm -f $(MASTER)
68            rm -f $(PARTS:%=db/%.inv)
69            rm -f $(PARTS:%=db/%.dsc)
70
71update: rebuild
72            rm -f $(PARTS:%=db/%.cache)
73
74spotless: update
75            rm -f $(PARTS:%=db/%.equ)
boom/dist/dk/dk-db.pl
6363    next if /^\s*$/;
6464    next if /^\s/;
6565    s/\s.*//;
66
67    $n++;
68    print STDERR "$n\r" unless $n % 100;
69
6670    next if $old =~ m#align=right>Digi-Key Part Number</th><td>$_</td#;
6771    push(@pn, $_);
6872    &flush if @pn > 1000;
boom/manu/panasonic/Makefile
11BOOM=PATH=/home/moko/svn.openmoko.org/trunk/eda/boom:../boom:$$PATH boom
22
3panasonic.chr: ../../dist/dk/db/digi-key.equ panasonic.gen
3panasonic.chr: ../../dist/dk/db/all.equ panasonic.gen
44    $(BOOM) gen2chr PANASONIC $^ >$@ || { rm -rf $@; exit 1; }
boom/manu/stackpole/Makefile
11BOOM=PATH=/home/moko/svn.openmoko.org/trunk/eda/boom:../boom:$$PATH boom
22
3stackpole.chr: ../../dist/dk/db/digi-key.equ stackpole.gen
3stackpole.chr: ../../dist/dk/db/all.equ stackpole.gen
44    $(BOOM) gen2chr STACKPOLE $^ >$@ || { rm -rf $@; exit 1; }
boom/manu/yageo/Makefile
11BOOM=PATH=/home/moko/svn.openmoko.org/trunk/eda/boom:../boom:$$PATH boom
22
3yageo.chr: ../../dist/dk/db/digi-key.equ yageo.gen
3yageo.chr: ../../dist/dk/db/all.equ yageo.gen
44    $(BOOM) gen2chr YAGEO $^ >$@ || { rm -rf $@; exit 1; }

Archive Download the corresponding diff file

Branches:
master



interactive