venti: 32-bit extensions to data structures

include/venti.h

@@ -86,7 +86,6 @@ enum
 {
 	VtScoreSize = 20,
 	VtMaxStringSize = 1024,
-	VtMaxLumpSize	= 56*1024,
 	VtPointerDepth	= 7
 };
 #define VtMaxFileSize ((1ULL<<48)-1)
@@ -137,7 +136,8 @@ enum
 	_VtEntryDir = 1<<1,		/* a directory */
 	_VtEntryDepthShift = 2,		/* shift for pointer depth */
 	_VtEntryDepthMask = 7<<2,	/* mask for pointer depth */
-	VtEntryLocal = 1<<5		/* for local storage only */
+	VtEntryLocal = 1<<5,		/* for local storage only */
+	_VtEntryBig = 1<<6,
 };
 enum
 {
@@ -146,8 +146,8 @@ enum
 struct VtEntry
 {
 	ulong gen;			/* generation number */
-	ushort psize;			/* pointer block size */
+	ulong psize;			/* pointer block size */
-	ushort dsize;			/* data block size */
+	ulong dsize;			/* data block size */
 	uchar type;
 	uchar flags;
 	uvlong size;
@@ -162,14 +162,15 @@ struct VtRoot
 	char name[128];
 	char type[128];
 	uchar score[VtScoreSize];	/* to a Dir block */
-	ushort blocksize;		/* maximum block size */
+	ulong blocksize;		/* maximum block size */
 	uchar prev[VtScoreSize];	/* last root block */
 };
 
 enum
 {
 	VtRootSize = 300,
-	VtRootVersion = 2
+	VtRootVersion = 2,
+	_VtRootVersionBig = 1<<15,
 };
 
 void vtrootpack(VtRoot*, uchar*);
@@ -394,7 +395,8 @@ struct VtBlock
 
 	uchar	*data;
 	uchar	score[VtScoreSize];
-	uchar	type;	/* BtXXX */
+	uchar	type;	/* VtXXX */
+	ulong	size;
 
 	/* internal to cache */
 	int		nlock;
@@ -412,14 +414,13 @@ struct VtBlock
 u32int vtglobaltolocal(uchar[VtScoreSize]);
 void vtlocaltoglobal(u32int, uchar[VtScoreSize]);
 
-VtCache *vtcachealloc(VtConn*, int blocksize, ulong nblocks);
+VtCache *vtcachealloc(VtConn*, ulong maxmem);
 void vtcachefree(VtCache*);
 VtBlock *vtcachelocal(VtCache*, u32int addr, int type);
-VtBlock *vtcacheglobal(VtCache*, uchar[VtScoreSize], int type);
+VtBlock *vtcacheglobal(VtCache*, uchar[VtScoreSize], int type, ulong size);
-VtBlock *vtcacheallocblock(VtCache*, int type);
+VtBlock *vtcacheallocblock(VtCache*, int type, ulong size);
 void vtcachesetwrite(VtCache*, int(*)(VtConn*,uchar[VtScoreSize],uint,uchar*,int));
 void vtblockput(VtBlock*);
-u32int vtcacheblocksize(VtCache*);
 int vtblockwrite(VtBlock*);
 VtBlock *vtblockcopy(VtBlock*);
 void vtblockduplock(VtBlock*);
@@ -438,6 +439,7 @@ struct VtFile
 	int local;
 	VtBlock *b;			/* block containing this file */
 	uchar score[VtScoreSize];	/* score of block containing this file */
+	int bsize;				/* size of block */
 
 /* immutable */
 	VtCache *c;

src/cmd/vac/file.c

@@ -1730,7 +1730,7 @@ Err1:
 static char EBadVacFormat[] = "bad format for vac file";
 
 static VacFs *
-vacfsalloc(VtConn *z, int bsize, int ncache, int mode)
+vacfsalloc(VtConn *z, int bsize, ulong cachemem, int mode)
 {
 	VacFs *fs;
 
@@ -1738,7 +1738,7 @@ vacfsalloc(VtConn *z, int bsize, int ncache, int mode)
 	fs->z = z;
 	fs->bsize = bsize;
 	fs->mode = mode;
-	fs->cache = vtcachealloc(z, bsize, ncache);
+	fs->cache = vtcachealloc(z, cachemem);
 	return fs;
 }
 
@@ -1767,7 +1767,7 @@ readscore(int fd, uchar score[VtScoreSize])
 }
 
 VacFs*
-vacfsopen(VtConn *z, char *file, int mode, int ncache)
+vacfsopen(VtConn *z, char *file, int mode, ulong cachemem)
 {
 	int fd;
 	uchar score[VtScoreSize];
@@ -1788,11 +1788,11 @@ vacfsopen(VtConn *z, char *file, int mode, int ncache)
 		}
 		close(fd);
 	}
-	return vacfsopenscore(z, score, mode, ncache);
+	return vacfsopenscore(z, score, mode, cachemem);
 }
 
 VacFs*
-vacfsopenscore(VtConn *z, u8int *score, int mode, int ncache)
+vacfsopenscore(VtConn *z, u8int *score, int mode, ulong cachemem)
 {
 	VacFs *fs;
 	int n;
@@ -1818,13 +1818,19 @@ vacfsopenscore(VtConn *z, u8int *score, int mode, int ncache)
 		return nil;
 	}
 
-	fs = vacfsalloc(z, rt.blocksize, ncache, mode);
+	fs = vacfsalloc(z, rt.blocksize, cachemem, mode);
 	memmove(fs->score, score, VtScoreSize);
 	fs->mode = mode;
 
 	memmove(e.score, rt.score, VtScoreSize);
 	e.gen = 0;
+	
+	// Don't waste cache memory on directories
+	// when rt.blocksize is large.
 	e.psize = (rt.blocksize/VtEntrySize)*VtEntrySize;
+	if(e.psize > 60000)
+		e.psize = (60000/VtEntrySize)*VtEntrySize;
+
 	e.dsize = rt.blocksize;
 	e.type = VtDirType;
 	e.flags = VtEntryActive;
@@ -1925,7 +1931,7 @@ vacfsclose(VacFs *fs)
  * Create a fresh vac fs.
  */
 VacFs *
-vacfscreate(VtConn *z, int bsize, int ncache)
+vacfscreate(VtConn *z, int bsize, ulong cachemem)
 {
 	VacFs *fs;
 	VtFile *f;
@@ -1937,18 +1943,24 @@ vacfscreate(VtConn *z, int bsize, int ncache)
 	MetaEntry me;
 	int psize;
 	
-	if((fs = vacfsalloc(z, bsize, ncache, VtORDWR)) == nil)
+	if((fs = vacfsalloc(z, bsize, cachemem, VtORDWR)) == nil)
 		return nil;
 
 	/*
 	 * Fake up an empty vac fs.
 	 */
 	psize = bsize/VtEntrySize*VtEntrySize;
+	if(psize > 60000)
+		psize = 60000/VtEntrySize*VtEntrySize;
+fprint(2, "create bsize %d psize %d\n", bsize, psize);
+
 	f = vtfilecreateroot(fs->cache, psize, bsize, VtDirType);
+	if(f == nil)
+		sysfatal("vtfilecreateroot: %r");
 	vtfilelock(f, VtORDWR);
 
 	/* Write metablock containing root directory VacDir. */
-	b = vtcacheallocblock(fs->cache, VtDataType);
+	b = vtcacheallocblock(fs->cache, VtDataType, bsize);
 	mbinit(&mb, b->data, bsize, bsize/BytesPerEntry);
 	memset(&vd, 0, sizeof vd);
 	vd.elem = "/";

src/cmd/vac/pack.c

@@ -77,7 +77,7 @@ mbunpack(MetaBlock *mb, uchar *p, int n)
 
 	magic = U32GET(p);
 	if(magic != MetaMagic && magic != MetaMagic+1) {
-		werrstr("bad meta block magic");
+		werrstr("bad meta block magic %#08ux", magic);
 		return -1;
 	}
 	mb->size = U16GET(p+4);

src/cmd/vac/unvac.c

@@ -94,7 +94,7 @@ threadmain(int argc, char *argv[])
 	if(vtconnect(conn) < 0)
 		sysfatal("vtconnect: %r");
 
-	fs = vacfsopen(conn, argv[0], VtOREAD, 128);
+	fs = vacfsopen(conn, argv[0], VtOREAD, 4<<20);
 	if(fs == nil)
 		sysfatal("vacfsopen: %r");
 

src/cmd/vac/vac.c

@@ -100,8 +100,6 @@ threadmain(int argc, char **argv)
 		u = unittoull(EARGF(usage()));
 		if(u < 512)
 			u = 512;
-		if(u > VtMaxLumpSize)
-			u = VtMaxLumpSize;
 		blocksize = u;
 		break;
 	case 'd':
@@ -170,10 +168,10 @@ threadmain(int argc, char **argv)
 			if((outfd = create(archivefile, OWRITE, 0666)) < 0)
 				sysfatal("create %s: %r", archivefile);
 			atexit(removevacfile);	// because it is new
-			if((fs = vacfscreate(z, blocksize, 512)) == nil)
+			if((fs = vacfscreate(z, blocksize, 4<<20)) == nil)
 				sysfatal("vacfscreate: %r");
 		}else{
-			if((fs = vacfsopen(z, archivefile, VtORDWR, 512)) == nil)
+			if((fs = vacfsopen(z, archivefile, VtORDWR, 4<<20)) == nil)
 				sysfatal("vacfsopen %s: %r", archivefile);
 			if((fdiff = recentarchive(fs, oldpath)) != nil){
 				if(verbose)
@@ -213,7 +211,7 @@ threadmain(int argc, char **argv)
 		else if((outfd = create(vacfile, OWRITE, 0666)) < 0)
 			sysfatal("create %s: %r", vacfile);
 		atexit(removevacfile);
-		if((fs = vacfscreate(z, blocksize, 512)) == nil)
+		if((fs = vacfscreate(z, blocksize, 4<<20)) == nil)
 			sysfatal("vacfscreate: %r");
 		f = vacfsgetroot(fs);
 
@@ -450,7 +448,7 @@ void
 vac(VacFile *fp, VacFile *diffp, char *name, Dir *d)
 {
 	char *elem, *s;
-	static char buf[65536];
+	static char *buf;
 	int fd, i, n, bsize;
 	vlong off;
 	Dir *dk;	// kids
@@ -541,6 +539,8 @@ vac(VacFile *fp, VacFile *diffp, char *name, Dir *d)
 	}else{
 		off = 0;
 		bsize = fs->bsize;
+		if(buf == nil)
+			buf = vtmallocz(bsize);
 		if(fdiff){
 			/*
 			 * Copy fdiff's contents into f by moving the score.
@@ -708,7 +708,7 @@ vacmerge(VacFile *fp, char *name)
 
 	if(strlen(name) < 4 || strcmp(name+strlen(name)-4, ".vac") != 0)
 		return -1;
-	if((mfs = vacfsopen(z, name, VtOREAD, 100)) == nil)
+	if((mfs = vacfsopen(z, name, VtOREAD, 4<<20)) == nil)
 		return -1;
 	if(verbose)
 		fprint(2, "merging %s\n", name);

src/cmd/vac/vac.h

@@ -97,9 +97,9 @@ struct VacFs
 	VtCache	*cache;
 };
 
-VacFs	*vacfsopen(VtConn *z, char *file, int mode, int ncache);
+VacFs	*vacfsopen(VtConn *z, char *file, int mode, ulong cachemem);
-VacFs	*vacfsopenscore(VtConn *z, u8int *score, int mode, int ncache);
+VacFs	*vacfsopenscore(VtConn *z, u8int *score, int mode, ulong cachemem);
-VacFs	*vacfscreate(VtConn *z, int bsize, int ncache);
+VacFs	*vacfscreate(VtConn *z, int bsize, ulong cachemem);
 void		vacfsclose(VacFs *fs);
 int		vacfssync(VacFs *fs);
 int		vacfssnapshot(VacFs *fs, char *src, char *dst);

src/cmd/vac/vacfs.c

@@ -118,6 +118,31 @@ notifyf(void *a, char *s)
 	noted(NDFLT);
 }
 
+#define TWID64 ~(u64int)0
+static u64int
+unittoull(char *s)
+{
+	char *es;
+	u64int n;
+
+	if(s == nil)
+		return TWID64;
+	n = strtoul(s, &es, 0);
+	if(*es == 'k' || *es == 'K'){
+		n *= 1024;
+		es++;
+	}else if(*es == 'm' || *es == 'M'){
+		n *= 1024*1024;
+		es++;
+	}else if(*es == 'g' || *es == 'G'){
+		n *= 1024*1024*1024;
+		es++;
+	}
+	if(*es != '\0')
+		return TWID64;
+	return n;
+}
+
 void
 threadmain(int argc, char *argv[])
 {
@@ -125,10 +150,10 @@ threadmain(int argc, char *argv[])
 	int p[2], fd;
 	int stdio;
 	char *host = nil;
-	long ncache;
+	ulong mem;
 
+	mem = 16<<20;
 	stdio = 0;
-	ncache = 256;
 	fmtinstall('H', encodefmt);
 	fmtinstall('V', vtscorefmt);
 	fmtinstall('F', vtfcallfmt);
@@ -140,9 +165,6 @@ threadmain(int argc, char *argv[])
 		fmtinstall('F', fcallfmt);
 		dflag = 1;
 		break;
-	case 'c':
-		ncache = atoi(EARGF(usage()));
-		break;
 	case 'i':
 		defmnt = nil;
 		stdio = 1;
@@ -158,6 +180,9 @@ threadmain(int argc, char *argv[])
 	case 's':
 		defsrv = "vacfs";
 		break;
+	case 'M':
+		mem = unittoull(EARGF(usage()));
+		break;
 	case 'm':
 		defmnt = EARGF(usage());
 		break;
@@ -206,7 +231,7 @@ threadmain(int argc, char *argv[])
 	if(vtconnect(conn) < 0)
 		sysfatal("vtconnect: %r");
 
-	fs = vacfsopen(conn, argv[0], VtOREAD, ncache);
+	fs = vacfsopen(conn, argv[0], VtOREAD, mem);
 	if(fs == nil)
 		sysfatal("vacfsopen: %r");
 

src/cmd/venti/copy.c

@@ -4,6 +4,12 @@
 #include <libsec.h>
 #include <thread.h>
 
+enum
+{
+	// XXX What to do here?
+	VtMaxLumpSize = 65536,
+};
+
 int changes;
 int rewrite;
 int ignoreerrors;

src/cmd/venti/dump.c

@@ -5,6 +5,12 @@
 #include <libsec.h>
 #include <thread.h>
 
+enum
+{
+	// XXX What to do here?
+	VtMaxLumpSize = 65536,
+};
+
 VtConn *z;
 char *host;
 

src/cmd/venti/read.c

@@ -4,6 +4,12 @@
 #include <libsec.h>
 #include <thread.h>
 
+enum
+{
+	// XXX What to do here?
+	VtMaxLumpSize = 65536,
+};
+
 void
 usage(void)
 {

src/cmd/venti/root.c

@@ -38,7 +38,7 @@ threadmain(int argc, char *argv[])
 	if(argc == 0)
 		usage();
 
-	buf = vtmallocz(VtMaxLumpSize);
+	buf = vtmallocz(8192);
 
 	z = vtdial(host);
 	if(z == nil)
@@ -52,7 +52,7 @@ threadmain(int argc, char *argv[])
 			fprint(2, "cannot parse score '%s': %r\n", argv[i]);
 			continue;
 		}
-		n = vtread(z, score, VtRootType, buf, VtMaxLumpSize);
+		n = vtread(z, score, VtRootType, buf, 8192);
 		if(n < 0){
 			fprint(2, "could not read block %V: %r\n", score);
 			continue;

src/cmd/venti/srv/dat.h

@@ -37,6 +37,12 @@ typedef struct Bloom	Bloom;
 
 enum
 {
+	/*
+	 * formerly fundamental constant,
+	 * now a server-imposed limitation.
+	 */
+	VtMaxLumpSize	= 56*1024,
+
 	ABlockLog		= 9,		/* log2(512), the quantum for reading arenas */
 	ANameSize		= 64,
 	MaxDiskBlock		= 64*1024,	/* max. allowed size for a disk block */

src/cmd/venti/write.c

@@ -4,6 +4,12 @@
 #include <libsec.h>
 #include <thread.h>
 
+enum
+{
+	// XXX What to do here?
+	VtMaxLumpSize = 65536,
+};
+
 void
 usage(void)
 {

src/libdiskfs/venti.c

@@ -96,13 +96,13 @@ _vtfileblock(VtCache *c, VtEntry *e, u32int bn)
 	}
 
 /*fprint(2, "vtread %V\n", e->score); */
-	b = vtcacheglobal(c, e->score, e->type);
+	b = vtcacheglobal(c, e->score, e->type, d == 0 ? e->dsize : e->psize);
 	for(i=d-1; i>=0 && b; i--){
 		t = VtDataType+i;
 /*fprint(2, "vtread %V\n", b->data+index[i]*VtScoreSize); */
 		memmove(score, b->data+index[i]*VtScoreSize, VtScoreSize);
 		vtblockput(b);
-		b = vtcacheglobal(c, score, t);
+		b = vtcacheglobal(c, score, t, i == 0 ? e->dsize : e->psize);
 	}
 	return b;
 }
@@ -122,7 +122,7 @@ diskopenventi(VtCache *c, uchar score[VtScoreSize])
 	VtRoot root;
 	VtBlock *b;
 
-	if((b = vtcacheglobal(c, score, VtRootType)) == nil)
+	if((b = vtcacheglobal(c, score, VtRootType, VtRootSize)) == nil)
 		goto Err;
 	if(vtrootunpack(&root, b->data) < 0)
 		goto Err;
@@ -132,7 +132,7 @@ diskopenventi(VtCache *c, uchar score[VtScoreSize])
 	}
 	vtblockput(b);
 
-	if((b = vtcacheglobal(c, root.score, VtDirType)) == nil)
+	if((b = vtcacheglobal(c, root.score, VtDirType, VtEntrySize)) == nil)
 		goto Err;
 	if(vtentryunpack(&e, b->data, 0) < 0)
 		goto Err;

src/libventi/cache.c

@@ -32,7 +32,6 @@ struct VtCache
 {
 	QLock	lk;
 	VtConn	*z;
-	u32int	blocksize;
 	u32int	now;		/* ticks for usage time stamps */
 	VtBlock	**hash;	/* hash table for finding addresses */
 	int		nhash;
@@ -40,41 +39,45 @@ struct VtCache
 	int		nheap;
 	VtBlock	*block;	/* all allocated blocks */
 	int		nblock;
-	uchar	*mem;	/* memory for all blocks and data */
 	int		(*write)(VtConn*, uchar[VtScoreSize], uint, uchar*, int);
+	VtBlock	*dead;	/* blocks we don't have memory for */
+	ulong	mem;
+	ulong	maxmem;
 };
 
 static void cachecheck(VtCache*);
 
 VtCache*
-vtcachealloc(VtConn *z, int blocksize, ulong nblock)
+vtcachealloc(VtConn *z, ulong maxmem)
 {
-	uchar *p;
 	VtCache *c;
 	int i;
+	int nblock;
 	VtBlock *b;
+	ulong maxmem0;
 
+	maxmem0 = maxmem;
 	c = vtmallocz(sizeof(VtCache));
-
+	nblock = maxmem/100/(sizeof(VtBlock)+2*sizeof(VtBlock*));
 	c->z = z;
-	c->blocksize = (blocksize + 127) & ~127;
 	c->nblock = nblock;
 	c->nhash = nblock;
 	c->hash = vtmallocz(nblock*sizeof(VtBlock*));
 	c->heap = vtmallocz(nblock*sizeof(VtBlock*));
 	c->block = vtmallocz(nblock*sizeof(VtBlock));
-	c->mem = vtmallocz(nblock*c->blocksize);
 	c->write = vtwrite;
+	maxmem -= nblock*(sizeof(VtBlock) + 2*sizeof(VtBlock*));
+	maxmem -= sizeof(VtCache);
+	if((long)maxmem < 0)
+		sysfatal("cache size far too small: %lud", maxmem0);
+	c->mem = maxmem;
 
-	p = c->mem;
 	for(i=0; i<nblock; i++){
 		b = &c->block[i];
 		b->addr = NilBlock;
 		b->c = c;
-		b->data = p;
 		b->heap = i;
 		c->heap[i] = b;
-		p += c->blocksize;
 	}
 	c->nheap = nblock;
 	cachecheck(c);
@@ -102,13 +105,14 @@ vtcachefree(VtCache *c)
 	qlock(&c->lk);
 
 	cachecheck(c);
-	for(i=0; i<c->nblock; i++)
+	for(i=0; i<c->nblock; i++) {
 		assert(c->block[i].ref == 0);
+		vtfree(c->block[i].data);
+	}
 
 	vtfree(c->hash);
 	vtfree(c->heap);
 	vtfree(c->block);
-	vtfree(c->mem);
 	vtfree(c);
 }
 
@@ -128,11 +132,10 @@ vtcachedump(VtCache *c)
 static void
 cachecheck(VtCache *c)
 {
-	u32int size, now;
+	u32int now;
 	int i, k, refed;
 	VtBlock *b;
 
-	size = c->blocksize;
 	now = c->now;
 
 	for(i = 0; i < c->nheap; i++){
@@ -151,8 +154,6 @@ cachecheck(VtCache *c)
 	refed = 0;
 	for(i = 0; i < c->nblock; i++){
 		b = &c->block[i];
-		if(b->data != &c->mem[i * size])
-			sysfatal("mis-blocked at %d", i);
 		if(b->ref && b->heap == BadHeap)
 			refed++;
 		else if(b->addr != NilBlock)
@@ -298,6 +299,57 @@ if(0)fprint(2, "droping %x:%V\n", b->addr, b->score);
 	return b;
 }
 
+/*
+ * evict blocks until there is enough memory for size bytes.
+ */
+static VtBlock*
+vtcacheevict(VtCache *c, ulong size)
+{
+	VtBlock *b;
+	
+	/*
+	 * If we were out of memory and put some blocks
+	 * to the side but now we have memory, grab one.
+	 */
+	if(c->mem >= size && c->dead) {
+		b = c->dead;
+		c->dead = b->next;
+		b->next = nil;
+		goto alloc;
+	}
+	
+	/*
+	 * Otherwise, evict until we have memory.
+	 */
+	for(;;) {
+		b = vtcachebumpblock(c);
+		if(c->mem+b->size >= size)
+			break;
+		/*
+		 * chain b onto dead list
+		 */
+		free(b->data);
+		b->data = nil;
+		c->mem += b->size;
+		b->size = 0;
+		b->next = c->dead;
+		c->dead = b;
+	}
+
+	/*
+	 * Allocate memory for block.
+	 */
+alloc:
+	if(size > b->size || size <= b->size/2) {
+		free(b->data);
+		c->mem += b->size;
+		c->mem -= size;
+		b->size = size;
+		b->data = vtmalloc(size);
+	}
+	return b;
+}
+
 /*
  * fetch a local block from the memory cache.
  * if it's not there, load it, bumping some other Block.
@@ -332,16 +384,16 @@ vtcachelocal(VtCache *c, u32int addr, int type)
 }
 
 VtBlock*
-vtcacheallocblock(VtCache *c, int type)
+vtcacheallocblock(VtCache *c, int type, ulong size)
 {
 	VtBlock *b;
 
 	qlock(&c->lk);
-	b = vtcachebumpblock(c);
+	b = vtcacheevict(c, size);
 	b->iostate = BioLocal;
 	b->type = type;
 	b->addr = (b - c->block)+1;
-	vtzeroextend(type, b->data, 0, c->blocksize);
+	vtzeroextend(type, b->data, 0, size);
 	vtlocaltoglobal(b->addr, b->score);
 	qunlock(&c->lk);
 
@@ -356,7 +408,7 @@ vtcacheallocblock(VtCache *c, int type)
  * if it's not there, load it, bumping some other block.
  */
 VtBlock*
-vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type)
+vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type, ulong size)
 {
 	VtBlock *b;
 	ulong h;
@@ -409,7 +461,7 @@ vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type)
 	/*
 	 * not found
 	 */
-	b = vtcachebumpblock(c);
+	b = vtcacheevict(c, size);
 	b->addr = NilBlock;
 	b->type = type;
 	memmove(b->score, score, VtScoreSize);
@@ -435,7 +487,7 @@ vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type)
 	qunlock(&c->lk);
 
 	vtcachenread++;
-	n = vtread(c->z, score, type, b->data, c->blocksize);
+	n = vtread(c->z, score, type, b->data, size);
 	if(n < 0){
 		if(chattyventi)
 			fprint(2, "read %V: %r\n", score);
@@ -445,7 +497,7 @@ vtcacheglobal(VtCache *c, uchar score[VtScoreSize], int type)
 		vtblockput(b);
 		return nil;
 	}
-	vtzeroextend(type, b->data, n, c->blocksize);
+	vtzeroextend(type, b->data, n, size);
 	b->iostate = BioVenti;
 	b->nlock = 1;
 	if(vttracelevel)
@@ -534,7 +586,7 @@ vtblockwrite(VtBlock *b)
 	}
 
 	c = b->c;
-	n = vtzerotruncate(b->type, b->data, c->blocksize);
+	n = vtzerotruncate(b->type, b->data, b->size);
 	vtcachenwrite++;
 	if(c->write(c->z, score, b->type, b->data, n) < 0)
 		return -1;
@@ -554,24 +606,18 @@ vtblockwrite(VtBlock *b)
 	return 0;
 }
 
-uint
-vtcacheblocksize(VtCache *c)
-{
-	return c->blocksize;
-}
-
 VtBlock*
 vtblockcopy(VtBlock *b)
 {
 	VtBlock *bb;
 
 	vtcachencopy++;
-	bb = vtcacheallocblock(b->c, b->type);
+	bb = vtcacheallocblock(b->c, b->type, b->size);
 	if(bb == nil){
 		vtblockput(b);
 		return nil;
 	}
-	memmove(bb->data, b->data, b->c->blocksize);
+	memmove(bb->data, b->data, b->size);
 	vtblockput(b);
 	bb->pc = getcallerpc(&b);
 	return bb;

src/libventi/entry.c

@@ -6,13 +6,37 @@
 static int
 checksize(int n)
 {
-	if(n < 256 || n > VtMaxLumpSize) {
+	if(n < 256) {
 		werrstr("bad block size %#ux", n);
 		return -1;
 	}
 	return 0;
 }
 
+// _VtEntryBig integer format is floating-point:
+// (n>>5) << (n&31).
+// Convert this number; must be exact or return -1.
+int
+vttobig(ulong n)
+{
+	int shift;
+	ulong n0;
+	
+	n0 = n;
+	shift = 0;
+	while(n >= (1<<(16 - 5))) {
+		if(n & 1)
+			return -1;
+		shift++;
+		n >>= 1;
+	}
+	
+	n = (n<<5) | shift;
+	if(((n>>5)<<(n&31)) != n0)
+		sysfatal("vttobig %#lux => %#lux failed", n0, n);
+	return n;
+}
+
 void
 vtentrypack(VtEntry *e, uchar *p, int index)
 {
@@ -20,21 +44,31 @@ vtentrypack(VtEntry *e, uchar *p, int index)
 	int flags;
 	uchar *op;
 	int depth;
+	int psize, dsize;
 
 	p += index * VtEntrySize;
 	op = p;
 
-	U32PUT(p, e->gen);
-	p += 4;
-	U16PUT(p, e->psize);
-	p += 2;
-	U16PUT(p, e->dsize);
-	p += 2;
 	depth = e->type&VtTypeDepthMask;
 	flags = (e->flags&~(_VtEntryDir|_VtEntryDepthMask));
 	flags |= depth << _VtEntryDepthShift;
 	if(e->type - depth == VtDirType)
 		flags |= _VtEntryDir;
+	U32PUT(p, e->gen);
+	p += 4;
+	psize = e->psize;
+	dsize = e->dsize;
+	if(psize >= (1<<16) || dsize >= (1<<16)) {
+		flags |= _VtEntryBig;
+		psize = vttobig(psize);
+		dsize = vttobig(dsize);
+		if(psize < 0 || dsize < 0)
+			sysfatal("invalid entry psize/dsize: %d/%d", e->psize, e->dsize);
+	}
+	U16PUT(p, psize);
+	p += 2;
+	U16PUT(p, dsize);
+	p += 2;
 	U8PUT(p, flags);
 	p++;
 	memset(p, 0, 5);
@@ -62,10 +96,14 @@ vtentryunpack(VtEntry *e, uchar *p, int index)
 	e->dsize = U16GET(p);
 	p += 2;
 	e->flags = U8GET(p);
+	p++;
+	if(e->flags & _VtEntryBig) {
+		e->psize = (e->psize>>5)<<(e->psize & 31);
+		e->dsize = (e->dsize>>5)<<(e->dsize & 31);
+	}
 	e->type = (e->flags&_VtEntryDir) ? VtDirType : VtDataType;
 	e->type += (e->flags & _VtEntryDepthMask) >> _VtEntryDepthShift;
-	e->flags &= ~(_VtEntryDir|_VtEntryDepthMask);
+	e->flags &= ~(_VtEntryDir|_VtEntryDepthMask|_VtEntryBig);
-	p++;
 	p += 5;
 	e->size = U48GET(p);
 	p += 6;

src/libventi/file.c

@@ -36,7 +36,6 @@ static VtFile *
 vtfilealloc(VtCache *c, VtBlock *b, VtFile *p, u32int offset, int mode)
 {
 	int epb;
-	u32int size;
 	VtEntry e;
 	VtFile *r;
 
@@ -74,15 +73,9 @@ vtfilealloc(VtCache *c, VtBlock *b, VtFile *p, u32int offset, int mode)
 		return nil;
 	}
 
-	size = vtcacheblocksize(c);
-	if(e.dsize > size || e.psize > size){
-		werrstr("block sizes %ud, %ud bigger than cache block size %ud",
-			e.psize, e.dsize, size);
-		return nil;
-	}
-
 	r = vtmallocz(sizeof(VtFile));
 	r->c = c;
+	r->bsize = b->size;
 	r->mode = mode;
 	r->dsize = e.dsize;
 	r->psize = e.psize;
@@ -126,7 +119,7 @@ vtfileopenroot(VtCache *c, VtEntry *e)
 	VtBlock *b;
 	VtFile *f;
 
-	b = vtcacheallocblock(c, VtDirType);
+	b = vtcacheallocblock(c, VtDirType, VtEntrySize);
 	if(b == nil)
 		return nil;
 
@@ -191,8 +184,6 @@ _vtfilecreate(VtFile *r, int o, int psize, int dsize, int type)
 	u32int offset;
 	
 	assert(ISLOCKED(r));
-	assert(psize <= VtMaxLumpSize);
-	assert(dsize <= VtMaxLumpSize);
 	assert(type == VtDirType || type == VtDataType);
 
 	if(!r->dir){
@@ -325,12 +316,12 @@ vtfilegetsize(VtFile *r)
 static int
 shrinksize(VtFile *r, VtEntry *e, uvlong size)
 {
-	int i, depth, type, isdir, ppb;
+	int i, depth, bsiz, type, isdir, ppb;
 	uvlong ptrsz;
 	uchar score[VtScoreSize];
 	VtBlock *b;
 
-	b = vtcacheglobal(r->c, e->score, e->type);
+	b = vtcacheglobal(r->c, e->score, e->type, r->dsize);
 	if(b == nil)
 		return -1;
 
@@ -369,7 +360,11 @@ shrinksize(VtFile *r, VtEntry *e, uvlong size)
 		type--;
 		memmove(score, b->data+i*VtScoreSize, VtScoreSize);
 		vtblockput(b);
-		b = vtcacheglobal(r->c, score, type);
+		if(type == VtDataType || type == VtDirType)
+			bsiz = r->dsize;
+		else
+			bsiz = r->psize;
+		b = vtcacheglobal(r->c, score, type, bsiz);
 		if(b == nil)
 			return -1;
 	}
@@ -498,10 +493,10 @@ vtfilesetentry(VtFile *r, VtEntry *e)
 }
 
 static VtBlock *
-blockwalk(VtBlock *p, int index, VtCache *c, int mode, VtEntry *e)
+blockwalk(VtFile *r, VtBlock *p, int index, VtCache *c, int mode, VtEntry *e)
 {
 	VtBlock *b;
-	int type;
+	int type, size;
 	uchar *score;
 	VtEntry oe;
 
@@ -519,12 +514,16 @@ blockwalk(VtBlock *p, int index, VtCache *c, int mode, VtEntry *e)
 	}
 /*print("walk from %V/%d ty %d to %V ty %d\n", p->score, index, p->type, score, type); */
 
+	if(type == VtDirType || type == VtDataType)
+		size = r->dsize;
+	else
+		size = r->psize;
 	if(mode == VtOWRITE && vtglobaltolocal(score) == NilBlock){
-		b = vtcacheallocblock(c, type);
+		b = vtcacheallocblock(c, type, size);
 		if(b)
 			goto HaveCopy;
 	}else
-		b = vtcacheglobal(c, score, type);
+		b = vtcacheglobal(c, score, type, size);
 
 	if(b == nil || mode == VtOREAD)
 		return b;
@@ -566,7 +565,7 @@ growdepth(VtFile *r, VtBlock *p, VtEntry *e, int depth)
 	assert(ISLOCKED(r));
 	assert(depth <= VtPointerDepth);
 
-	b = vtcacheglobal(r->c, e->score, e->type);
+	b = vtcacheglobal(r->c, e->score, e->type, r->dsize);
 	if(b == nil)
 		return -1;
 
@@ -577,7 +576,7 @@ growdepth(VtFile *r, VtBlock *p, VtEntry *e, int depth)
 	 * or an error occurs.
 	 */
 	while(DEPTH(e->type) < depth){
-		bb = vtcacheallocblock(r->c, e->type+1);
+		bb = vtcacheallocblock(r->c, e->type+1, r->psize);
 		if(bb == nil)
 			break;
 		memmove(bb->data, b->score, VtScoreSize);
@@ -605,7 +604,7 @@ shrinkdepth(VtFile *r, VtBlock *p, VtEntry *e, int depth)
 	assert(ISLOCKED(r));
 	assert(depth <= VtPointerDepth);
 
-	rb = vtcacheglobal(r->c, e->score, e->type);
+	rb = vtcacheglobal(r->c, e->score, e->type, r->psize);
 	if(rb == nil)
 		return -1;
 
@@ -618,7 +617,7 @@ shrinkdepth(VtFile *r, VtBlock *p, VtEntry *e, int depth)
 	ob = nil;
 	b = rb;
 	for(; DEPTH(e->type) > depth; e->type--){
-		nb = vtcacheglobal(r->c, b->data, e->type-1);
+		nb = vtcacheglobal(r->c, b->data, e->type-1, r->psize);
 		if(nb == nil)
 			break;
 		if(ob!=nil && ob!=rb)
@@ -720,7 +719,7 @@ assert(b->type == VtDirType);
 		m = VtORDWR;
 
 	for(i=DEPTH(e.type); i>=0; i--){
-		bb = blockwalk(b, index[i], r->c, i==0 ? mode : m, &e);
+		bb = blockwalk(r, b, index[i], r->c, i==0 ? mode : m, &e);
 		if(bb == nil)
 			goto Err;
 		vtblockput(b);
@@ -768,7 +767,7 @@ vtfileblockscore(VtFile *r, u32int bn, uchar score[VtScoreSize])
 	index[DEPTH(e.type)] = r->offset % r->epb;
 
 	for(i=DEPTH(e.type); i>=1; i--){
-		bb = blockwalk(b, index[i], r->c, VtOREAD, &e);
+		bb = blockwalk(r, b, index[i], r->c, VtOREAD, &e);
 		if(bb == nil)
 			goto Err;
 		vtblockput(b);
@@ -837,7 +836,7 @@ fileloadblock(VtFile *r, int mode)
 	case VtORDWR:
 		assert(r->mode == VtORDWR);
 		if(r->local == 1){
-			b = vtcacheglobal(r->c, r->score, VtDirType);
+			b = vtcacheglobal(r->c, r->score, VtDirType, r->bsize);
 			if(b == nil)
 				return nil;
 			b->pc = getcallerpc(&r);
@@ -861,7 +860,7 @@ fileloadblock(VtFile *r, int mode)
 		}
 		addr = vtglobaltolocal(r->score);
 		if(addr == NilBlock)
-			return vtcacheglobal(r->c, r->score, VtDirType);
+			return vtcacheglobal(r->c, r->score, VtDirType, r->bsize);
 
 		b = vtcachelocal(r->c, addr, VtDirType);
 		if(b)
@@ -1220,7 +1219,7 @@ vtfileflushbefore(VtFile *r, u64int offset)
 	 */
 	index[depth] = r->offset % r->epb;
 	for(i=depth; i>=0; i--){
-		bb = blockwalk(b, index[i], r->c, VtORDWR, &e);
+		bb = blockwalk(r, b, index[i], r->c, VtORDWR, &e);
 		if(bb == nil)
 			goto Err;
 		bi[i] = bb;

src/libventi/root.c

@@ -6,19 +6,30 @@
 static int
 checksize(int n)
 {
-	if(n < 256 || n > VtMaxLumpSize) {
+	if(n < 256) {
 		werrstr("bad block size");
 		return -1;
 	}
 	return 0;
 }
 
+extern int vttobig(ulong);
+
 void
 vtrootpack(VtRoot *r, uchar *p)
 {
 	uchar *op = p;
+	int vers, bsize;
 
-	U16PUT(p, VtRootVersion);
+	vers = VtRootVersion;
+	bsize = r->blocksize;
+	if(bsize >= (1<<16)) {
+		vers |= _VtRootVersionBig;
+		bsize = vttobig(bsize);
+		if(bsize < 0)
+			sysfatal("invalid root blocksize: %#x", r->blocksize);
+	}
+	U16PUT(p, vers);
 	p += 2;
 	memmove(p, r->name, sizeof(r->name));
 	p += sizeof(r->name);
@@ -26,7 +37,7 @@ vtrootpack(VtRoot *r, uchar *p)
 	p += sizeof(r->type);
 	memmove(p, r->score, VtScoreSize);
 	p +=  VtScoreSize;
-	U16PUT(p, r->blocksize);
+	U16PUT(p, bsize);
 	p += 2;
 	memmove(p, r->prev, VtScoreSize);
 	p += VtScoreSize;
@@ -42,7 +53,7 @@ vtrootunpack(VtRoot *r, uchar *p)
 	memset(r, 0, sizeof(*r));
 
 	vers = U16GET(p);
-	if(vers != VtRootVersion) {
+	if((vers&~_VtRootVersionBig) != VtRootVersion) {
 		werrstr("unknown root version");
 		return -1;
 	}
@@ -56,6 +67,8 @@ vtrootunpack(VtRoot *r, uchar *p)
 	memmove(r->score, p, VtScoreSize);
 	p +=  VtScoreSize;
 	r->blocksize = U16GET(p);
+	if(vers & _VtRootVersionBig)
+		r->blocksize = (r->blocksize >> 5) << (r->blocksize & 31);
 	if(checksize(r->blocksize) < 0)
 		return -1;
 	p += 2;