/*
 * Copyright (c) 1983 Regents of the University of California.
 * All rights reserved.  The Berkeley software License Agreement
 * specifies the terms and conditions for redistribution.
 */

/*
 * static char sccsid[] = "@(#)ndbm.c	5.4 (Berkeley) 9/4/87";
 */

#ident	"$Header: ndbm.c 1.3 89/07/18 $@(#)ndbm.c  5.4 (Berkeley) 9/4/87"

/*
 * $Copyright:	$
 * Copyright (c) 1984, 1985, 1986, 1987, 1988, 1989, 1990 
 * Sequent Computer Systems, Inc.   All rights reserved.
 *  
 * This software is furnished under a license and may be used
 * only in accordance with the terms of that license and with the
 * inclusion of the above copyright notice.   This software may not
 * be provided or otherwise made available to, or used by, any
 * other person.  No title to or ownership of the software is
 * hereby transferred.
 *
 * -------------------------------------------------------------------
 *
 * This routine originally taken from the 4.3-tahoe release.
 * Code has been added to do some consistency checking.  Should the
 * data base be corrupted the original code would often generate bad
 * memory references and take a segmentation fault.
 *
 * >>>  IMPORTANT NOTE:
 *
 * End of file contains comments from the net that are useful if
 * trying to understand operation of the data base.
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <errno.h>
#include "ndbm.h"

#define BYTESIZ 8
#undef setbit

#ifndef L_SET
#define L_SET	0
#endif

static  datum makdatum();
static  long hashinc();
static  long dcalchash();
extern  int errno;
static	int internal_error;	/* pass errors up from low-level routines */

			/* See end of file. */
static void bcopy();	/* C version that handles overlapped copies. */

DBM *
dbm_open(file, flags, mode)
	char *file;
	int flags, mode;
{
	struct stat statb;
	register DBM *db;

	if ((db = (DBM *)malloc(sizeof *db)) == 0) {
		errno = ENOMEM;
		return ((DBM *)0);
	}
	db->dbm_flags = (flags & 03) == O_RDONLY ? _DBM_RDONLY : 0;
	if ((flags & 03) == O_WRONLY)
		flags = (flags & ~03) | O_RDWR;
	strcpy(db->dbm_pagbuf, file);
	strcat(db->dbm_pagbuf, ".pag");
	db->dbm_pagf = open(db->dbm_pagbuf, flags, mode);
	if (db->dbm_pagf < 0)
		goto bad;
	strcpy(db->dbm_pagbuf, file);
	strcat(db->dbm_pagbuf, ".dir");
	db->dbm_dirf = open(db->dbm_pagbuf, flags, mode);
	if (db->dbm_dirf < 0)
		goto bad1;
	fstat(db->dbm_dirf, &statb);
	db->dbm_maxbno = statb.st_size*BYTESIZ-1;
	db->dbm_pagbno = db->dbm_dirbno = -1;
	return (db);
bad1:
	(void) close(db->dbm_pagf);
bad:
	free((char *)db);
	return ((DBM *)0);
}

void
dbm_blkflush(db)
	DBM *db;
{
	struct stat statb;

	fstat(db->dbm_dirf, &statb);
	db->dbm_maxbno = statb.st_size*BYTESIZ-1;
	db->dbm_pagbno = db->dbm_dirbno = -1;
}

void
dbm_close(db)
	DBM *db;
{
	(void) close(db->dbm_dirf);
	(void) close(db->dbm_pagf);
	free((char *)db);
}

long
dbm_forder(db, key)
	register DBM *db;
	datum key;
{
	long hash;

	internal_error = 0;
	hash = dcalchash(key);
	for (db->dbm_hmask=0;; db->dbm_hmask=(db->dbm_hmask<<1)+1) {
		db->dbm_blkno = hash & db->dbm_hmask;
		db->dbm_bitno = db->dbm_blkno + db->dbm_hmask;
		if (getbit(db) == 0)
			break;
	}
	return (db->dbm_blkno);
}

datum
dbm_fetch(db, key)
	register DBM *db;
	datum key;
{
	register i;
	datum item;

	internal_error = 0;
	if (dbm_error(db))
		goto err;
	dbm_access(db, dcalchash(key));
	if ((i = finddatum(db->dbm_pagbuf, key)) >= 0) {
		item = makdatum(db->dbm_pagbuf, i+1);
		if (item.dptr != NULL)
			return (item);
	}

	if (internal_error)
		db->dbm_flags |= _DBM_IOERR;

err:
	item.dptr = NULL;
	item.dsize = 0;
	return (item);
}

dbm_delete(db, key)
	register DBM *db;
	datum key;
{
	register i;
	datum item;

	internal_error = 0;
	if (dbm_error(db))
		return (-1);
	if (dbm_rdonly(db)) {
		errno = EPERM;
		return (-1);
	}
	dbm_access(db, dcalchash(key));
	if ((i = finddatum(db->dbm_pagbuf, key)) < 0)
		return (-1);
	if (!delitem(db->dbm_pagbuf, i))
		goto err;
	db->dbm_pagbno = db->dbm_blkno;
	(void) lseek(db->dbm_pagf, db->dbm_blkno*PBLKSIZ, L_SET);
	if (write(db->dbm_pagf, db->dbm_pagbuf, PBLKSIZ) != PBLKSIZ) {
err:
		db->dbm_flags |= _DBM_IOERR;
		return (-1);
	}
	return (0);
}

dbm_store(db, key, dat, replace)
	register DBM *db;
	datum key, dat;
	int replace;
{
	register i;
	datum item, item1;
	char ovfbuf[PBLKSIZ];

	internal_error = 0;
	if (dbm_error(db))
		return (-1);
	if (dbm_rdonly(db)) {
		errno = EPERM;
		return (-1);
	}
loop:
	dbm_access(db, dcalchash(key));
	if ((i = finddatum(db->dbm_pagbuf, key)) >= 0) {
		if (!replace)
			return (1);
		if (!delitem(db->dbm_pagbuf, i)) {
			db->dbm_flags |= _DBM_IOERR;
			return (-1);
		}
	}
	if (internal_error) {
		db->dbm_flags |= _DBM_IOERR;
		return(-1);
	}
	if (!additem(db->dbm_pagbuf, key, dat)) {
		if (internal_error) {
			db->dbm_flags |= _DBM_IOERR;
			return(-1);
		}
		goto split;
	}
	db->dbm_pagbno = db->dbm_blkno;
	(void) lseek(db->dbm_pagf, db->dbm_blkno*PBLKSIZ, L_SET);
	if (write(db->dbm_pagf, db->dbm_pagbuf, PBLKSIZ) != PBLKSIZ) {
		db->dbm_flags |= _DBM_IOERR;
		return (-1);
	}
	return (0);

split:

	if (key.dsize+dat.dsize+3*sizeof(short) >= PBLKSIZ) {
		db->dbm_flags |= _DBM_IOERR;
		errno = ENOSPC;
		return (-1);
	}
#ifdef _BSD_
	bzero(ovfbuf, PBLKSIZ);
#else
	(void) memset(ovfbuf, '\0', PBLKSIZ);
#endif
	for (i=0;;) {
		item = makdatum(db->dbm_pagbuf, i);
		if (item.dptr == NULL)
			break;
		if (dcalchash(item) & (db->dbm_hmask+1)) {
			item1 = makdatum(db->dbm_pagbuf, i+1);
			if (item1.dptr == NULL) {
				fprintf(stderr, "ndbm: split not paired\n");
				db->dbm_flags |= _DBM_IOERR;
				break;
			}
			if (!additem(ovfbuf, item, item1) ||
			    !delitem(db->dbm_pagbuf, i)) {
				db->dbm_flags |= _DBM_IOERR;
				return (-1);
			}
			continue;
		}
		i += 2;
	}
	db->dbm_pagbno = db->dbm_blkno;
	(void) lseek(db->dbm_pagf, db->dbm_blkno*PBLKSIZ, L_SET);
	if (write(db->dbm_pagf, db->dbm_pagbuf, PBLKSIZ) != PBLKSIZ) {
		db->dbm_flags |= _DBM_IOERR;
		return (-1);
	}
	(void) lseek(db->dbm_pagf, (db->dbm_blkno+db->dbm_hmask+1)*PBLKSIZ, L_SET);
	if (write(db->dbm_pagf, ovfbuf, PBLKSIZ) != PBLKSIZ) {
		db->dbm_flags |= _DBM_IOERR;
		return (-1);
	}
	setbit(db);
	goto loop;
}

datum
dbm_firstkey(db)
	DBM *db;
{

	db->dbm_blkptr = 0L;
	db->dbm_keyptr = 0;
	return (dbm_nextkey(db));
}

datum
dbm_nextkey(db)
	register DBM *db;
{
	struct stat statb;
	datum item;

	internal_error = 0;
	if (dbm_error(db) || fstat(db->dbm_pagf, &statb) < 0)
		goto err;
	statb.st_size /= PBLKSIZ;
	for (;;) {
		if (db->dbm_blkptr != db->dbm_pagbno) {
			db->dbm_pagbno = db->dbm_blkptr;
			(void) lseek(db->dbm_pagf, db->dbm_blkptr*PBLKSIZ, L_SET);
			if (read(db->dbm_pagf, db->dbm_pagbuf, PBLKSIZ) != PBLKSIZ)
#ifdef _BSD_
				bzero(db->dbm_pagbuf, PBLKSIZ);
#else
				(void) memset(db->dbm_pagbuf, '\0', PBLKSIZ);
#endif
#ifdef DEBUG
			else if (chkblk(db->dbm_pagbuf) < 0)
				db->dbm_flags |= _DBM_IOERR;
#endif
		}
		if (((short *)db->dbm_pagbuf)[0] != 0) {
			item = makdatum(db->dbm_pagbuf, db->dbm_keyptr);
			if (item.dptr != NULL) {
				db->dbm_keyptr += 2;
				return (item);
			}
			db->dbm_keyptr = 0;
		}
		if (++db->dbm_blkptr >= statb.st_size)
			break;
	}
err:
	item.dptr = NULL;
	item.dsize = 0;
	return (item);
}

static
dbm_access(db, hash)
	register DBM *db;
	long hash;
{

	for (db->dbm_hmask=0;; db->dbm_hmask=(db->dbm_hmask<<1)+1) {
		db->dbm_blkno = hash & db->dbm_hmask;
		db->dbm_bitno = db->dbm_blkno + db->dbm_hmask;
		if (getbit(db) == 0)
			break;
	}
	if (db->dbm_blkno != db->dbm_pagbno) {
		db->dbm_pagbno = db->dbm_blkno;
		(void) lseek(db->dbm_pagf, db->dbm_blkno*PBLKSIZ, L_SET);
		if (read(db->dbm_pagf, db->dbm_pagbuf, PBLKSIZ) != PBLKSIZ)
#ifdef _BSD_
			bzero(db->dbm_pagbuf, PBLKSIZ);
#else
			(void) memset(db->dbm_pagbuf, '\0', PBLKSIZ);
#endif
#ifdef DEBUG
		else if (chkblk(db->dbm_pagbuf) < 0)
			db->dbm_flags |= _DBM_IOERR;
#endif
	}
}

static
getbit(db)
	register DBM *db;
{
	long bn;
	register b, i, n;
	

	if (db->dbm_bitno > db->dbm_maxbno)
		return (0);
	n = db->dbm_bitno % BYTESIZ;
	bn = db->dbm_bitno / BYTESIZ;
	i = bn % DBLKSIZ;
	b = bn / DBLKSIZ;
	if (b != db->dbm_dirbno) {
		db->dbm_dirbno = b;
		(void) lseek(db->dbm_dirf, (long)b*DBLKSIZ, L_SET);
		if (read(db->dbm_dirf, db->dbm_dirbuf, DBLKSIZ) != DBLKSIZ)
#ifdef _BSD_
			bzero(db->dbm_dirbuf, DBLKSIZ);
#else
			(void) memset(db->dbm_dirbuf, '\0',  DBLKSIZ);
#endif
	}
	return (db->dbm_dirbuf[i] & (1<<n));
}

static
setbit(db)
	register DBM *db;
{
	long bn;
	register i, n, b;

	if (db->dbm_bitno > db->dbm_maxbno)
		db->dbm_maxbno = db->dbm_bitno;
	n = db->dbm_bitno % BYTESIZ;
	bn = db->dbm_bitno / BYTESIZ;
	i = bn % DBLKSIZ;
	b = bn / DBLKSIZ;
	if (b != db->dbm_dirbno) {
		db->dbm_dirbno = b;
		(void) lseek(db->dbm_dirf, (long)b*DBLKSIZ, L_SET);
		if (read(db->dbm_dirf, db->dbm_dirbuf, DBLKSIZ) != DBLKSIZ)
#ifdef _BSD_
			bzero(db->dbm_dirbuf, DBLKSIZ);
#else
			(void) memset(db->dbm_dirbuf, '\0', DBLKSIZ);
#endif
	}
	db->dbm_dirbuf[i] |= 1<<n;
	db->dbm_dirbno = b;
	(void) lseek(db->dbm_dirf, (long)b*DBLKSIZ, L_SET);
	if (write(db->dbm_dirf, db->dbm_dirbuf, DBLKSIZ) != DBLKSIZ)
		db->dbm_flags |= _DBM_IOERR;
}

static datum
makdatum(buf, n)
	char buf[PBLKSIZ];
{
	register short *sp;
	register t;
	datum item;

	sp = (short *)buf;
	if ((unsigned)n >= sp[0]) {
		item.dptr = NULL;
		item.dsize = 0;
		return (item);
	}
	t = PBLKSIZ;
	if (n > 0)
		t = sp[n];
	item.dptr = buf+sp[n+1];
	item.dsize = t - sp[n+1];
	return (item);
}

static
finddatum(buf, item)
	char buf[PBLKSIZ];
	datum item;
{
	register short *sp;
	register int i, n, j;

	sp = (short *)buf;
	n = PBLKSIZ;
	for (i=0, j=sp[0]; i<j; i+=2, n = sp[i]) {
		/* insure iterations stay within block */
		if (i >= (PBLKSIZ/(sizeof(short)))) {
			internal_error = 1;
			return(-1);
		}

		/*
		 * if (a || b)
		 *  a: end of key (n) extends out of buffer
		 *  b: beginning of key after end of key
		 */

		if ( (n > PBLKSIZ) || (sp[i+1] > n) ) {
			internal_error = 1;
			return(-1);
		}
		n -= sp[i+1];
		if (n != item.dsize)
			continue;
#ifdef _BSD_
		if (n == 0 || bcmp(&buf[sp[i+1]], item.dptr, n) == 0)
#else
		if (n == 0 || memcmp(item.dptr, &buf[sp[i+1]], n) == 0)
#endif
		{
			/*
			 * if (a || b)
			 *  a: beginning of datum is before beginning of block
			 *  b: beginning of datum is after end of datum
			 */

			if ( (sp[i+2] < 0) || (sp[i+2] > sp[i+1]) ) {
				internal_error = 1;
				return(-1);
			}
			return (i);
		}
	}
	return (-1);
}

static  int hitab[16]
/* ken's
{
	055,043,036,054,063,014,004,005,
	010,064,077,000,035,027,025,071,
};
*/
 = {    61, 57, 53, 49, 45, 41, 37, 33,
	29, 25, 21, 17, 13,  9,  5,  1,
};
static  long hltab[64]
 = {
	06100151277L,06106161736L,06452611562L,05001724107L,
	02614772546L,04120731531L,04665262210L,07347467531L,
	06735253126L,06042345173L,03072226605L,01464164730L,
	03247435524L,07652510057L,01546775256L,05714532133L,
	06173260402L,07517101630L,02431460343L,01743245566L,
	00261675137L,02433103631L,03421772437L,04447707466L,
	04435620103L,03757017115L,03641531772L,06767633246L,
	02673230344L,00260612216L,04133454451L,00615531516L,
	06137717526L,02574116560L,02304023373L,07061702261L,
	05153031405L,05322056705L,07401116734L,06552375715L,
	06165233473L,05311063631L,01212221723L,01052267235L,
	06000615237L,01075222665L,06330216006L,04402355630L,
	01451177262L,02000133436L,06025467062L,07121076461L,
	03123433522L,01010635225L,01716177066L,05161746527L,
	01736635071L,06243505026L,03637211610L,01756474365L,
	04723077174L,03642763134L,05750130273L,03655541561L,
};

static long
hashinc(db, hash)
	register DBM *db;
	long hash;
{
	long bit;

	hash &= db->dbm_hmask;
	bit = db->dbm_hmask+1;
	for (;;) {
		bit >>= 1;
		if (bit == 0)
			return (0L);
		if ((hash & bit) == 0)
			return (hash | bit);
		hash &= ~bit;
	}
}

static long
dcalchash(item)
	datum item;
{
	register int s, c, j;
	register char *cp;
	register long hashl;
	register int hashi;

	hashl = 0;
	hashi = 0;
	for (cp = item.dptr, s=item.dsize; --s >= 0; ) {
		c = *cp++;
		for (j=0; j<BYTESIZ; j+=4) {
			hashi += hitab[c&017];
			hashl += hltab[hashi&63];
			c >>= 4;
		}
	}
	return (hashl);
}

/*
 * Delete pairs of items (n & n+1).
 */
static
delitem(buf, n)
	char buf[PBLKSIZ];
{
	register short *sp, *sp1;
	register i1, i2;

	sp = (short *)buf;
	i2 = sp[0];
	if ((unsigned)n >= i2 || (n & 1))
		return (0);
	if (n == i2-2) {	/* deleting item in middle of block	*/
		sp[0] -= 2;	/* it borders on free area, so no 	*/
		return (1);	/* compaction is necessary		*/
	}
	i1 = PBLKSIZ;
	if (n > 0)
		i1 = sp[n];
	i1 -= sp[n+2];
	if (i1 > 0) {
		i2 = sp[i2];
		/* moved item has negative length */
		if ((sp[n+2] - i2) < 0) {
			internal_error = 1;
			return(0);
		}
		/* start or end location is outside of buffer */
		if ( (&buf[i2 + (sp[n+2] - i2)] > &buf[PBLKSIZ]) ||
		     (&buf[i2 +i1 + (sp[n+2] - i2)] > &buf[PBLKSIZ])) {
			internal_error = 1;
			return(0);
		}
		bcopy(&buf[i2], &buf[i2 + i1], sp[n+2] - i2);
	}
	/* This loop both does compaction and adjusts the offset pointers. */
	sp[0] -= 2;
	for (sp1 = sp + sp[0], sp += n+1; sp <= sp1; sp++)
		sp[0] = sp[2] + i1;
	return (1);
}

/*
 * Add pairs of items (item & item1).
 */
static
additem(buf, item, item1)
	char buf[PBLKSIZ];
	datum item, item1;
{
	register short *sp;
	register i1, i2;

	/* Make sure sizes are valid. */
	if ((item.dsize < 0) || (item1.dsize < 0)) {
		internal_error = 1;
		return(0);
	}
	sp = (short *)buf;
	i1 = PBLKSIZ;
	i2 = sp[0];
	/* Insure location of index of last item is within block. */
	if ( (i2 < 0) || (i2 >= (PBLKSIZ / (sizeof(short)))) ) {
		internal_error = 1;
		return(0);
	}
	if (i2 > 0)
		i1 = sp[i2];
	/* Insure last item falls within block. */
	if ( (i1 < 0) || (i1 > PBLKSIZ) ) {
		internal_error = 1;
		return(0);
	}
	i1 -= item.dsize + item1.dsize;
	if (i1 <= (i2+3) * (int)sizeof(short))
		return (0);	/* insufficient free space to add item */
	if ((&buf[i1 + item1.dsize + item.dsize] > &buf[PBLKSIZ]) ||
	    (&buf[i1 + item1.dsize] > &buf[PBLKSIZ]) ) {
		internal_error = 1;
		return(0);
	}
	sp[0] += 2;
	sp[++i2] = i1 + item1.dsize;
	bcopy(item.dptr, &buf[i1 + item1.dsize], item.dsize);
	sp[++i2] = i1;
	bcopy(item1.dptr, &buf[i1], item1.dsize);
	return (1);
}

#ifdef DEBUG
static
chkblk(buf)
	char buf[PBLKSIZ];
{
	register short *sp;
	register t, i;

	sp = (short *)buf;
	t = PBLKSIZ;
	for (i=0; i<sp[0]; i++) {
		if (sp[i+1] > t)
			return (-1);
		t = sp[i+1];
	}
	if (t < (sp[0]+1)*sizeof(short))
		return (-1);
	return (0);
}
#endif


/*
 * C version of bcopy that handles overlapped copies.
 *
 * bcopy(source, destination, count)
 *	char *source, *destination;
 *	unsigned int count;
 *
 * There are four cases to consider:
 *
 * 1. S < D no overlap		3. S < D with overlap**
 * 2. D < S no overlap		4. D < S with overlap
 *
 * ** - Case 3 must be done in the reverse direction.
 *
 */

static void
bcopy(s, d, count)
	char *s, *d;
	int count;
{
	if (s == d)				/* no work to do */
		return;

	if (d < s) {				/* case 2 or 3 */
		while ((count--) > 0)
			*d++ = *s++;
		return;
	}

	if ((s + count) > d) {			/* case 3 */
		s += count - 1;
		d += count - 1;
		while ((count--) > 0)
			*d-- = *s--;
		return;
	}

	while ((count--) > 0)			/* case 4 */
		*d++ = *s++;
}

/*
 ***************************************************************************
 ***************************************************************************
 *  The following comments are from the net.
 */

#ifdef _THIS_IS_A_COMMENT_FOR_NDBM_


Path: sequent!tektronix!uunet!seismo!sundc!pitstop!male!sun-arpa!ames!oliveb!
      apple!vsi1!icldata!altos86!dtynan
From: chris@mimsy.UUCP (Chris Torek)
Newsgroups: comp.unix
Subject: Re: dbm.a and ndbm.a archives
Date: 17 Apr 89 02:49:26 GMT
Sender: dtynan@altos86.UUCP
Organization: U of Maryland, Dept. of Computer Science, Coll. Pk., MD 20742

[Moderators Note:-  Another 'long' reply.  Although, this one probably
  has justification in the subject matter.  Save this everone!  - Der]

In article <976@altos86.UUCP> grego@unocss.unl.edu (Greg Ostravich) writes:
>I am looking to find out the format that 'dbm' and 'ndbm' use.

dbm and ndbm use the same format; dbm is implemented as a compatibility
layer on top of ndbm.  One description suffices for both.  The key idea
behind dbm is a variant of a database technique known as `extensible
hashing'.  In dbm, it works like this:

For each object that you wish to find again later (call this a `key'),
compute a 32 bit hash number.  Make the hash function depend on the
bits in the key, but sufficiently `random-looking' that two nearly-
identical keys give radically different hashes.  DES encryption, for
instance, would give a fairly well-spread-out bit spectrum.

For each key, the 32 bit hash number tells us in which `block' that key
can be found.  (A database can thus consist of up to 4 billion blocks.)
But, since we do not want to scatter eight keys across an entire file
system, we modify it thus:  Instead of using all 32 bits, we use only
as many bits as needed to make all the keys fit in their blocks.

Initially, we use no bits, and all keys fit in block zero: (hash & 0)
is always 0.  Every store goes to block 0.

Eventually, block 0 gets full.  We then declare that, for everything
that was in block 0, we will now use another bit.  Now some keys fit in
block 0, and some in block 1: (hash & 1) is either 0 or 1.  Move all
the keys that were in block 0, but should now be in block 1, to block
1.  (This is called a `split'.)

Now either block 0 or block 1 can get full.  If block 0 fills, we
declare that, for everything that (was in block 0 for 0 bits) and (was
in block 0 for 1 bit) should have two bits treated.  Change the mask
from 1 to 3, and instead of (hash&1)==0, we will have (hash&3) which
will be either 0 or 2.  (It cannot be 3 since (hash&1)==0.)  Move
the appropriate keys to block 2.

If block 1 fills instead, we do the same, but for block 1 rather
than zero; some of these keys move to block 3.  If both blocks fill,
we wind up doing this for both.

Now blocks 0, 1, 2, or 3 could fill.  If any one does, we declare that,
for everything that (was in block 0 for 0 bits) and (was in block 0/1
for 1 bit) and (was in block 0/1/2/3 for 2 bits) should have three bits
treated, and we change the mask from 3 to 7.  If all goes well, about
half the keys move to another block (from 0 to 4, 1 to 5, 2 to 6, or 3
to 7).  We repeat this process as often as necessary.

To find a key, we find its block number.  First, compute the 32 bit
hash value for that key.  Next, see if we declared that (block 0 for 0
bits) split.  If so, see if we declared that (block (hash&1) for 1 bit)
split; if so, see if (block (hash&3) for 2 bits) split; and so on,
until we finally reach a block that has not split.  The key is
either in this block, or not in the database at all.

The way we tell whether (block B for K bits) has split is to keep a bit
array (`.dir' file) with zeroes where blocks have not split and ones
where they have.  (Block B for K bits) is denoted by bit number (B +
(1<<K)-1).  EOF on the `.dir' file implies an infinite sea of zero
bits (so that only the 1 bits need be stored).

The search loop can be represented quite simply algorithmically:

	hash = calculate_hash(key);
	mask = 0;
	while (bit_is_set((hash & mask) + mask))
		mask = (mask << 1) | 1;
	block = hash & mask;

or more concretely (with the obvious optimisations)

	hash = calchash(key);
	for (hmask = 0;; hmask = (hmask << 1) | 1) {
		bit = (hash & hmask) + hmask;
		if (bit < max_possible_bit) {
			figure out which .dir `block' holds the bit;
			read it in, if not already in core;
			if (the bit is not set)
				break;
		}
	}
	block = hash & hmask;
	read that block, if not already in core;

The `max possible bit' value can be computed initially from the size
(in bytes) of the .dir file times the number of bits in a byte, and
updated in the `split' code whenever it sets a bit.

The above algorithm is the real heart of this extensible hash.  Once
you understand how and why it works, you can write a dbm: the rest is
just support structure.

The split algorithm is the next tricky part.  In pseudo-code:

	if (key will never fit in a single block)
		return (too big);	/* prevents looping below */
    store:
	hash = calchash(key);
	find the block as above;
	if (key is already here) {
		if (we are to overwrite it)
			delete it;
		else
			return (already there);
	}
	if (it fits) {
		jam it in;
		return (ok);
	}
    split:
	if (oldmask == ~0)
		database got full; /* how did this happen? */
	newbit = oldmask + 1;
	for (each object in the block) {
		hash = calchash(object);
		if (hash & newbit) {
			move object from this block to
				this block + newbit;
		} else {
			it stays here;
		}
	}
	goto store;	/* try again; should make it eventually */

It is, of course, not this simple.  If all you want to do is find keys,
this algorithm is fine; but dbm stores (key, content) pairs, so that
you can associate an unknown (such as a mail alias) with a known (the
name of the alias).  To do this, it stores these quite literally as
*pairs*: each `.pag' file block consists of a sequence of (key,
content; key, content; key, content) groupings.  Every even numbered
`item' is a key and every odd `item' is a content.  When searching a
block for a key, dbm looks only at the even items.  Upon finding it,
dbm returns the next item, which is that key's content.

Each item, whether key or content, is stored in a `.pag' block of
1024 bytes.  Each .pag block is treated as the union of:

	an array of shorts (size 1024/sizeof(short) or 512)
	an array of bytes (size 1024)

The bytes are used starting at the end, and the shorts starting at the
beginning.  The first short counts the total number of items (not
pairs, so it always winds up even) in the block.  The next tells where
the first item (item number zero) *begins*.  Item zero *ends* at byte
1024, non-inclusive.  The third short tells where the second item
begins; that item ends where the first item begins.  Items are thus
allocated from both ends, working towards the middle.  That is:

	int i, n;
	short *sp;
	char *endp, *startp;
	char blkbuf[BLKSIZE];

	sp = (short *)blkbuf;
	endp = blkbuf + BLKSIZE;
	n = *sp++;	/* sp now points to first item index */
	for (i = 0; i < n; i++) {
		startp = blkbuf + sp[i];
		/* item i is at bytes [startp..endp) */
		if (endp - startp == key.length &&
		    memcmp(startp, key.data, key.length) == 0)
			found it;
		endp = startp;
	}

except that we only want to look at even-numbered items (keys,
not contents).

The last task is iterating through the database.  To do this we make a
tricky observation: (imagine italics here)  The highest possible
numbered block in the database can be found by walking a tree formed of
the bits set during split operations.

Huh?

Put it this way: if (block 0 for 0 bits was split) and (block 0 for 1
bit was split) and (block 0 for two bits was split) and (block zero for
3 bits was split), but (block 0 for 4 bits was NOT split), then there
may be stuff in blocks 1 (for 1 bit), or 3 or 2 (for 2 bits), or 7 or 6
or 5 or 4 (for 3 bits), but definitely not in 8.  From (block 1 for 1
bit) we might split several times and get (9 for 4 bits), or from 2 or
3 or ... or 7 we might split to other blocks, but never to 8.  So we
use block 0 as a starting point.  After going over everything in block
0, we see how far we can go with block-zero-splits, by taking the mask
computed by that first loop when given a `hash' of 0.  If it was split
3 times, this will get us to block 4.  Here is how (as I slip into
TeXese):

Given a starting point hash value $h$ and the mask $m$ needed to
determine that keys with hash $h$ are in block $h & m$, set $h$ to
$h & m$ and set $b$ to $\log_2 (m + 1)$.  Then loop, decrementing
$b$, until either $b = -1$ or bit $b$ of $h$ is {\it not} set; clear
bit $b$ of $h$ and repeat.  If $b = -1$, you are done (there are
no more blocks); otherwise, the value $h | 2^b$ is the next block to
look at.  In C code:

	compute mask needed for hash, as before;
	hash &= mask;
	bit = mask + 1;		/* `bit' is 2^b */
	for (;;) {
		if ((bit >>= 1) == 0) {
			/* ran out of bits */
			return 0;	/* no more blocks */
		}
		if ((hash & bit) == 0)
			return (hash | bit);	/* do this block next */
		hash &= ~bit;
	}

I am not sure how to explain this intuitively, but it really does
hit all the blocks that have been used (and each only once).  This
`walks the tree' of hash-zero values (0, 4, 2, 1 in the example
above), inserting between each walk any blocks due to splits from
4, 2, or 1.  It does occasionally try a block that is empty, but
this is not a problem.

Within a block, the order is up to the person implementing firstkey()
and nextkey(); dbm produces them sorted, with the sort being based on
shortest first, then strcmp()-like order within same-length keys:

	if (key1.size < key2.size)
		return (-1);	/* less */
	if (key1.size > key2.size)
		return (1);	/* greater */
	return (compare(key1.dptr, key2.dptr, key1.dsize));
	/* where compare() is like strcmp() but does not treat \0 specially */

This has the advantage of needing no state across calls.

Last-minute details: the .pag block size is 1024 bytes; the
.dir file does not really have a block size, but 4.3BSD uses
4096 bytes (which is 32768 bits, and so covers up to about 32 MB
directly).  The external functions are

	#define dbm_dirfno(db)	((db)->dbm_dirf)
	#define dbm_pagfno(db)	((db)->dbm_pagf)

	typedef struct {
		char	*dptr;
		int	dsize;
	} datum;

	/*
	 * flags to dbm_store()
	 */
	#define DBM_INSERT	0
	#define DBM_REPLACE	1

	DBM	*dbm_open(char *name, int flags, int mode);
	void	dbm_close(DBM *db);
	datum	dbm_fetch(DBM *db, datum key);
	datum	dbm_firstkey(DBM *db);
	datum	dbm_nextkey(DBM *db, datum prevkey);
	int	dbm_delete(DBM *db, datum key);
	int	dbm_store(DBM *db, datum key, datum content, int flags);
-- 
In-Real-Life: Chris Torek, Univ of MD Comp Sci Dept (+1 301 454 7163)
Domain:	chris@mimsy.umd.edu	Path:	uunet!mimsy!chris


----------------------------------------------------------------------

Path: sequent!tektronix!uunet!attcan!utgpu!utzoo!yunexus!oz
From: oz@yunexus.UUCP (Ozan Yigit)
Newsgroups: comp.unix.wizards,comp.lang.c
Subject: dbm/ndbm notes and some code.
Keywords: ndbm, sdbm, dbm, extendible-hashing.
Message-ID: <1889@yunexus.UUCP>
Date: 11 May 89 19:33:53 GMT
Reply-To: oz@yunexus.UUCP (Ozan Yigit)
Organization: York U. Communications Research & Development
Lines: 234
Xref: sequent comp.unix.wizards:15206 comp.lang.c:17385

During my development of sdbm, a ndbm/dbm substitute (soon to beta: do not
send mail), I have found one bug that has gone unnoticed on probably most
of ndbm/dbm implementations.  One can write a simple workaround to this,
but a bug is a bug nevertheless.  Remember that line from the man page: 

| ...
| A linear pass through all keys in a database may be made, in an
| [apparently] random order, by use of db_firstkey and dbm_nextkey.
| ... This code will traverse the data base:
| 
| for (key=dbm_firstkey(db); key.dptr!=NULL;key=dbm_nextkey(db))
| ...

This is all very nice, if you just want to extract some or all keys.
But what if you want to "selectively delete" some keys ?? Something
like this perhaps:

for (key = dbm_firstkey(db); key.dptr != 0; key = dbm_nextkey(db)) {
	prdatum(key);
	printf(": delete? ");
	if (getyesno(stdin) == 'y')	/* illusory */
		dbm_delete(db, key);
}

This is where the "database traversal" turns into a pumpkin.  Because of
internal caching of the key position for dbm_nextkey (dbm_keyptr ??),
which is appearently NOT adjusted for deletions, this traversal will never
display the key right after the one just deleted. Workaround is to save
all keys to be deleted, then perform all deletions once the sequential
traversal is complete.

On another topic: If you ever wanted to process the ".pag" file of a
dbm/ndbm database yourself, in some unorthodox fashion, perhaps for
recovery, analysis or fast dumping of the keys/records in the pages, here
are the routines that illustrate the basic structure and the operations on
those pages. These routines are derived by careful "od" of the page file,
after test insertions, deletions etc., and NOT derived from the actual
source code of dbm/ndbm. (I hear it is easier to read od output anyway)
These routines are PD. [sdbm uses an ever so slightly different format:
there is an additional short after the n (entry count) field, but sdbm will
come with the appropriate utilities for analysis, recovery etc. anyway.]

---- pair.c -------------------------------------------------
#include <whatever suitable>

#define NULL (char *) 0

/*
 * routines dealing with a dbm/ndbm data page
 * author: oz@nexus.yorku.ca
 *
 * page format:
 *	+------------------------------+
 * ino	| n | keyoff | datoff | keyoff |
 * 	+------------+--------+--------+
 *	| datoff | - - - ---->	       |
 *	+--------+---------------------+
 *	|	 F R E E A R E A       |
 *	+--------------+---------------+
 *	|  <---- - - - | data          |
 *	+--------+-----+----+----------+
 *	|  key   | data     | key      |
 *	+--------+----------+----------+
 *
 * calculating the offsets for free area:  if the number
 * of entries (ino[0]) is zero, the offset to the END of
 * the free area is the block size. Otherwise, it is the
 * nth (ino[ino[0]]) entry's offset.
 */

putpair(pag, key, val)
char *pag;
datum key;
datum val;
{
	register n;
	register off;
	register free;
	register need;
	register short *ino = (short *) pag;

	off = ((n = ino[0]) > 0) ? ino[n] : PBLKSIZ;
/*
 * calculate free space and needed space
 */
	free = off - (n + 1) * sizeof(short);
	need = key.dsize + val.dsize + 2 * sizeof(short);
#ifdef DEBUG
	printf("offset %d: free %d need %d\n", off, free, need);
#endif
	if (need > free)
		return (0);
/*
 * enter the key first
 */
	off -= key.dsize;
	(void) memcpy(pag + off, key.dptr, key.dsize);
	ino[n + 1] = off;
/*
 * now the data
 */
	off -= val.dsize;
	(void) memcpy(pag + off, val.dptr, val.dsize);
	ino[n + 2] = off;
/*
 * adjust item count
 */
	ino[0] += 2;
	return (1);
}

datum null = { NULL, 0 };

datum
getpair(pag, key)
char *pag;
datum key;
{
	register i;
	register n;
	datum val;
	register short *ino = (short *) pag;

	if (!(n = ino[0]))
		return (null);

	if (!(i = seepair(pag, n, key.dptr, key.dsize)))
		return (null);

	val.dptr = pag + ino[i + 1];
	val.dsize = ino[i] - ino[i + 1];
	return (val);
}

delpair(pag, key)
char *pag;
datum key;
{
	register i;
	register n;
	register m;
	register zoo;
	register short *ino = (short *) pag;
	register char *src;
	register char *dst;

	if (!(n = ino[0]))
		return (0);

	if (!(i = seepair(pag, n, key.dptr, key.dsize)))
		return (0);
/*
 * found the key. if it is the last entry
 * [i.e. i == n - 1] we just adjust the entry count.
 * hard case: [i.e. 0 < i < n] move all data down onto the 
 * deleted pair, shift offsets onto deleted offsets, 
 * and adjust them.
 */
	if (i < n - 1) {
		dst = pag + (i == 1 ? PBLKSIZ : ino[i - 1]);
		src = pag + ino[i + 1];
		zoo = dst - src;
#ifdef DEBUG
		printf("free-up %d ", zoo);
#endif
	/*
	 * shift data/keys down
	 */
		m = ino[i + 1] - ino[n];
/*
 * should use memcpy here, but I do not trust all
 * implementations. Perhaps one day...
 */
#ifdef DUFF
#define MOVB 	*--dst = *--src;

		if (m > 0) {
			register int loop = (m + 8 - 1) >> 3;

			switch (m & (8 - 1)) {
			case 0:
				do {
				MOVB;	case 7:	MOVB;
			case 6:	MOVB;	case 5:	MOVB;
			case 4:	MOVB;	case 3:	MOVB;
			case 2:	MOVB;	case 1:	MOVB;
				} while (--loop);
			}
		}
#else
		while (m--)
			*--dst = *--src;
#endif
	/*
	 * adjust offset index
	 */
		while (i < n - 1) {
			ino[i] = ino[i + 2] + zoo;
			i++;
		}
	}
	ino[0] -= 2;
	return (1);
}

/*
 * search for the key in the page.
 * return offset index in the range 0 < i < n.
 * return 0 if not found.
 */
seepair(pag, n, key, siz)
char *pag;
register n;
register char *key;
register int siz;
{
	register i;
	register off;
	register short *ino = (short *) pag;

	off = PBLKSIZ;
	for (i = 1; i < n; i += 2) {
		if (siz == off - ino[i] &&
		    strncmp(key, pag + ino[i], siz) == 0)
			return (i);
		off = ino[i + 1];
	}
	return (0);
}
-- 
use the source, luke !!     	        Usenet:    oz@nexus.yorku.ca
uh... we forgot to tell you...          ......!uunet!utai!yunexus!oz
it is unintelligible, but hey, you      Bitnet: oz@[yulibra|yuyetti]
got it, for free (!).                   Phonet: +1 416 736-5257x3976

----------------------------------------------------------------------

Path: sequent!tektronix!reed!ogccse!blake!uw-beaver!mit-eddie!
      bloom-beacon!tut.cis.ohio-state.edu!ukma!uflorida!haven!mimsy!chris
From: chris@mimsy.UUCP (Chris Torek)
Newsgroups: comp.unix.wizards,comp.lang.c
Subject: Re: dbm/ndbm notes and some code.
Keywords: ndbm, sdbm, dbm, extendible-hashing.
Message-ID: <17472@mimsy.UUCP>
Date: 12 May 89 14:49:50 GMT
References: <1889@yunexus.UUCP> <6847@cbmvax.UUCP>
Organization: U of Maryland, Dept. of Computer Science, Coll. Pk., MD 20742
Lines: 36
Xref: sequent comp.unix.wizards:15233 comp.lang.c:17404
Posted: Fri May 12 07:49:50 1989

>In article <1889@yunexus.UUCP> oz@yunexus.UUCP (Ozan Yigit) writes:
>>This is where the "database traversal" turns into a pumpkin.  Because of
>>internal caching of the key position for dbm_nextkey (dbm_keyptr ??),
>>which is appearently NOT adjusted for deletions, this traversal will never
>>display the key right after the one just deleted. Workaround is to save
>>all keys to be deleted, then perform all deletions once the sequential
>>traversal is complete.

In article <6847@cbmvax.UUCP> grr@cbmvax.UUCP (George Robbins) writes:
>Say what?  Where are you going to save this potentially unbounded list of
>to be deleted keys?  Surely there is a better solution???

Indeed there is.  dbm_nextkey does not cache the key position.  Rather,
what is going on is that the dptr of the key returned by dbm_nextkey
points into a private region of memory (one holding a page from the .pag
file) which is modified by the dbm_delete operation.  This effectively
changes the text of the key (in a predictable manner, but which requires
knowing the current contents of the .pag page) so that the next
dbm_nextkey does not fetch the key following the one just deleted.

If all you are going to do is delete the just-found <key,datum>
pair, it suffices to save the contents of the key:

	newspace = malloc(key.dsize);
	if (newspace == NULL) ... error ...
	memcpy(newspace, key.dptr, key.dsize);
	key.dptr = newspace;
	dbm_delete(...);
	key = dbm_nextkey(key);
	free(newspace);

You must not, however, do a dbm_store, which could completely
discombobulate the current order of the database (by splitting).
-- 
In-Real-Life: Chris Torek, Univ of MD Comp Sci Dept (+1 301 454 7163)
Domain:	chris@mimsy.umd.edu	Path:	uunet!mimsy!chris

#endif	/* _THIS_IS_A_COMMENT_FOR_NDBM_	*/
