/* $Copyright:	$
 * Copyright (c) 1984, 1985, 1986, 1987, 1988, 1989, 1990 
 * Sequent Computer Systems, Inc.   All rights reserved.
 *  
 * This software is furnished under a license and may be used
 * only in accordance with the terms of that license and with the
 * inclusion of the above copyright notice.   This software may not
 * be provided or otherwise made available to, or used by, any
 * other person.  No title to or ownership of the software is
 * hereby transferred.
 */

#ident "@(#)$Header: team.c 1.1 89/04/17 $"

/* second generation microtasking--out of line prototype version */
/* A real implementation will generate all this stuff in-line */

#include <errno.h>
#include "team.h"
#include "microtask.h"
#include "parallel.h"
#include "parc.h"

#define DYNAMICTEAM
static private int *save_sp;
extern private int *_priv_sp;
shared struct teamblock *m_myteam[MAXPROCS];
private struct teamblock *m_teamblocks;
/* SPIN may be redefined to be nothing if compiled with -i */
#define SPIN(x) m_spin(x)
/*	 ceil(log2(n))	     0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 */
static int ceil_log2n[32] = {0, 0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4,
/*	16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 */
	 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5};
#ifdef DYNAMICTEAM
#ifdef DYNAMIC_INST
shared int dynamic_helps_[MAXPROCS];
#endif
#endif

static struct barrier_cell z = {0,0,0,0};
static struct barrier_cell one = {0,1,0,1};

#define CABLKSZ 32
/* create a new teamblock or use an existing one */
static struct teamblock *get_new_teamblock()
{
	register struct teamblock *team, *t;
	team = m_teamblocks;
	if (team) {
		team->teamsize = team->initial_teamsize = 1;
		m_teamblocks = team->free;
		return(team);
	}
	team = (struct teamblock *)
		((((int)(shmalloc(sizeof(struct teamblock)+CABLKSZ)))
		 + (CABLKSZ - 1)) & -CABLKSZ);
	/* link our previous team (of which we are master) to this block */
	for (t = m_myteam[m_myid]; t && t->pcells[m_myid].in_team_id;
		t = t->pcells[m_myid].prev);
	if (t) t->free = team;
	team->teamsize = team->initial_teamsize = 1;
	team->chunk_shift = ceil_log2n[m_numprocs + 1];
	team->bar_trip = 0;
	Q_INIT_LOCK(&team->triplock);
	Q_INIT_LOCK(&team->teamlock);
	Q_INIT_LOCK(&team->userlock);
	Q_INIT_LOCK(&team->userlock2);
	team->tripcounter = 0;
	team->free = 0;
	team->initial_tripcounter = 0;
	team->fork_address = 0;
	team->fork_trip = 128;
	team->cells[0] = one;
	team->cells[1] = one;
	team->cells[2] = z;
	team->cells[3] = z;
	team->cells[4] = z;
	team->cells[5] = z;
	team->cells[6] = z;
	team->pcells[m_myid].bar_cell_offset = 1;
	team->pcells[m_myid].in_team_id = 0;
	return(team);
}
/* activate DO loop */
int m_fork0(tc,p,fp)
register int tc;
int (*p)();
int fp;
{
	register struct teamblock *team;
	register int j, k;
	team = m_myteam[m_myid];
	if (!team) {
		team = get_new_teamblock();
		for (j = 0; j < 30; j++) m_myteam[j] = team;
#ifdef lint
		save_sp = 0;
#endif
#ifdef i386
		;asm(" movl	%esp, save_sp");
		;asm(" movl	_priv_sp, %esp");
		;asm(" call	_m_fork_init");
		;asm(" movl	save_sp, %esp");
#endif /* i386 */
	}
	if (m_myid || team->bar_trip == team->fork_trip) {
		/* nested loop */
		team = get_new_teamblock();
		team->pcells[m_myid].prev = m_myteam[m_myid];
		j = m_myteam[m_myid]->tripcounter;
		if (team->initial_tripcounter != tc)
			team->initial_tripcounter = tc;
		if (j > 0) j *= tc;
		else j = tc;
		j = (j / m_numprocs);
		if (j <= 0) j = 1;
		if (j > tc) j = tc;
		if (j != team->initial_chunk) team->initial_chunk = j;
		team->tripcounter = tc - j;
		k = fp;
		/* bet that the address hasn't changed */
		if (k != team->static_link) team->static_link = k;
		j = (int) p;
		/* bet that the address hasn't changed */
		if (j != (int) team->fork_address)
			team->fork_address = (int (*)())j;
		/* poke the fork cell block and make address valid */
		team->fork_trip = team->bar_trip;
		/* let other processes join team */
		m_myteam[m_myid] = team;
		/* join in loop */
		j = (*team->fork_address)(k);
		/* checkin and leave team (temporarily) */
		k = team->bar_trip;
		if (!atomic_addl_nonzero(
			&team->cells[team->pcells[m_myid].bar_cell_offset],
			0xfeffff00)) {
			if (!atomic_addl_nonzero(&team->cells[0], 0xfeffff00)) {
				team->bar_trip++;
				atomic_barrier_reset(&team->cells[0]);
			}
			atomic_barrier_reset(
			   &team->cells[team->pcells[m_myid].bar_cell_offset]);
		}
		asm("/PEEPOFF");
		while (k == team->bar_trip) {
			if (_m_scan_ne(m_myteam[m_myid], &m_myteam[0],
				m_numprocs))
				_m_choose_work();
		}
		asm("/PEEPON");
		/* join team again for next time */
		/* NOTE: we never decremented the team size */
		Q_LOCK(&team->teamlock);
		inc_bar_cell(&team->cells[1].N);
		if (team->cells[1].NP == 1) inc_bar_cell(&team->cells[0].N);
		Q_UNLOCK(&team->teamlock);
		m_teamblocks = team;
		m_myteam[m_myid] = team->pcells[m_myid].prev;
		return(j);
	} else {
		/* outermost parallel loop--the idle team */
		if (team->teamsize != team->initial_teamsize)
			team->initial_teamsize = team->teamsize;
		if (team->chunk_shift !=
			ceil_log2n[team->initial_teamsize])
			team->chunk_shift =
				ceil_log2n[team->initial_teamsize];
		if (team->initial_tripcounter != tc)
			team->initial_tripcounter = tc;
		j = (tc / m_numprocs);
		if (!j) j = 1;
		if (j != team->initial_chunk) team->initial_chunk = j;
		team->tripcounter = tc -
			team->initial_teamsize * team->initial_chunk;
		k = fp;
		/* bet that the address hasn't changed */
		if (k != team->static_link) team->static_link = k;
		j = (int) p;
		/* bet that the address hasn't changed */
		if (j != (int) team->fork_address)
			team->fork_address = (int(*)())j;
		/* poke the fork cell block and make address valid */
		team->fork_trip = team->bar_trip;
		/* join in loop */
		j = (*team->fork_address)(k);
		/* checkin */
		k = team->bar_trip;
		if (!atomic_decb_nonzero(
		    &team->cells[team->pcells[m_myid].bar_cell_offset].count)) {
			if (!atomic_decb_nonzero(&team->cells[0].count)) {
				team->bar_trip++;
				atomic_barrier_reset(&team->cells[0]);
			}
			atomic_barrier_reset(
			    &team->cells[team->pcells[m_myid].bar_cell_offset]);
		}
		asm("/PEEPOFF");
		while (k == team->bar_trip) {
			if (_m_scan_ne(m_myteam[m_myid], &m_myteam[0],
				m_numprocs))
				_m_choose_work();
		}
		asm("/PEEPON");
		return(j);
	}
}
/* activate DO loop for loop with ordered section */
int m_fork1(tc,p, fp)
register int tc;
int (*p)();
int fp;
{
	register struct teamblock *team;
	register int j, k;
	team = m_myteam[m_myid];
	if (!team) {
		team = get_new_teamblock();
		for (j = 0; j < 30; j++) m_myteam[j] = team;
#ifdef lint
			save_sp = 0;
#endif /* lint */
#ifdef i386
		;asm(" movl	%esp, save_sp");
		;asm(" movl	_priv_sp, %esp");
		;asm(" call	_m_fork_init");
		;asm(" movl	save_sp, %esp");
#endif /* i386 */
	}
	if (m_myid || team->bar_trip == team->fork_trip) {
		/* nested loop */
		team = get_new_teamblock();
		team->pcells[m_myid].prev = m_myteam[m_myid];
		team->tripcounter = tc;
		team->initial_tripcounter = tc;
		k = fp;
		/* bet that the address hasn't changed */
		if (k != team->static_link) team->static_link = k;
		j = (int) p;
		/* bet that the address hasn't changed */
		if (j != (int) team->fork_address)
			team->fork_address = (int (*)())j;
		/* poke the fork cell block and make address valid */
		team->fork_trip = team->bar_trip;
		/* let other processes join team */
		m_myteam[m_myid] = team;
		/* join in loop */
		j = (*team->fork_address)(k);
		/* checkin and leave team (temporarily) */
		k = team->bar_trip;
		if (!atomic_addl_nonzero(
			&team->cells[team->pcells[m_myid].bar_cell_offset],
			0xfeffff00)) {
			if (!atomic_addl_nonzero(&team->cells[0], 0xfeffff00)) {
				team->bar_trip++;
				atomic_barrier_reset(&team->cells[0]);
			}
			atomic_barrier_reset(
			   &team->cells[team->pcells[m_myid].bar_cell_offset]);
		}
		asm("/PEEPOFF");
		while (k == team->bar_trip) {
			if (_m_scan_ne(m_myteam[m_myid], &m_myteam[0],
				m_numprocs))
				_m_choose_work();
		}
		asm("/PEEPON");
		/* join team again for next time */
		/* NOTE: we never decremented the team size */
		Q_LOCK(&team->teamlock);
		inc_bar_cell(&team->cells[1].N);
		if (team->cells[1].NP == 1) inc_bar_cell(&team->cells[0].N);
		Q_UNLOCK(&team->teamlock);
		m_teamblocks = team;
		m_myteam[m_myid] = team->pcells[m_myid].prev;
		return(j);
	} else {
		/* outermost parallel loop--the idle team */
		team->tripcounter = tc;
		team->initial_tripcounter = tc;
		k = fp;
		/* bet that the address hasn't changed */
		if (k != team->static_link) team->static_link = k;
		j = (int) p;
		/* bet that the address hasn't changed */
		if (j != (int) team->fork_address)
			team->fork_address = (int(*)())j;
		/* poke the fork cell block and make address valid */
		team->fork_trip = team->bar_trip;
		/* join in loop */
		j = (*team->fork_address)(k);
		/* checkin */
		k = team->bar_trip;
		if (!atomic_decb_nonzero(
		    &team->cells[team->pcells[m_myid].bar_cell_offset].count)) {
			if (!atomic_decb_nonzero(&team->cells[0].count)) {
				team->bar_trip++;
				atomic_barrier_reset(&team->cells[0]);
			}
			atomic_barrier_reset(
			    &team->cells[team->pcells[m_myid].bar_cell_offset]);
		}
		asm("/PEEPOFF");
		while (k == team->bar_trip) {
			if (_m_scan_ne(m_myteam[m_myid], &m_myteam[0],
				m_numprocs))
				_m_choose_work();
		}
		asm("/PEEPON");
		return(j);
	}
}
/* add one process to barrier cell (while the loop is running) */
/* CAUTION: c really points to 4 adjacent 8-bit fields arranged so that
	we can do these things using the 386 hardware support */
static inc_bar_cell(c)
register int *c;
{
	register int i;
	i = atomic_exchange(c, 0x40002001);
	if (i > (int) 0xffffff) {
		i &= 0xff00ff00;
		i += 0xc0ffe0ff;
		atomic_addl(c, i);
	} else if (i == 0) {
		*c = 0x01000100;
	} else {
		while (*c < ((int) 0x40ffffff)) SPIN(c);
		if (*c & 0x10000) {
			i &= 0xff00ff00;
			i += (i << 16) + 0xa0fee0ff;
			atomic_addl(c, i);
		} else {
			i &= 0xff00ff00;
			i += 0xc0ffe0ff;
			atomic_addl(c, i);
	}	}
}
static char split_table[MAXPROCS] = {
	1,1,1,1,
	2,2,2,2,
	3,3,3,3,3,
	4,4,4,4,4,
	5,5,5,5,5,5,
	6,6,6,6,6,6
};
#define MAXSAVEDREGS 20
/* worker process main routine */
m_worker()
{
	register struct teamblock *team;
	register int j, k;
	extern int _stack_limit;
	/* switch to shared stack permanently */
	k = my_frame_pointer();
	j = _m_find_shstk(m_myid);
	load_stack_pointer(j);
	/* joint permanent, outermost team, lead by process 0 */
	for (team = m_myteam[0];team->pcells[0].prev;)
		team = team->pcells[0].prev;
	team->pcells[m_myid].prev = m_myteam[m_myid];
	m_myteam[m_myid] = team;
	Q_LOCK(&team->teamlock);
	team->pcells[m_myid].in_team_id = j = team->teamsize;
	j = split_table[j];
	team->pcells[m_myid].bar_cell_offset = j;
	inc_bar_cell(&team->cells[j].N);
	if (team->cells[j].NP == 1) inc_bar_cell(&team->cells[0].N);
	atomic_incl(&team->teamsize);
	Q_UNLOCK(&team->teamlock);
	/* wait for a fork */
	/* we use our local copy "j" instead of reading the bar_trip each
		iteration. This saves one read per processor which chokes
		the bus as the last processor checks into the barrier.
		This allows process 0 to get on the bus and proceed without
		interference. This improvement alone has produced a 30%
		improvement on an empty loop, back-to-back fork test for
		16 Symmetry B processors. Modification for nested parallelism:
		only look at the bar_trip cell occasionally. In between, look
		for other work. */
	for (j = team->bar_trip;;) {
		asm("/PEEPOFF");
		while (j != team->fork_trip) {
			if (_m_scan_ne(m_myteam[m_myid], &m_myteam[0],
				m_numprocs))
				_m_choose_work();
		}
		asm("/PEEPON");
		(*team->fork_address)(team->static_link);
		/* checkin and stay */
		j = team->bar_trip + 1;
		if (!atomic_decb_nonzero(
		    &team->cells[team->pcells[m_myid].bar_cell_offset].count)) {
			if (!atomic_decb_nonzero(&team->cells[0].count)) {
				team->bar_trip++;
				atomic_barrier_reset(&team->cells[0]);
			}
			atomic_barrier_reset(
			   &team->cells[team->pcells[m_myid].bar_cell_offset]);
		}
	}
}
/* select loop to join, if any available */
_m_choose_work()
{
#ifdef DYNAMICTEAM
	register struct teamblock *t, *best, *mine, *tm;
	int bestscore, score, i;
	best = 0;
	bestscore = 0;
	mine = m_myteam[m_myid];
	for (i = 0; i < m_numprocs; i++) {
	    t = m_myteam[i];
	    /* don't even look at HOT tripcounter cache block on my team */
	    if (t && t != mine && (score = t->tripcounter) > 0) {
	        for (tm = mine; tm; tm = tm->pcells[m_myid].prev)
			if (tm == t) break;
		if (!tm) {
			score >>= t->chunk_shift;
			if (!best || score > bestscore) {
				best = t;
				bestscore = score;
	}   }	}	}
	if (best) {
		t = best;
		t->pcells[m_myid].prev = m_myteam[m_myid];
		m_myteam[m_myid] = t = best;
		Q_LOCK(&t->teamlock);
		/* it's not critical which value we get */
		/* so teamsize is not fully protected by a lock */
		/* see atomic decl below */
		i = t->teamsize;
		t->pcells[m_myid].in_team_id = i;
		i = split_table[i];
		t->pcells[m_myid].bar_cell_offset = i;
		inc_bar_cell(&t->cells[i].N);
		if (t->cells[i].NP == 1) inc_bar_cell(&t->cells[0].N);
		atomic_incl(&t->teamsize);
		Q_UNLOCK(&t->teamlock);
		/* don't hang around if the loop finished */
		if (t->fork_trip == t->bar_trip) {
			(*t->fork_address)(t->static_link);
		}
		/* checkin and leave team */
		if (!atomic_addl_nonzero(
			&t->cells[t->pcells[m_myid].bar_cell_offset],
			0xfeffff00)) {
			if (!atomic_addl_nonzero(&t->cells[0], 0xfeffff00)) {
				t->bar_trip++;
				atomic_barrier_reset(&t->cells[0]);
			}
			atomic_barrier_reset(
			   &t->cells[t->pcells[m_myid].bar_cell_offset]);
		}
		atomic_decb(&t->teamsize);
		m_myteam[m_myid] = t->pcells[m_myid].prev;
#ifdef DYNAMIC_INST
		dynamic_helps_[m_myid]++;
#endif /* DYNAMIC_INST */
	}
#endif /* DYNAMICTEAM */
}
/* get *i = iteration number (from 0)*/
/* return number of iterations to do */
int m_get_iterations(i)
int *i;
{
	register struct teamblock *t;
	register int chunk, j;
	t = m_myteam[m_myid];
	chunk = t->tripcounter;
	if (chunk <= 0) return(0);
	chunk = (chunk >> t->chunk_shift) | 1;
	Q_LOCK(&t->triplock);
	j = t->tripcounter;
	t->tripcounter -= chunk;
	Q_UNLOCK(&t->triplock);
	if (j <= 0) return(0);
	*i = t->initial_tripcounter - j;
	return(chunk < j ? chunk : j);
}
/* get one iteration for loops with ordered sections */
int m_get_1_iteration(i)
int *i;
{
	register struct teamblock *t;
	register int j;
	t = m_myteam[m_myid];
	Q_LOCK(&t->triplock);
	j = t->tripcounter;
	t->tripcounter--;
	Q_UNLOCK(&t->triplock);
	if (j <= 0) return(0);
	*i = t->initial_tripcounter - j;
	return(1);
}
/* initial get number of iterations */
int m_get_initial_iterations(i)
int *i;
{
	register struct teamblock *t;
	register int chunk, j;
	t = m_myteam[m_myid];
	chunk = t->initial_chunk;
	j = t->pcells[m_myid].in_team_id;
	if (j < t->initial_teamsize) {
		*i = j = j * chunk;
		j = t->initial_tripcounter - j;
		if (chunk > j) chunk = j;
		return(chunk);
	} else {
		return(m_get_iterations(i));
	}
}
/* shut down tasks */
m_shutdown()
{
	extern m_exit();
	struct teamblock *t;
	int k, i;
	t = m_myteam[m_myid];
	if (!t) return;
	/* wait for all processes to join outer team */
	asm("/PEEPOFF");
	while (t->teamsize < m_numprocs || Q_IS_LOCKED(&t->teamlock));
	asm("/PEEPON");
	/* let everyone else exit while we return */
	t->fork_address = m_exit;
	t->fork_trip = t->bar_trip;
	/* let everyone else exit while we return */
	/* clear team structure so we will re-fork next m_fork */
	t->free = m_teamblocks;
	m_teamblocks = t;
	m_myteam[0] = 0;
	t->teamsize = 1;
	t->bar_trip++;
	t->cells[0] = one;
	t->cells[1] = one;
	t->cells[2] = z;
	t->cells[3] = z;
	t->cells[4] = z;
	t->cells[5] = z;
	t->cells[6] = z;
}
/* checkin, leave team, and exit */
static m_exit()
{
	m_myteam[m_myid] = 0;
	exit(0);
}
