libthread: add pthreadperthread mode and use under ASAN

ASAN can't deal with the coroutine stacks.
In theory we can call into ASAN runtime to let it know about them,
but ASAN still has problems with fork or exit happening from a
non-system stack. Bypass all possible problems by just having
a full OS thread for each libthread thread. The threads are still
cooperatively scheduled within a proc (in thos mode, a group of OS threads).

Setting the environment variable LIBTHREAD=pthreadperthread
will enable the pthreadperthread mode, as will building with
CC9FLAGS='-fsanitize=address' in $PLAN9/config.

This solution is much more general than ASAN - for example if
you are trying to find all the thread stacks in a reproducible crash
you can use pthreadperthread mode with any debugger that
knows only about OS threads.

src/libthread/channel.c

@@ -141,7 +141,7 @@ altdequeue(Alt *a)
 			delarray(ar, i);
 			return;
 		}
-	fprint(2, "cannot find self in altdq\n");
+	fprint(2, "cannot find self in altdequeue\n");
 	abort();
 }
 

src/libthread/pthread.c

@@ -99,6 +99,23 @@ startprocfn(void *v)
 	pthread_exit(0);
 }
 
+static void
+startpthreadfn(void *v)
+{
+	void **a;
+	Proc *p;
+	_Thread *t;
+
+	a = (void**)v;
+	p = a[0];
+	t = a[1];
+	free(a);
+	t->osprocid = pthread_self();
+	pthread_detach(t->osprocid);
+	_threadpthreadmain(p, t);
+	pthread_exit(0);
+}
+
 void
 _procstart(Proc *p, void (*fn)(Proc*))
 {
@@ -116,6 +133,22 @@ _procstart(Proc *p, void (*fn)(Proc*))
 	}
 }
 
+void
+_threadpthreadstart(Proc *p, _Thread *t)
+{
+	void **a;
+
+	a = malloc(3*sizeof a[0]);
+	if(a == nil)
+		sysfatal("_pthreadstart malloc: %r");
+	a[0] = p;
+	a[1] = t;
+	if(pthread_create(&t->osprocid, nil, (void*(*)(void*))startpthreadfn, (void*)a) < 0){
+		fprint(2, "pthread_create: %r\n");
+		abort();
+	}
+}
+
 static pthread_key_t prockey;
 
 Proc*

src/libthread/thread.c

@@ -7,6 +7,7 @@ static	uint		threadnsysproc;
 static	Lock		threadnproclock;
 static	Ref		threadidref;
 static	Proc		*threadmainproc;
+static	int		pthreadperthread;
 
 static	void		addproc(Proc*);
 static	void		delproc(Proc*);
@@ -176,12 +177,16 @@ _threadcreate(Proc *p, void (*fn)(void*), void *arg, uint stack)
 	if(stack < (256<<10))
 		stack = 256<<10;
 
-	if(p->nthread == 0)
+	if(p->nthread == 0 || pthreadperthread)
 		stack = 0; // not using it
 	t = threadalloc(fn, arg, stack);
 	t->proc = p;
-	addthreadinproc(p, t);
+	if(p->nthread == 0)
+		p->thread0 = t;
+	else if(pthreadperthread)
+		_threadpthreadstart(p, t);
 	p->nthread++;
+	addthreadinproc(p, t);
 	_threadready(t);
 	return t;
 }
@@ -209,6 +214,29 @@ proccreate(void (*fn)(void*), void *arg, uint stack)
 	return id;
 }
 
+// For pthreadperthread mode, procswitch flips
+// between the threads.
+static void
+procswitch(Proc *p, _Thread *from, _Thread *to)
+{
+// fprint(2, "procswitch %p %d %d\n", p, from?from->id:-1, to?to->id:-1);
+	lock(&p->schedlock);
+	from->schedrend.l = &p->schedlock;
+	if(to) {
+		p->schedthread = to;
+		to->schedrend.l = &p->schedlock;
+		_procwakeup(&to->schedrend);
+	}
+	if(p->schedthread != from) {
+		if(from->exiting) {
+			unlock(&p->schedlock);
+			_threadpexit();
+		}
+		_procsleep(&from->schedrend);
+	}
+	unlock(&p->schedlock);
+}
+
 void
 _threadswitch(void)
 {
@@ -216,9 +244,13 @@ _threadswitch(void)
 
 	needstack(0);
 	p = proc();
+
 /*print("threadswtch %p\n", p); */
-	if(p->thread->stk == nil)
+
+	if(p->thread == p->thread0)
 		procscheduler(p);
+	else if(pthreadperthread)
+		procswitch(p, p->thread, p->thread0);
 	else
 		contextswitch(&p->thread->context, &p->schedcontext);
 }
@@ -346,6 +378,15 @@ procmain(Proc *p)
 		threadexits(nil);
 }
 
+void
+_threadpthreadmain(Proc *p, _Thread *t)
+{
+	_threadsetproc(p);
+	procswitch(p, t, nil);
+	t->startfn(t->startarg);
+	threadexits(nil);
+}
+
 static void
 procscheduler(Proc *p)
 {
@@ -401,9 +442,12 @@ Top:
 		p->nswitch++;
 		_threaddebug("run %d (%s)", t->id, t->name);
 //print("run %p %p %p %p\n", t, *(uintptr*)(t->context.uc.mc.sp), t->context.uc.mc.di, t->context.uc.mc.si);
-		if(t->stk == nil)
+		if(t == p->thread0)
 			return;
-		contextswitch(&p->schedcontext, &t->context);
+		if(pthreadperthread)
+			procswitch(p, p->thread0, t);
+		else
+			contextswitch(&p->schedcontext, &t->context);
 /*print("back in scheduler\n"); */
 		goto Top;
 	}
@@ -757,10 +801,24 @@ int
 main(int argc, char **argv)
 {
 	Proc *p;
+	char *opts;
 
 	argv0 = argv[0];
 
-	if(getenv("NOLIBTHREADDAEMONIZE") == nil)
+	opts = getenv("LIBTHREAD");
+	if(opts == nil)
+		opts = "";
+
+	pthreadperthread = (strstr(opts, "pthreadperthread") != nil);
+#ifdef PLAN9PORT_ASAN
+	// ASAN can't deal with the coroutine stack switches.
+	// In theory it has support for informing it about stack switches,
+	// but even with those calls added it can't deal with things
+	// like fork or exit from a coroutine stack.
+	// Easier to just run in pthread-per-thread mode.
+	pthreadperthread = 1;
+#endif
+	if(strstr(opts, "nodaemon") || getenv("NOLIBTHREADDAEMONIZE") == nil)
 		_threadsetupdaemonize();
 
 	threadargc = argc;

src/libthread/threadimpl.h

@@ -102,6 +102,17 @@ struct Execjob
 	Channel *c;
 };
 
+struct _Procrendez
+{
+	Lock		*l;
+	int		asleep;
+#ifdef PLAN9PORT_USING_PTHREADS
+	pthread_cond_t	cond;
+#else
+	int		pid;
+#endif
+};
+
 struct _Thread
 {
 	_Thread	*next;
@@ -112,6 +123,11 @@ struct _Thread
 	void	(*startfn)(void*);
 	void	*startarg;
 	uint	id;
+#ifdef PLAN9PORT_USING_PTHREADS
+	pthread_t	osprocid;
+#else
+	int		osprocid;
+#endif
 	uchar	*stk;
 	uint	stksize;
 	int		exiting;
@@ -120,17 +136,7 @@ struct _Thread
 	char	state[256];
 	void *udata;
 	Alt	*alt;
-};
+	_Procrendez schedrend;
-
-struct _Procrendez
-{
-	Lock		*l;
-	int		asleep;
-#ifdef PLAN9PORT_USING_PTHREADS
-	pthread_cond_t	cond;
-#else
-	int		pid;
-#endif
 };
 
 extern	void	_procsleep(_Procrendez*);
@@ -149,6 +155,7 @@ struct Proc
 #endif
 	Lock		lock;
 	int			nswitch;
+	_Thread		*thread0;
 	_Thread		*thread;
 	_Thread		*pinthread;
 	_Threadlist	runqueue;
@@ -157,6 +164,8 @@ struct Proc
 	uint		nthread;
 	uint		sysproc;
 	_Procrendez	runrend;
+	Lock		schedlock;
+	_Thread	*schedthread;
 	Context	schedcontext;
 	void		*udata;
 	Jmp		sigjmp;
@@ -188,6 +197,8 @@ extern void _threadpexit(void);
 extern void _threaddaemonize(void);
 extern void *_threadstkalloc(int);
 extern void _threadstkfree(void*, int);
+extern void _threadpthreadmain(Proc*, _Thread*);
+extern void _threadpthreadstart(Proc*, _Thread*);
 
 #define USPALIGN(ucp, align) \
 	(void*)((((uintptr)(ucp)->uc_stack.ss_sp+(ucp)->uc_stack.ss_size)-(align))&~((align)-1))