/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */
/* 
 * This (is a placeholder for a) test that creates 4 threads, each of which 
 * does a concurrent spawn of 4 more processes, for a total of 17 MPI processes
 * The resulting intercomms are tested for consistency (to ensure that the
 * spawns didn't get confused among the threads).
 * 
 * As an option, it will time the Spawn calls.  If the spawn calls block the
 * calling thread, this may show up in the timing of the calls
 */

#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>
#include "mpitest.h"
#include "mpithreadtest.h"

/*
static char MTEST_Descrip[] = "Spawn jobs from multiple threads";
*/

#define NTHREADS 4
#define NPERTHREAD 4
int activeThreads = 0;

MPI_Comm intercomms[NTHREADS];

MTEST_THREAD_RETURN_TYPE spawnProcess(void *p);

MTEST_THREAD_RETURN_TYPE spawnProcess(void *p)
{
    int rank, i;
    int errcodes[NPERTHREAD];

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);

    /* The thread number is passed into this routine through the value of the
       argument */
    i = (int)(long)p;

    /* Synchronize */
    MTest_thread_barrier(NTHREADS);

    /* Spawn */
    MPI_Comm_spawn( (char*)"./multispawn", MPI_ARGV_NULL, NPERTHREAD, 
		    MPI_INFO_NULL, 0, MPI_COMM_SELF, &intercomms[i], errcodes );

    MPI_Bcast( &i, 1, MPI_INT, MPI_ROOT, intercomms[i] );

    return MTEST_THREAD_RETVAL_IGN;
}

int main( int argc, char *argv[] )
{
    int rank, size, i, wasParent = 0;
    int provided;
    int err;
    MPI_Comm   parentcomm;

    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    if (provided != MPI_THREAD_MULTIPLE)
    {
	if (rank == 0)
	{
	    printf("MPI_Init_thread must return MPI_THREAD_MULTIPLE in order for this test to run.\n");
	    fflush(stdout);
	}
	MPI_Finalize();
	return -1;
    }

    MPI_Comm_get_parent( &parentcomm );
    if (parentcomm == MPI_COMM_NULL) {
	wasParent = 1;

        err = MTest_thread_barrier_init();
        if (err) {
            printf("barrier_init failed\n");
            fflush(stdout);
            MPI_Finalize();
            return 1;
        }

	for (i=0; i<NTHREADS-1; i++) {
            MTest_Start_thread(spawnProcess, (void *)(long)i);
	}

	/* spawn the processes */
	spawnProcess( (void*)(NTHREADS-1) );

	/* Exit the threads (but the spawned processes remain) */
	MTest_Join_threads();

        err = MTest_thread_barrier_free();
        if (err) {
            printf("barrier_free failed\n");
            fflush(stdout);
            MPI_Finalize();
            return 1;
        }

	/* The master thread (this thread) checks the created communicators */
	for (i=0; i<NTHREADS; i++) {
	    MPI_Bcast( &i, 1, MPI_INT, MPI_ROOT, intercomms[i] );
	}

	/* Free the created processes */
	for (i=0; i<NTHREADS; i++) {
	    MPI_Comm_disconnect( &intercomms[i] );
	}
    }
    else {
	int num, threadnum;

	/* I'm the created process */
	MPI_Bcast( &threadnum, 1, MPI_INT, 0, parentcomm );
	
	/* Form an intra comm with my parent */

	/* receive from my parent */
	MPI_Bcast( &num, 1, MPI_INT, 0, parentcomm );

	if (num != threadnum) {
	    fprintf( stderr, "Unexpected thread num (%d != %d)\n", 
		     threadnum, num );
	}

	/* Let the parent free the intercomms */
	MPI_Comm_disconnect( &parentcomm );
	
    }

    if (wasParent) 
	MTest_Finalize(0);

    MPI_Finalize();
    return 0;
}
