domdec.c File Reference

contains domain decomposition functions More...

#include <stdio.h>
#include <stdlib.h>
#include "global.h"

Include dependency graph for domdec.c:

Go to the source code of this file.

Functions
int	ztnReturnDimension (void *data, int *ierr)
void	ztnReturnCoords (void *data, int numGidEntries, int numLidEntries, ZOLTAN_ID_PTR globalId, ZOLTAN_ID_PTR localId, double *geomVec, int *ierr)
int	ztnReturnNumNode (void *data, int *ierr)
void	ztnReturnOwnedNodes (void *data, int numGIdEntries, int numLIdEntries, ZOLTAN_ID_PTR globalIds, ZOLTAN_ID_PTR localIds, int wgtDim, float *objWgts, int *ierr)
int	ztnReturnParticleDataSize (void *data, int numGIdEntries, int numLIdEntries, ZOLTAN_ID_PTR globalId, ZOLTAN_ID_PTR localId, int *ierr)
void	ztnPack (void *data, int numGIdEntries, int numLIdEntries, ZOLTAN_ID_PTR globalId, ZOLTAN_ID_PTR localId, int dest, int size, char *buf, int *ierr)
void	ztnPre (void *data, int numGIdEntries, int numLIdEntries, int numImport, ZOLTAN_ID_PTR importGlobalIds, ZOLTAN_ID_PTR importLocalIds, int *importProcs, int *importToPart, int numExport, ZOLTAN_ID_PTR exportGlobalIds, ZOLTAN_ID_PTR exportLocalIds, int *exportProcs, int *exportToPart, int *ierr)
void	ztnMid (void *data, int numGIdEntries, int numLIdEntries, int numImport, ZOLTAN_ID_PTR importGlobalIds, ZOLTAN_ID_PTR importLocalIds, int *importProcs, int *importToPart, int numExport, ZOLTAN_ID_PTR exportGlobalIds, ZOLTAN_ID_PTR exportLocalIds, int *exportProcs, int *exportToPart, int *ierr)
void	ztnPost (void *data, int numGIdEntries, int numLIdEntries, int numImport, ZOLTAN_ID_PTR importGlobalIds, ZOLTAN_ID_PTR importLocalIds, int *importProcs, int *importToPart, int numExport, ZOLTAN_ID_PTR exportGlobalIds, ZOLTAN_ID_PTR exportLocalIds, int *exportProcs, int *exportToPart, int *ierr)
void	ztnUnpack (void *data, int numGIdEntries, ZOLTAN_ID_PTR globalId, int size, char *buf, int *ierr)
void	decompositionInit (int argc, char **argv, MPI_Comm Comm)
void	decompositionExecute ()
void	decompositionFinalize ()

Variables
int	changes
int	numGidEntries
int	numLidEntries
int	numImport
ZOLTAN_ID_PTR	importGlobalGids
ZOLTAN_ID_PTR	importLocalGids
int *	importProcs
int *	importToPart
int	numExport
ZOLTAN_ID_PTR	exportGlobalGids
ZOLTAN_ID_PTR	exportLocalGids
int *	exportProcs
int *	exportToPart

Detailed Description

contains domain decomposition functions

Definition in file domdec.c.

Function Documentation

void decompositionExecute

(

)

This function calls the Zoltan's domain decomposition and migration functions. It is called at the beginning of each simulation step.

Definition at line 240 of file domdec.c.

References changes, exportGlobalGids, exportLocalGids, exportProcs, exportToPart, importGlobalGids, importLocalGids, importProcs, importToPart, MPIsize, nc, numExport, numGidEntries, numImport, numLidEntries, stopRun(), and ztn.

{
  int rc;
  int i;

  if (nc < MPIsize)
    return;

  rc = Zoltan_LB_Partition(ztn, /* input (all remaining fields are output) */
               &changes,    /* 1 if partitioning was changed, 0 otherwise */
               &numGidEntries,  /* number of integers used for a global ID */
               &numLidEntries,  /* number of integers used for a local ID */
               &numImport,  /* number of objects to be sent to me */
               &importGlobalGids,   /* global IDs of objects to be sent to me */
               &importLocalGids,    /* local IDs of objects to be sent to me */
               &importProcs,    /* process rank for source of each incoming object */
               &importToPart,   /* new partition for each incoming object */
               &numExport,  /* number of objects I must send to other processes */
               &exportGlobalGids,   /* global IDs of the objects I must send */
               &exportLocalGids,    /* local IDs of the objects I must send */
               &exportProcs,    /* process to which I send each of the objects */
               &exportToPart);  /* partition to which each object will belong */

  if (rc != ZOLTAN_OK)
    stopRun(112, NULL, __FILE__, __LINE__);

  /* free the arrays allocated by Zoltan_LB_Partiotion */
  Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs,
              &importToPart);
  Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs,
              &exportToPart);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void decompositionFinalize

(

)

This function deactivates the Zoltan library. It is called at the end of the simulation.

Definition at line 277 of file domdec.c.

References ztn.

{
  Zoltan_Destroy(&ztn);
}

Here is the caller graph for this function:

void decompositionInit	(	int	argc,
		char **	argv,
		MPI_Comm	Comm
	)

This function initializes the Zoltan library. It is called at the beginning of the simulation.

Definition at line 191 of file domdec.c.

References cells, MPIrank, stopRun(), ztn, ztnMid(), ztnPack(), ztnPost(), ztnPre(), ztnReturnCoords(), ztnReturnDimension(), ztnReturnNumNode(), ztnReturnOwnedNodes(), ztnReturnParticleDataSize(), and ztnUnpack().

{
  int rc;
  float version;

  rc = Zoltan_Initialize(argc, argv, &version);
  if (rc != ZOLTAN_OK)
    stopRun(112, NULL, __FILE__, __LINE__);

  if (MPIrank == 0)
    printf("Zoltan Version %.3f. Initialized.\n", version);

  ztn = Zoltan_Create(MPI_COMM_WORLD);

  Zoltan_Set_Param(ztn, "IMBALANCE_TOL", "2.0");
  Zoltan_Set_Param(ztn, "LB_METHOD", "HSFC");   /* Hilbert Space-Filling Curve Partitioning */
  Zoltan_Set_Param(ztn, "NUM_GID_ENTRIES", "1");    /* global ID is 1 integer */
  Zoltan_Set_Param(ztn, "NUM_LID_ENTRIES", "1");    /* local ID is 1 integer */
  Zoltan_Set_Param(ztn, "OBJ_WEIGHT_DIM", "1"); /* we use object weights */
  Zoltan_Set_Param(ztn, "DEBUG_LEVEL", "0");    /* quiet mode; no output unless an error or warning is produced */
  Zoltan_Set_Param(ztn, "KEEP_CUTS", "1");  /* save the cuts for later use */
  Zoltan_Set_Param(ztn, "AUTO_MIGRATE", "1");   /* use the auto migration mechanism */

  Zoltan_Set_Fn(ztn, ZOLTAN_NUM_GEOM_FN_TYPE,
        (void (*)()) ztnReturnDimension, cells);
  Zoltan_Set_Fn(ztn, ZOLTAN_GEOM_FN_TYPE, (void (*)()) ztnReturnCoords,
        cells);
  Zoltan_Set_Fn(ztn, ZOLTAN_NUM_OBJ_FN_TYPE, (void (*)()) ztnReturnNumNode,
        cells);
  Zoltan_Set_Fn(ztn, ZOLTAN_OBJ_LIST_FN_TYPE,
        (void (*)()) ztnReturnOwnedNodes, cells);
  Zoltan_Set_Fn(ztn, ZOLTAN_OBJ_SIZE_FN_TYPE,
        (void (*)()) ztnReturnParticleDataSize, cells);
  Zoltan_Set_Fn(ztn, ZOLTAN_PACK_OBJ_FN_TYPE, (void (*)()) ztnPack, cells);
  Zoltan_Set_Fn(ztn, ZOLTAN_UNPACK_OBJ_FN_TYPE, (void (*)()) ztnUnpack,
        cells);
  Zoltan_Set_Fn(ztn, ZOLTAN_PRE_MIGRATE_PP_FN_TYPE, (void (*)()) ztnPre,
        cells);
  Zoltan_Set_Fn(ztn, ZOLTAN_MID_MIGRATE_PP_FN_TYPE, (void (*)()) ztnMid,
        cells);
  Zoltan_Set_Fn(ztn, ZOLTAN_POST_MIGRATE_PP_FN_TYPE, (void (*)()) ztnPost,
        cells);

}

Here is the call graph for this function:

Here is the caller graph for this function:

void ztnMid	(	void *	data,
		int	numGIdEntries,
		int	numLIdEntries,
		int	numImport,
		ZOLTAN_ID_PTR	importGlobalIds,
		ZOLTAN_ID_PTR	importLocalIds,
		int *	importProcs,
		int *	importToPart,
		int	numExport,
		ZOLTAN_ID_PTR	exportGlobalIds,
		ZOLTAN_ID_PTR	exportLocalIds,
		int *	exportProcs,
		int *	exportToPart,
		int *	ierr
	)

Zoltan callback function. This function is executed after packing of send buffer and unpacking of receive buffer during migration.

Definition at line 140 of file domdec.c.

References cellData::gid, lnc, and numExport.

{
  int pos, i;
  struct cellData *c = (struct cellData *) data;
  pos = 0;
  for (i = 0; i < lnc; i++) {
    if (i != pos && c[i].gid != -1) {
      c[pos] = c[i];
    }
    if (c[i].gid != -1)
      pos++;
  }
  lnc = lnc - numExport;
}

Here is the caller graph for this function:

void ztnPack	(	void *	data,
		int	numGIdEntries,
		int	numLIdEntries,
		ZOLTAN_ID_PTR	globalId,
		ZOLTAN_ID_PTR	localId,
		int	dest,
		int	size,
		char *	buf,
		int *	ierr
	)

Zoltan callback function. This function packs data into a send buffer before migration.

Definition at line 115 of file domdec.c.

References cellData::gid.

{
  struct cellData *c = (struct cellData *) data;
  memcpy(buf, &(c[localId[0]]), sizeof(struct cellData));
  c[(int) (*localId)].gid = -1; /* mark local particle as exported */
}

Here is the caller graph for this function:

void ztnPost	(	void *	data,
		int	numGIdEntries,
		int	numLIdEntries,
		int	numImport,
		ZOLTAN_ID_PTR	importGlobalIds,
		ZOLTAN_ID_PTR	importLocalIds,
		int *	importProcs,
		int *	importToPart,
		int	numExport,
		ZOLTAN_ID_PTR	exportGlobalIds,
		ZOLTAN_ID_PTR	exportLocalIds,
		int *	exportProcs,
		int *	exportToPart,
		int *	ierr
	)

Zoltan callback function. This function is executed after migration of data between processes.

Definition at line 163 of file domdec.c.

References lnc, and tlnc.

{
  /* any post communication operations should go here */
  /* gather number of cells from each process */
  MPI_Allgather(&lnc, 1, MPI_LONG_LONG, tlnc, 1, MPI_LONG_LONG,
        MPI_COMM_WORLD);
}

Here is the caller graph for this function:

void ztnPre	(	void *	data,
		int	numGIdEntries,
		int	numLIdEntries,
		int	numImport,
		ZOLTAN_ID_PTR	importGlobalIds,
		ZOLTAN_ID_PTR	importLocalIds,
		int *	importProcs,
		int *	importToPart,
		int	numExport,
		ZOLTAN_ID_PTR	exportGlobalIds,
		ZOLTAN_ID_PTR	exportLocalIds,
		int *	exportProcs,
		int *	exportToPart,
		int *	ierr
	)

Zoltan callback function. This function is executed before migration of data between processes.

Definition at line 127 of file domdec.c.

{
  /* any pre communication operations should go here */
}

Here is the caller graph for this function:

void ztnReturnCoords	(	void *	data,
		int	numGidEntries,
		int	numLidEntries,
		ZOLTAN_ID_PTR	globalId,
		ZOLTAN_ID_PTR	localId,
		double *	geomVec,
		int *	ierr
	)

Zoltan callback function. This function returns the spatial coordinates of the cell identified by its global and local id.

Definition at line 61 of file domdec.c.

References cells, sdim, cellData::x, cellData::y, and cellData::z.

{
  if (sdim == 3) {
    geomVec[0] = cells[localId[0]].x;
    geomVec[1] = cells[localId[0]].y;
    geomVec[2] = cells[localId[0]].z;
  }
  if (sdim == 2) {
    geomVec[0] = cells[localId[0]].x;
    geomVec[1] = cells[localId[0]].y;
  }
}

Here is the caller graph for this function:

int ztnReturnDimension	(	void *	data,
		int *	ierr
	)

Zoltan callback function. This function returns the dimension (2D or 3D) of the system.

Definition at line 50 of file domdec.c.

References sdim.

{
  if (sdim == 3)
    return 3;
  if (sdim == 2)
    return 2;
}

Here is the caller graph for this function:

int ztnReturnNumNode	(	void *	data,
		int *	ierr
	)

Zoltan callback function. This function returns the number of cells assigned to this process.

Definition at line 79 of file domdec.c.

References lnc.

{
  return lnc;
}

Here is the caller graph for this function:

void ztnReturnOwnedNodes	(	void *	data,
		int	numGIdEntries,
		int	numLIdEntries,
		ZOLTAN_ID_PTR	globalIds,
		ZOLTAN_ID_PTR	localIds,
		int	wgtDim,
		float *	objWgts,
		int *	ierr
	)

Zoltan callback function. This function fills the tables of global ids, local ids and weights for all cells assigned to this process.

Definition at line 87 of file domdec.c.

References cells, cellData::gid, and lnc.

{
  int i;
  struct cell *p = (struct cell *) data;
  for (i = 0; i < lnc; i++) {
    globalIds[i * numGIdEntries] = cells[i].gid;
    localIds[i * numLIdEntries] = i;
    objWgts[i] = 1.0;
    //if(nc==1 || step==0) obj_wgts[i]=1.0;
    //else obj_wgts[i]=cells[i].density;
  }
}

Here is the caller graph for this function:

int ztnReturnParticleDataSize	(	void *	data,
		int	numGIdEntries,
		int	numLIdEntries,
		ZOLTAN_ID_PTR	globalId,
		ZOLTAN_ID_PTR	localId,
		int *	ierr
	)

Zoltan callback function. This function returns the size of a data structure used for keeping the data of a single cell.

Definition at line 105 of file domdec.c.

{
  return sizeof(struct cellData);
}

Here is the caller graph for this function:

void ztnUnpack	(	void *	data,
		int	numGIdEntries,
		ZOLTAN_ID_PTR	globalId,
		int	size,
		char *	buf,
		int *	ierr
	)

Zoltan callback function. This function unpacks data from the receive buffer.

Definition at line 179 of file domdec.c.

References lnc.

{
  struct cellData *c = (struct cellData *) data;
  memcpy(&c[lnc], buf, sizeof(struct cellData));
  lnc++;
}

Here is the caller graph for this function:

Variable Documentation

int changes

Definition at line 33 of file domdec.c.

ZOLTAN_ID_PTR exportGlobalGids

Definition at line 42 of file domdec.c.

ZOLTAN_ID_PTR exportLocalGids

Definition at line 43 of file domdec.c.

int* exportProcs

Definition at line 44 of file domdec.c.

int* exportToPart

Definition at line 45 of file domdec.c.

ZOLTAN_ID_PTR importGlobalGids

Definition at line 37 of file domdec.c.

ZOLTAN_ID_PTR importLocalGids

Definition at line 38 of file domdec.c.

int* importProcs

Definition at line 39 of file domdec.c.

int* importToPart

Definition at line 40 of file domdec.c.

int numExport

Definition at line 41 of file domdec.c.

int numGidEntries

Definition at line 34 of file domdec.c.

int numImport

Definition at line 36 of file domdec.c.

int numLidEntries

Definition at line 35 of file domdec.c.