Test_Parallel.cpp

Test of parallel netcdf4 I/O.

//-----------------------------------------------------------------------------
//      Code from netcdf test program
//        tst_parallel.c,v 1.7 2009/08/19
//-----------------------------------------------------------------------------

#include <iostream>
using std::cout;
using std::endl;
#include "netcdf4"
using namespace::netcdf;
#include "TimerMPI.h"
#include "AssertEx.h"

//-----------------------------------------------------------------------------
char const * const fileName = "Test_Parallel.nc";

// define a 3-D data block
short const NDIMS = 3;

// slabs sizes to write to data block
short const numSlabEdgeDims = 6;
short const slabEdgeSize[numSlabEdgeDims] = { 16, 24, 32, 48, 64, 80 };

//-----------------------------------------------------------------------------
void Write (
    netcdf::NcVar & var,
    std::vector<int> const & slab_data,
    int const numProcesses,
    int const rank,
    int const numSlabs,
    int const slabEdgeSize,
    bool const displayDetails = false)
{
    // Set up slab to write for this process
    std::vector<size_t> start (NDIMS);
    start[0] = rank / numProcesses * slabEdgeSize;      // per-process start
    start[1] = 0;

    std::vector<size_t> count (NDIMS);
    count[0] = slabEdgeSize / numProcesses;             // per-process edge size
    count[1] = slabEdgeSize;
    count[2] = 1;

    for (start[2] = 0; (int)start[2] < numSlabs; ++start[2])
    {
        if ( displayDetails )
        {
            cout << "rank " << rank
                 << ": start = " << start[0] << ", " << start[1] << ", " << start[2]
                 << endl;
            cout << "rank " << rank
                 << ": count = " << count[0] << ", " << count[1] << ", " << count[2]
                 << endl;
        }
        var.Put( start, count, slab_data );
    }
}

//-----------------------------------------------------------------------------
void WriteTableHeader ()
{
    char const TAB = '\t';      // assume 8 char tabs
    // line 1
    cout << '\n'
         << TAB << "slab"
         << TAB << "slab"
         << TAB << "slab"
         << TAB << "total"
         << TAB  << TAB << "write"
         << endl;
    // line 2
    cout << "rank"
         << TAB << "edge"
         << TAB << "size"
         << TAB << "count"
         << TAB << "data values"
         << TAB << "seconds"
         << endl;
}

void WriteTableRecord (
    int const rank,
    int const slabEdgeSize,
    int const slabSize,
    int const numSlabs,
    int const numValues)
{
    char const TAB = '\t';      // assume 8 char tabs
    cout << rank
         << TAB << slabEdgeSize
         << TAB << slabSize
         << TAB << numSlabs
         << TAB << numValues;
    if ( numValues > 9999999 )
        cout << TAB;
    else
        cout << TAB << TAB;
}

//-----------------------------------------------------------------------------
int main ()
{
    int rank = -1;
    try
    {
        MPI::Init();
        rank = MPI::COMM_WORLD.Get_rank();
        int const numProcesses = MPI::COMM_WORLD.Get_size();
        if ( rank == 0 )
        {
            cout << "\nParallel netCDF4 I/O test." << endl;
            cout << "number of processes: " << numProcesses << endl;
            cout << "File: " << ::fileName << endl;
        }
        // all processes
        {
            char name[MPI_MAX_PROCESSOR_NAME];
            int nameLength = 0;
            MPI::Get_processor_name( name, nameLength);
            cout << "rank " << rank << ": processor: " << name << endl;
        }

        // init MPI
        MPI::Comm & mpiComm = MPI::COMM_WORLD;
        mpiComm.Set_errhandler( MPI::ERRORS_THROW_EXCEPTIONS );
        MPI::Info mpiInfo = MPI::INFO_NULL;

        if ( rank == 0 )
            WriteTableHeader();
        double cumSeconds = 0;
        for ( short n = 0; n < numSlabEdgeDims; ++n )
        {
            int const slabSize = slabEdgeSize[n] * slabEdgeSize[n];
            int const numSlabs = slabSize / 4;
            int const numValues = slabSize * numSlabs;
            WriteTableRecord( rank, slabEdgeSize[n], slabSize, numSlabs, numValues );

            // create the netcdf file
            netcdf::NcFile ncFile ( mpiComm, mpiInfo, fileName, netcdf::NcFile::Replace );
            Assert( ncFile.IsParallel() );
            std::vector<netcdf::NcDim> dimVec;
            dimVec.push_back( ncFile.AddDim( "DIMSIZE1", slabEdgeSize[n] ) );
            dimVec.push_back( ncFile.AddDim( "DIMSIZE2", slabEdgeSize[n] ) );
            dimVec.push_back( ncFile.AddDim( "NUM_SLABS", numSlabs ) );
            netcdf::NcVar var = ncFile.AddVar( "v1", netcdf::ncInt, dimVec );
            var.DoCollectiveIO (false);

            // make and write data array
            std::vector<int> slab_data( slabSize, rank );       /* one slab */
            nrel::time::TimerMPI timer;
            timer.start();
            Write ( var, slab_data, numProcesses, rank, numSlabs, slabEdgeSize[n] );
            cumSeconds += timer.stop();
            cout << timer.read() << endl;       // complete record

            ncFile.Close();
        }
        cout << "\ncumulative seconds in Write = " << cumSeconds << endl;

        MPI::Finalize();
    }
    catch (MPI::Exception & e)
    {
        cout << "MPI error: " << e.Get_error_string() << endl;
        MPI::COMM_WORLD.Abort (-1);
    }
    catch (std::exception const & e)
    {
        cout << "Exception: " << e.what() << endl;
    }
    catch (...)
    {
        cout << "Error: unknown error." << endl;
    }
    if (rank == 0)
        cout << "\n   all done!" << endl;
    return 0;
}
//-----------------------------------------------------------------------------