Proto_Copier.H

#pragma once
#ifndef _PROTO_COPIER_
#define _PROTO_COPIER_

#include "Proto_DisjointBoxLayout.H"
#include "Proto.H"
#include "Proto_SPMD.H"
#include "Proto_LevelIterators.H"
#include <unordered_map>
#include <cstdint>
#include "Proto_MayDay.H"

namespace Proto

{
// =======================================================================
// DECLARATIONS

    struct BufferEntry;
    struct MotionItem;
    class CopierIterator;
   
// =======================================================================
// ABSTRACT COPIER CLASS


    template<class OP>
    class Copier
    {
        friend class MotionItem;
        friend class CopierIterator; 
        
        public:
        
        enum MotionType
        {
            LOCAL,
            FROM,
            TO
        };

        Copier();
        Copier(OP a_op);

        // Destructor is virtual to handle potential polymorphic destruct
        inline virtual ~Copier();
        // 
        inline void define(OP a_op); 
        inline void clear(); 
        virtual void buildMotionPlans(OP& a_op) = 0;

        inline Copier<OP>& operator=(const Copier<OP>& a_rhs);
        inline bool operator==(const Copier<OP>& a_rhs) const;
        inline void reverse();
        inline void execute();
        inline void sort();
        inline CopierIterator begin(MotionType a_type) const;
        protected:

        OP m_op;

        std::vector<MotionItem*> m_localMotionPlan;
        std::vector<MotionItem*> m_fromMotionPlan;
        std::vector<MotionItem*> m_toMotionPlan;

        bool m_isDefined = false;
        
        private: 
        
        void makeItSo();
        void makeItSoBegin();
        void makeItSoLocal();
        void makeItSoEnd();

        void completePendingSends() const;
        void allocateBuffers();
        void writeToSendBuffers();
        void postSends() const;
        void readFromRecvBuffers();
        void postRecvs() const;

        // Copy buffers        
        void clearBuffers();
        
        mutable size_t m_sendCapacity;
        mutable size_t m_recvCapacity;
        mutable void* m_sendBuffer;
        mutable void* m_recvBuffer;

        // Probably can't do stuff like this on the device...
        //mutable std::unique_ptr<char[]> m_sendBuffer;
        //mutable std::unique_ptr<char[]> m_recvBuffer;

        mutable std::vector<BufferEntry> m_fromMe;
        mutable std::vector<BufferEntry> m_toMe;
        mutable std::vector<std::vector<BufferEntry>> m_toMeUnpack;        
        
        mutable int m_numSends;
        mutable int m_numRecvs;

#ifdef PR_MPI
        mutable std::vector<MPI_Request> m_sendRequests, m_recvRequests;
        mutable std::vector<MPI_Status>  m_sendStatus,  m_recvStatus;
#endif
    }; // end class Copier
    
// =======================================================================
// STRUCTURES

    class BufferEntry
    {
        public:
        void* bufferPointer;
        size_t size;
        const MotionItem* item;
        unsigned int procID;

        inline bool operator<(const BufferEntry& a_rhs) const;
    }; // end struct BufferEntry
    
    class MotionItem
    {
        public:
        DataIndex fromIndex, toIndex;
        Box fromRegion;
        Box toRegion;
        int procID;

        inline MotionItem(const DataIndex& a_fromIndex,
                const DataIndex& a_toIndex,
                const Box& a_fromRegion,
                const Box& a_toRegion);

        inline bool operator==(const MotionItem& a_rhs) const;
        inline bool operator<(const MotionItem& a_rhs) const;
    }; // end struct MotionItem
  
    class MotionItemSorter
    {
        public:
        inline bool operator()(MotionItem* const & a_lhs,
                MotionItem* const& a_rhs) const
                {
                    return ((*a_lhs) < (*a_rhs));
                }
    }; // end class MotionItemSorter
    
// =======================================================================
// COPIER ITERATOR

    class CopierIterator
    {
        public:

            inline CopierIterator(const std::vector<MotionItem*>* a_motionPlan);
            inline const MotionItem& operator()() const;
            inline const MotionItem& operator[](size_t a_index) const;
            inline const MotionItem& operator*() const;
            inline void operator++();
            inline bool ok(); 
            inline void reset();
            inline size_t size() const;

        private:

            const vector<MotionItem*>* m_motionPlan;
            unsigned int m_current;
    }; // end class CopierIterator
    

#include "implem/Proto_CopierImplem.H"
} // end namespace Proto
#endif //end include guard