diff --git a/common/include/villas/memory.hpp b/common/include/villas/memory.hpp
index b4c3497a0..7ce425e70 100644
--- a/common/include/villas/memory.hpp
+++ b/common/include/villas/memory.hpp
@@ -76,12 +76,6 @@ public:
 	    memoryBlock(std::move(mem))
 	{ }
 
-	// Take ownership of the MemoryBlock
-	MemoryAccessor(std::shared_ptr<MemoryBlock> mem) :
-	    translation(MemoryManager::get().getTranslationFromProcess(mem->getAddrSpaceId())),
-	    memoryBlock(std::move(mem))
-	{ }
-
 	// Just act as an accessor, do not take ownership of MemoryBlock
 	MemoryAccessor(const MemoryBlock &mem) :
 	    translation(MemoryManager::get().getTranslationFromProcess(mem.getAddrSpaceId()))
@@ -112,10 +106,10 @@ public:
 		return reinterpret_cast<T*>(translation.getLocalAddr(0));
 	}
 
-	const std::shared_ptr<MemoryBlock>
+	const MemoryBlock&
 	getMemoryBlock() const
 	{
-		if (not memoryBlock) throw std::bad_alloc(); else return memoryBlock;
+		if (not memoryBlock) throw std::bad_alloc(); else return *memoryBlock;
 	}
 
 private:
@@ -123,7 +117,7 @@ private:
 	MemoryTranslation translation;
 
 	// Take the unique pointer in case user wants this class to have ownership
-	std::shared_ptr<MemoryBlock> memoryBlock;
+	std::unique_ptr<MemoryBlock, MemoryBlock::deallocator_fn> memoryBlock;
 };
 
 /**