diff --git a/autotests/libs/itemstoretest.h b/autotests/libs/itemstoretest.h
--- a/autotests/libs/itemstoretest.h
+++ b/autotests/libs/itemstoretest.h
@@ -38,6 +38,7 @@
     void testRevisionCheck();
     void testModificationTime();
     void testRemoteIdRace();
+    void itemModifyJobShouldOnlySendModifiedAttributes();
 };
 
 #endif
diff --git a/autotests/libs/itemstoretest.cpp b/autotests/libs/itemstoretest.cpp
--- a/autotests/libs/itemstoretest.cpp
+++ b/autotests/libs/itemstoretest.cpp
@@ -382,3 +382,41 @@
     QVERIFY(fetchJob->items().first().remoteId().isEmpty());
 }
 
+void ItemStoreTest::itemModifyJobShouldOnlySendModifiedAttributes()
+{
+    // Given an item with an attribute (created on the server)
+    Item item;
+    item.setMimeType(QStringLiteral("text/directory"));
+    item.attribute<TestAttribute>(Item::AddIfMissing)->data = "initial";
+    ItemCreateJob *job = new ItemCreateJob(item, res1_foo);
+    AKVERIFYEXEC(job);
+    item = job->item();
+    QCOMPARE(item.attributes().count(), 1);
+
+    // When one job modifies this attribute, and another one does an unrelated change
+    Item item1(item.id());
+    item1.attribute<TestAttribute>(Item::AddIfMissing)->data = "modified";
+    ItemModifyJob *mjob = new ItemModifyJob(item1);
+    mjob->disableRevisionCheck();
+    AKVERIFYEXEC(mjob);
+
+    item.setFlag("added_test_flag_1");
+    // this job shouldn't send the old attribute again
+    ItemModifyJob *mjob2 = new ItemModifyJob(item);
+    mjob2->disableRevisionCheck();
+    AKVERIFYEXEC(mjob2);
+
+    // Then the item has the new value for the attribute (the other one didn't send the old attribute value)
+    {
+        auto *fetchJob = new ItemFetchJob(Item(item.id()));
+        ItemFetchScope fetchScope;
+        fetchScope.fetchAllAttributes(true);
+        fetchJob->setFetchScope(fetchScope);
+        AKVERIFYEXEC(fetchJob);
+        QCOMPARE(fetchJob->items().size(), 1);
+        const Item fetchedItem = fetchJob->items().first();
+        QCOMPARE(fetchedItem.flags().count(), 1);
+        QCOMPARE(fetchedItem.attributes().count(), 1);
+        QCOMPARE(fetchedItem.attribute<TestAttribute>()->data, "modified");
+    }
+}
diff --git a/src/core/item.cpp b/src/core/item.cpp
--- a/src/core/item.cpp
+++ b/src/core/item.cpp
@@ -229,49 +229,32 @@
 
 void Item::addAttribute(Attribute *attr)
 {
-    Q_ASSERT(attr);
-    Attribute *existing = d_ptr->mAttributes.value(attr->type());
-    if (existing) {
-        if (attr == existing) {
-            return;
-        }
-        d_ptr->mAttributes.remove(attr->type());
-        delete existing;
-    }
-    d_ptr->mAttributes.insert(attr->type(), attr);
-    ItemChangeLog::instance()->deletedAttributes(d_ptr).remove(attr->type());
+    ItemChangeLog::instance()->attributeStorage(d_ptr).addAttribute(attr);
 }
 
 void Item::removeAttribute(const QByteArray &type)
 {
-    ItemChangeLog::instance()->deletedAttributes(d_ptr).insert(type);
-    delete d_ptr->mAttributes.take(type);
+    ItemChangeLog::instance()->attributeStorage(d_ptr).removeAttribute(type);
 }
 
 bool Item::hasAttribute(const QByteArray &type) const
 {
-    return d_ptr->mAttributes.contains(type);
+    return ItemChangeLog::instance()->attributeStorage(d_ptr).hasAttribute(type);
 }
 
 Attribute::List Item::attributes() const
 {
-    return d_ptr->mAttributes.values();
+    return ItemChangeLog::instance()->attributeStorage(d_ptr).attributes();
 }
 
 void Akonadi::Item::clearAttributes()
 {
-    ItemChangeLog *changelog = ItemChangeLog::instance();
-    QSet<QByteArray> &deletedAttributes = changelog->deletedAttributes(d_ptr);
-    for (Attribute *attr : qAsConst(d_ptr->mAttributes)) {
-        deletedAttributes.insert(attr->type());
-        delete attr;
-    }
-    d_ptr->mAttributes.clear();
+    ItemChangeLog::instance()->attributeStorage(d_ptr).clearAttributes();
 }
 
 Attribute *Item::attribute(const QByteArray &type) const
 {
-    return d_ptr->mAttributes.value(type);
+    return ItemChangeLog::instance()->attributeStorage(d_ptr).attribute(type);
 }
 
 Collection &Item::parentCollection()
@@ -747,22 +730,8 @@
     setParentCollection(other.parentCollection());
     setStorageCollectionId(other.storageCollectionId());
 
-    QList<QByteArray> attrs;
-    attrs.reserve(other.attributes().count());
-    const Akonadi::Attribute::List lstAttrs = other.attributes();
-    for (Attribute *attribute : lstAttrs) {
-        addAttribute(attribute->clone());
-        attrs.append(attribute->type());
-    }
-
-    QMutableHashIterator<QByteArray, Attribute *> it(d_ptr->mAttributes);
-    while (it.hasNext()) {
-        it.next();
-        if (!attrs.contains(it.key())) {
-            delete it.value();
-            it.remove();
-        }
-    }
+    ItemChangeLog *changelog = ItemChangeLog::instance();
+    changelog->attributeStorage(d_ptr) = changelog->attributeStorage(other.d_ptr);
 
     ItemSerializer::apply(*this, other);
     d_ptr->resetChangeLog();
diff --git a/src/core/item_p.h b/src/core/item_p.h
--- a/src/core/item_p.h
+++ b/src/core/item_p.h
@@ -308,9 +308,6 @@
         mRemoteId = other.mRemoteId;
         mRemoteRevision = other.mRemoteRevision;
         mPayloadPath = other.mPayloadPath;
-        Q_FOREACH (Attribute *attr, other.mAttributes) {
-            mAttributes.insert(attr->type(), attr->clone());
-        }
         if (other.mParent) {
             mParent = new Collection(*(other.mParent));
         }
@@ -338,15 +335,14 @@
         changelog->deletedFlags(this) = changelog->deletedFlags(&other);
         changelog->addedTags(this) = changelog->addedTags(&other);
         changelog->deletedTags(this) = changelog->deletedTags(&other);
-        changelog->deletedAttributes(this) = changelog->deletedAttributes(&other);
+        changelog->attributeStorage(this) = changelog->attributeStorage(&other);
     }
 
     ~ItemPrivate()
     {
-        qDeleteAll(mAttributes);
         delete mParent;
 
-        ItemChangeLog::instance()->clearItemChangelog(this);
+        ItemChangeLog::instance()->removeItem(this);
     }
 
     void resetChangeLog()
@@ -440,7 +436,6 @@
     QString mRemoteId;
     QString mRemoteRevision;
     mutable QString mPayloadPath;
-    QHash<QByteArray, Attribute *> mAttributes;
     mutable Collection *mParent;
     mutable _detail::clone_ptr<Internal::PayloadBase> mLegacyPayload;
     mutable PayloadContainer mPayloads;
diff --git a/src/core/itemchangelog.cpp b/src/core/itemchangelog.cpp
--- a/src/core/itemchangelog.cpp
+++ b/src/core/itemchangelog.cpp
@@ -57,9 +57,19 @@
     return m_deletedTags[const_cast<ItemPrivate *>(priv)];
 }
 
-QSet<QByteArray> &ItemChangeLog::deletedAttributes(const ItemPrivate *priv)
+AttributeStorage &ItemChangeLog::attributeStorage(const ItemPrivate *priv)
 {
-    return m_deletedAttributes[const_cast<ItemPrivate *>(priv)];
+    return m_attributeStorage[const_cast<ItemPrivate *>(priv)];
+}
+
+void ItemChangeLog::removeItem(const ItemPrivate *priv)
+{
+    ItemPrivate *p = const_cast<ItemPrivate *>(priv);
+    m_addedFlags.remove(p);
+    m_deletedFlags.remove(p);
+    m_addedTags.remove(p);
+    m_deletedTags.remove(p);
+    m_attributeStorage.remove(p);
 }
 
 void ItemChangeLog::clearItemChangelog(const ItemPrivate *priv)
@@ -69,5 +79,5 @@
     m_deletedFlags.remove(p);
     m_addedTags.remove(p);
     m_deletedTags.remove(p);
-    m_deletedAttributes.remove(p);
+    m_attributeStorage[p].resetChangeLog(); // keep the attributes
 }
diff --git a/src/core/itemchangelog_p.h b/src/core/itemchangelog_p.h
--- a/src/core/itemchangelog_p.h
+++ b/src/core/itemchangelog_p.h
@@ -25,6 +25,7 @@
 #include "item.h"
 
 #include "akonaditests_export.h"
+#include "attributestorage_p.h"
 
 namespace Akonadi
 {
@@ -40,8 +41,9 @@
     Tag::List &addedTags(const ItemPrivate *priv);
     Tag::List &deletedTags(const ItemPrivate *priv);
 
-    QSet<QByteArray> &deletedAttributes(const ItemPrivate *priv);
+    AttributeStorage &attributeStorage(const ItemPrivate *priv);
 
+    void removeItem(const ItemPrivate *priv);
     void clearItemChangelog(const ItemPrivate *priv);
 
 private:
@@ -53,7 +55,7 @@
     QHash<ItemPrivate *, Item::Flags> m_deletedFlags;
     QHash<ItemPrivate *, Tag::List> m_addedTags;
     QHash<ItemPrivate *, Tag::List> m_deletedTags;
-    QHash<ItemPrivate *, QSet<QByteArray>> m_deletedAttributes;
+    QHash<ItemPrivate *, AttributeStorage> m_attributeStorage;
 };
 
 } // namespace Akonadi
diff --git a/src/core/jobs/itemmodifyjob.cpp b/src/core/jobs/itemmodifyjob.cpp
--- a/src/core/jobs/itemmodifyjob.cpp
+++ b/src/core/jobs/itemmodifyjob.cpp
@@ -225,20 +225,23 @@
         cmd->setParts(parts);
     }
 
-    const auto deletedAttributes = ItemChangeLog::instance()->deletedAttributes(item.d_ptr);
+    const AttributeStorage &attributeStorage = ItemChangeLog::instance()->attributeStorage(item.d_ptr);
+    const QSet<QByteArray> deletedAttributes = attributeStorage.deletedAttributes();
     if (!deletedAttributes.isEmpty()) {
         QSet<QByteArray> removedParts;
         removedParts.reserve(deletedAttributes.size());
         for (const QByteArray &part : deletedAttributes) {
             removedParts.insert("ATR:" + part);
         }
         cmd->setRemovedParts(removedParts);
     }
+    if (attributeStorage.hasModifiedAttributes()) {
+        cmd->setAttributes(ProtocolHelper::attributesToProtocol(attributeStorage.modifiedAttributes()));
+    }
 
     // nothing to do
     if (cmd->modifiedParts() == Protocol::ModifyItemsCommand::None
         && mParts.isEmpty()
-        && item.attributes().isEmpty()
         && !cmd->invalidateCache()) {
         return cmd;
     }
@@ -252,8 +255,6 @@
         cmd->setItemSize(item.size());
     }
 
-    cmd->setAttributes(ProtocolHelper::attributesToProtocol(item));
-
     return cmd;
 }