try to adjust recycling strategy for large message cache

mutouyun · mutouyun · commit 628914d4288a · 2021-07-11T13:13:30.000+08:00
diff --git a/src/ipc.cpp b/src/ipc.cpp
@@ -31,8 +31,8 @@
 
 namespace {
 
-using msg_id_t = std::uint32_t;
-using acc_t    = std::atomic<msg_id_t>;
+using msg_id_t   = std::uint32_t;
+using acc_t      = std::atomic<msg_id_t>;
 
 template <std::size_t DataSize, std::size_t AlignSize>
 struct msg_t;
@@ -91,6 +91,14 @@ auto cc_acc() {
     return static_cast<acc_t*>(acc_h.get());
 }
 
+IPC_CONSTEXPR_ std::size_t align_chunk_size(std::size_t size) noexcept {
+    return (((size - 1) / ipc::large_msg_align) + 1) * ipc::large_msg_align;
+}
+
+IPC_CONSTEXPR_ std::size_t calc_chunk_size(std::size_t size) noexcept {
+    return ipc::make_align(alignof(std::max_align_t), align_chunk_size(size));
+}
+
 struct chunk_info_t {
     ipc::id_pool<> pool_;
     ipc::spin_lock lock_;
@@ -99,9 +107,13 @@ struct chunk_info_t {
         return ipc::id_pool<>::max_count * chunk_size;
     }
 
-    ipc::byte_t *at(std::size_t chunk_size, ipc::storage_id_t id) noexcept {
-        if (id < 0) return nullptr;
-        return reinterpret_cast<ipc::byte_t *>(this + 1) + (chunk_size * id);
+    ipc::byte_t* chunks_mem() noexcept {
+        return reinterpret_cast<ipc::byte_t*>(this + 1);
+    }
+
+    ipc::byte_t* at(std::size_t chunk_size, ipc::storage_id_t id) noexcept {
+        assert(id >= 0);
+        return chunks_mem() + (chunk_size * id);
     }
 };
 
@@ -129,16 +141,11 @@ auto& chunk_storages() {
     return chunk_s;
 }
 
-IPC_CONSTEXPR_ std::size_t calc_chunk_size(std::size_t size) noexcept {
-    return ipc::make_align(alignof(std::max_align_t), 
-                          (((size - 1) / ipc::large_msg_align) + 1) * ipc::large_msg_align);
-}
-
 chunk_info_t *chunk_storage_info(std::size_t chunk_size) {
     return chunk_storages()[chunk_size].get_info(chunk_size);
 }
 
-std::pair<ipc::storage_id_t, void*> apply_storage(std::size_t size) {
+std::pair<ipc::storage_id_t, void*> acquire_storage(std::size_t size) {
     std::size_t chunk_size = calc_chunk_size(size);
     auto info = chunk_storage_info(chunk_size);
     if (info == nullptr) return {};
@@ -165,14 +172,12 @@ void *find_storage(ipc::storage_id_t id, std::size_t size) {
 
 void release_storage(ipc::storage_id_t id, std::size_t size) {
     if (id < 0) {
-        ipc::error("[clear_storage] id is invalid: id = %ld, size = %zd\n", (long)id, size);
+        ipc::error("[release_storage] id is invalid: id = %ld, size = %zd\n", (long)id, size);
         return;
     }
-
     std::size_t chunk_size = calc_chunk_size(size);
     auto info = chunk_storage_info(chunk_size);
     if (info == nullptr) return;
-
     info->lock_.lock();
     info->pool_.release(id);
     info->lock_.unlock();
@@ -381,7 +386,7 @@ static bool send(F&& gen_push, ipc::handle_t h, void const * data, std::size_t s
     auto msg_id   = acc->fetch_add(1, std::memory_order_relaxed);
     auto try_push = std::forward<F>(gen_push)(info_of(h), que, msg_id);
     if (size > ipc::large_msg_limit) {
-        auto   dat = apply_storage(size);
+        auto   dat = acquire_storage(size);
         void * buf = dat.second;
         if (buf != nullptr) {
             std::memcpy(buf, data, size);
@@ -416,7 +421,7 @@ static bool send(ipc::handle_t h, void const * data, std::size_t size, std::size
         return [tm, info, que, msg_id](std::int32_t remain, void const * data, std::size_t size) {
             if (!wait_for(info->wt_waiter_, [&] {
                     return !que->push(
-                        recycle_message<typename queue_t::value_t>, 
+                        [](void*) { return true; },
                         info->cc_id_, msg_id, remain, data, size);
                 }, tm)) {
                 ipc::log("force_push: msg_id = %zd, remain = %d, size = %zd\n", msg_id, remain, size);
@@ -437,7 +442,7 @@ static bool try_send(ipc::handle_t h, void const * data, std::size_t size, std::
         return [tm, info, que, msg_id](std::int32_t remain, void const * data, std::size_t size) {
             if (!wait_for(info->wt_waiter_, [&] {
                     return !que->push(
-                        recycle_message<typename queue_t::value_t>, 
+                        [](void*) { return true; },
                         info->cc_id_, msg_id, remain, data, size);
                 }, tm)) {
                 return false;
@@ -462,7 +467,10 @@ static ipc::buff_t recv(ipc::handle_t h, std::size_t tm) {
     for (;;) {
         // pop a new message
         typename queue_t::value_t msg;
-        if (!wait_for(info_of(h)->rd_waiter_, [que, &msg] { return !que->pop(msg); }, tm)) {
+        bool recycled = false;
+        if (!wait_for(info_of(h)->rd_waiter_, [que, &msg, &recycled] {
+                return !que->pop(msg, [&recycled](bool r) { recycled = r; });
+            }, tm)) {
             // pop failed, just return.
             return {};
         }
@@ -477,23 +485,29 @@ static ipc::buff_t recv(ipc::handle_t h, std::size_t tm) {
             return {};
         }
         std::size_t msg_size = static_cast<std::size_t>(r_size);
+        // large message
+        if (msg.storage_) {
+            ipc::storage_id_t buf_id = *reinterpret_cast<ipc::storage_id_t*>(&msg.data_);
+            void* buf = find_storage(buf_id, msg_size);
+            if (buf != nullptr) {
+                if (recycled) {
+                    return ipc::buff_t{buf, msg_size, [](void* pmid, std::size_t size) {
+                        release_storage(reinterpret_cast<ipc::storage_id_t>(pmid) - 1, size);
+                    }, reinterpret_cast<void*>(buf_id + 1)};
+                } else {
+                    return ipc::buff_t{buf, msg_size}; // no recycle
+                }
+            } else {
+                ipc::log("fail: shm::handle for large message. msg_id: %zd, buf_id: %zd, size: %zd\n", msg.id_, buf_id, msg_size);
+                continue;
+            }
+        }
         // find cache with msg.id_
         auto cac_it = rc.find(msg.id_);
         if (cac_it == rc.end()) {
             if (msg_size <= ipc::data_length) {
                 return make_cache(msg.data_, msg_size);
             }
-            if (msg.storage_) {
-                std::size_t buf_id = *reinterpret_cast<std::size_t*>(&msg.data_);
-                void      * buf    = find_storage(buf_id, msg_size);
-                if (buf != nullptr) {
-                    return ipc::buff_t{buf, msg_size};
-                }
-                else {
-                    ipc::log("fail: shm::handle for big message. msg_id: %zd, buf_id: %zd, size: %zd\n", msg.id_, buf_id, msg_size);
-                    continue;
-                }
-            }
             // gc
             if (rc.size() > 1024) {
                 std::vector<msg_id_t> need_del;
diff --git a/src/libipc/circ/elem_array.h b/src/libipc/circ/elem_array.h
@@ -130,10 +130,10 @@ class elem_array : public ipc::circ::conn_head<Policy> {
         return head_.force_push(que, std::forward<F>(f), block_);
     }
 
-    template <typename Q, typename F>
-    bool pop(Q* que, cursor_t* cur, F&& f) {
+    template <typename Q, typename F, typename R>
+    bool pop(Q* que, cursor_t* cur, F&& f, R&& out) {
         if (cur == nullptr) return false;
-        return head_.pop(que, *cur, std::forward<F>(f), block_);
+        return head_.pop(que, *cur, std::forward<F>(f), std::forward<R>(out), block_);
     }
 };
 
diff --git a/src/libipc/prod_cons.h b/src/libipc/prod_cons.h
@@ -58,13 +58,14 @@ struct prod_cons_impl<wr<relat::single, relat::single, trans::unicast>> {
         return false;
     }
 
-    template <typename W, typename F, typename E>
-    bool pop(W* /*wrapper*/, circ::u2_t& /*cur*/, F&& f, E* elems) {
+    template <typename W, typename F, typename R, typename E>
+    bool pop(W* /*wrapper*/, circ::u2_t& /*cur*/, F&& f, R&& out, E* elems) {
         auto cur_rd = circ::index_of(rd_.load(std::memory_order_relaxed));
         if (cur_rd == circ::index_of(wt_.load(std::memory_order_acquire))) {
             return false; // empty
         }
         std::forward<F>(f)(&(elems[cur_rd].data_));
+        std::forward<R>(out)(true);
         rd_.fetch_add(1, std::memory_order_release);
         return true;
     }
@@ -80,8 +81,9 @@ struct prod_cons_impl<wr<relat::single, relat::multi , trans::unicast>>
         return false;
     }
 
-    template <typename W, typename F, template <std::size_t, std::size_t> class E, std::size_t DS, std::size_t AS>
-    bool pop(W* /*wrapper*/, circ::u2_t& /*cur*/, F&& f, E<DS, AS>* elems) {
+    template <typename W, typename F, typename R, 
+              template <std::size_t, std::size_t> class E, std::size_t DS, std::size_t AS>
+    bool pop(W* /*wrapper*/, circ::u2_t& /*cur*/, F&& f, R&& out, E<DS, AS>* elems) {
         byte_t buff[DS];
         for (unsigned k = 0;;) {
             auto cur_rd = rd_.load(std::memory_order_relaxed);
@@ -92,6 +94,7 @@ struct prod_cons_impl<wr<relat::single, relat::multi , trans::unicast>>
             std::memcpy(buff, &(elems[circ::index_of(cur_rd)].data_), sizeof(buff));
             if (rd_.compare_exchange_weak(cur_rd, cur_rd + 1, std::memory_order_release)) {
                 std::forward<F>(f)(buff);
+                std::forward<R>(out)(true);
                 return true;
             }
             ipc::yield(k);
@@ -156,8 +159,9 @@ struct prod_cons_impl<wr<relat::multi , relat::multi, trans::unicast>>
         return false;
     }
 
-    template <typename W, typename F, template <std::size_t, std::size_t> class E, std::size_t DS, std::size_t AS>
-    bool pop(W* /*wrapper*/, circ::u2_t& /*cur*/, F&& f, E<DS, AS>* elems) {
+    template <typename W, typename F, typename R, 
+              template <std::size_t, std::size_t> class E, std::size_t DS, std::size_t AS>
+    bool pop(W* /*wrapper*/, circ::u2_t& /*cur*/, F&& f, R&& out, E<DS, AS>* elems) {
         byte_t buff[DS];
         for (unsigned k = 0;;) {
             auto cur_rd = rd_.load(std::memory_order_relaxed);
@@ -179,6 +183,7 @@ struct prod_cons_impl<wr<relat::multi , relat::multi, trans::unicast>>
                 std::memcpy(buff, &(elems[circ::index_of(cur_rd)].data_), sizeof(buff));
                 if (rd_.compare_exchange_weak(cur_rd, cur_rd + 1, std::memory_order_release)) {
                     std::forward<F>(f)(buff);
+                    std::forward<R>(out)(true);
                     return true;
                 }
                 ipc::yield(k);
@@ -263,20 +268,20 @@ struct prod_cons_impl<wr<relat::single, relat::multi, trans::broadcast>> {
         return true;
     }
 
-    template <typename W, typename F, typename E>
-    bool pop(W* wrapper, circ::u2_t& cur, F&& f, E* elems) {
+    template <typename W, typename F, typename R, typename E>
+    bool pop(W* wrapper, circ::u2_t& cur, F&& f, R&& out, E* elems) {
         if (cur == cursor()) return false; // acquire
         auto* el = elems + circ::index_of(cur++);
         std::forward<F>(f)(&(el->data_));
         for (unsigned k = 0;;) {
             auto cur_rc = el->rc_.load(std::memory_order_acquire);
-            circ::cc_t rem_cc = cur_rc & ep_mask;
-            if (rem_cc == 0) {
+            if ((cur_rc & ep_mask) == 0) {
+                std::forward<R>(out)(true);
                 return true;
             }
-            if (el->rc_.compare_exchange_weak(cur_rc,
-                        cur_rc & ~static_cast<rc_t>(wrapper->connected_id()),
-                        std::memory_order_release)) {
+            auto nxt_rc = cur_rc & ~static_cast<rc_t>(wrapper->connected_id());
+            if (el->rc_.compare_exchange_weak(cur_rc, nxt_rc, std::memory_order_release)) {
+                std::forward<R>(out)((nxt_rc & ep_mask) == 0);
                 return true;
             }
             ipc::yield(k);
@@ -395,8 +400,8 @@ struct prod_cons_impl<wr<relat::multi, relat::multi, trans::broadcast>> {
         return true;
     }
 
-    template <typename W, typename F, typename E, std::size_t N>
-    bool pop(W* wrapper, circ::u2_t& cur, F&& f, E(& elems)[N]) {
+    template <typename W, typename F, typename R, typename E, std::size_t N>
+    bool pop(W* wrapper, circ::u2_t& cur, F&& f, R&& out, E(& elems)[N]) {
         auto* el = elems + circ::index_of(cur);
         auto cur_fl = el->f_ct_.load(std::memory_order_acquire);
         if (cur_fl != ~static_cast<flag_t>(cur)) {
@@ -406,17 +411,18 @@ struct prod_cons_impl<wr<relat::multi, relat::multi, trans::broadcast>> {
         std::forward<F>(f)(&(el->data_));
         for (unsigned k = 0;;) {
             auto cur_rc = el->rc_.load(std::memory_order_acquire);
-            circ::cc_t rem_cc = cur_rc & rc_mask;
-            if (rem_cc == 0) {
+            if ((cur_rc & rc_mask) == 0) {
+                std::forward<R>(out)(true);
                 el->f_ct_.store(cur + N - 1, std::memory_order_release);
                 return true;
             }
-            if ((rem_cc & ~wrapper->connected_id()) == 0) {
+            auto nxt_rc = inc_rc(cur_rc) & ~static_cast<rc_t>(wrapper->connected_id());
+            bool last_one = false;
+            if ((last_one = (nxt_rc & rc_mask) == 0)) {
                 el->f_ct_.store(cur + N - 1, std::memory_order_release);
             }
-            if (el->rc_.compare_exchange_weak(cur_rc,
-                        inc_rc(cur_rc) & ~static_cast<rc_t>(wrapper->connected_id()),
-                        std::memory_order_release)) {
+            if (el->rc_.compare_exchange_weak(cur_rc, nxt_rc, std::memory_order_release)) {
+                std::forward<R>(out)(last_one);
                 return true;
             }
             ipc::yield(k);
diff --git a/src/libipc/queue.h b/src/libipc/queue.h
@@ -171,14 +171,14 @@ class queue_base : public queue_conn {
         });
     }
 
-    template <typename T>
-    bool pop(T& item) {
+    template <typename T, typename F>
+    bool pop(T& item, F&& out) {
         if (elems_ == nullptr) {
             return false;
         }
         return elems_->pop(this, &(this->cursor_), [&item](void* p) {
             ::new (&item) T(std::move(*static_cast<T*>(p)));
-        });
+        }, std::forward<F>(out));
     }
 };
 
@@ -204,7 +204,12 @@ class queue final : public detail::queue_base<typename Policy::template elems_t<
     }
 
     bool pop(T& item) {
-        return base_t::pop(item);
+        return base_t::pop(item, [](bool) {});
+    }
+
+    template <typename F>
+    bool pop(T& item, F&& out) {
+        return base_t::pop(item, std::forward<F>(out));
     }
 };
 

Original file line number	Diff line number	Diff line change
`@@ -130,10 +130,10 @@ class elem_array : public ipc::circ::conn_head<Policy> {`
`130`	`130`	`return head_.force_push(que, std::forward<F>(f), block_);`
`131`	`131`	`}`
`132`	`132`
`133`		`- template <typename Q, typename F>`
`134`		`- bool pop(Q* que, cursor_t* cur, F&& f) {`
	`133`	`+ template <typename Q, typename F, typename R>`
	`134`	`+ bool pop(Q* que, cursor_t* cur, F&& f, R&& out) {`
`135`	`135`	`if (cur == nullptr) return false;`
`136`		`- return head_.pop(que, *cur, std::forward<F>(f), block_);`
	`136`	`+ return head_.pop(que, *cur, std::forward<F>(f), std::forward<R>(out), block_);`
`137`	`137`	`}`
`138`	`138`	`};`
`139`	`139`
Original file line number	Diff line number	Diff line change
`@@ -171,14 +171,14 @@ class queue_base : public queue_conn {`
`171`	`171`	`});`
`172`	`172`	`}`
`173`	`173`
`174`		`- template <typename T>`
`175`		`- bool pop(T& item) {`
	`174`	`+ template <typename T, typename F>`
	`175`	`+ bool pop(T& item, F&& out) {`
`176`	`176`	`if (elems_ == nullptr) {`
`177`	`177`	`return false;`
`178`	`178`	`}`
`179`	`179`	`return elems_->pop(this, &(this->cursor_), [&item](void* p) {`
`180`	`180`	`::new (&item) T(std::move(static_cast<T>(p)));`
`181`		`- });`
	`181`	`+ }, std::forward<F>(out));`
`182`	`182`	`}`
`183`	`183`	`};`
`184`	`184`
`@@ -204,7 +204,12 @@ class queue final : public detail::queue_base<typename Policy::template elems_t<`
`204`	`204`	`}`
`205`	`205`
`206`	`206`	`bool pop(T& item) {`
`207`		`- return base_t::pop(item);`
	`207`	`+ return base_t::pop(item, [](bool) {});`
	`208`	`+ }`
	`209`	`+`
	`210`	`+ template <typename F>`
	`211`	`+ bool pop(T& item, F&& out) {`
	`212`	`+ return base_t::pop(item, std::forward<F>(out));`
`208`	`213`	`}`
`209`	`214`	`};`
`210`	`215`