@Jason Bosco I've reset my cluster from scratch, set openfiles to a very high number, upgrade typesense to 27.1 and I have a new crash loop:

^[[FE20241016 23:34:05.146075  7904 backward.hpp:4200] Stack trace (most recent call last) in thread 7904:
E20241016 23:34:05.146214  7904 backward.hpp:4200] #13   Object "", at 0xffffffffffffffff, in 
E20241016 23:34:05.146294  7904 backward.hpp:4200] #12   Object "/usr/lib/x86_64-linux-gnu/libc-2.31.so", at 0x7f22d7021352, in __clone
E20241016 23:34:05.146370  7904 backward.hpp:4200] #11   Object "/usr/lib/x86_64-linux-gnu/libpthread-2.31.so", at 0x7f22d710c608, in start_thread
E20241016 23:34:05.146447  7904 backward.hpp:4200] #10   Object "/usr/bin/typesense-server", at 0x55bffb1f7c33, in execute_native_thread_routine
E20241016 23:34:05.146521  7904 backward.hpp:4200] #9  | Source "include/threadpool.h", line 59, in operator()
E20241016 23:34:05.146597  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/future", line 1592, in _M_run
E20241016 23:34:05.146669  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/future", line 1459, in _M_set_result
E20241016 23:34:05.146750  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/future", line 412, in call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter>()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter>()>*, bool*>
E20241016 23:34:05.146831  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/mutex", line 729, in __gthread_once
E20241016 23:34:05.146907  7904 backward.hpp:4200]       Source "/usr/include/x86_64-linux-gnu/c++/10/bits/gthr-default.h", line 700, in ThreadPool [0x55bff868b945]
E20241016 23:34:05.146986  7904 backward.hpp:4200] #8    Object "/usr/lib/x86_64-linux-gnu/libpthread-2.31.so", at 0x7f22d71154de, in __pthread_once_slow
E20241016 23:34:05.147078  7904 backward.hpp:4200] #7  | Source "/usr/include/c++/10/future", line 572, in operator()
E20241016 23:34:05.147154  7904 backward.hpp:4200]       Source "/usr/include/c++/10/bits/std_function.h", line 622, in _M_do_set [0x55bff868ac92]
E20241016 23:34:05.147234  7904 backward.hpp:4200] #6  | Source "/usr/include/c++/10/bits/std_function.h", line 292, in __invoke_r<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter>, std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, std::__future_base::_Task_state<_Fn, _Alloc, _Res(_Args ...)>::_M_run<std::_Bind<BatchedIndexer::run()::<lambda()>()>, std::allocator<int>, void, {}>::<lambda()>, void>&>
E20241016 23:34:05.147316  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/bits/invoke.h", line 115, in __invoke_impl<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, std::__future_base::_Task_state<_Fn, _Alloc, _Res(_Args ...)>::_M_run<std::_Bind<BatchedIndexer::run()::<lambda()>()>, std::allocator<int>, void, {}>::<lambda()>, void>&>
E20241016 23:34:05.147393  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/bits/invoke.h", line 60, in operator()
E20241016 23:34:05.147464  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/future", line 1397, in operator()
E20241016 23:34:05.147536  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/future", line 1456, in __invoke_r<void, std::_Bind<BatchedIndexer::run()::<lambda()>()>&>
E20241016 23:34:05.147614  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/bits/invoke.h", line 110, in __invoke_impl<void, std::_Bind<BatchedIndexer::run()::<lambda()>()>&>
E20241016 23:34:05.147691  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/bits/invoke.h", line 60, in operator()<>
E20241016 23:34:05.147768  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/functional", line 499, in __call<void>
E20241016 23:34:05.147845  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/functional", line 416, in __invoke<BatchedIndexer::run()::<lambda()>&>
E20241016 23:34:05.147922  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/bits/invoke.h", line 95, in __invoke_impl<void, BatchedIndexer::run()::<lambda()>&>
E20241016 23:34:05.148003  7904 backward.hpp:4200]       Source "/usr/include/c++/10/bits/invoke.h", line 60, in _M_invoke [0x55bff8684a58]
E20241016 23:34:05.148077  7904 backward.hpp:4200] #5    Source "src/batched_indexer.cpp", line 292, in operator() [0x55bff8683a8b]
E20241016 23:34:05.148150  7904 backward.hpp:4200] #4    Source "src/core_api.cpp", line 2870, in put_upsert_analytics_rules [0x55bff87ccf56]
E20241016 23:34:05.148227  7904 backward.hpp:4200] #3    Source "src/analytics_manager.cpp", line 27, in create_rule [0x55bff8616425]
E20241016 23:34:05.148304  7904 backward.hpp:4200] #2    Source "src/analytics_manager.cpp", line 239, in create_index [0x55bff8614ffc]
E20241016 23:34:05.148380  7904 backward.hpp:4200] #1  | Source "src/store.cpp", line 68, in shared_lock
E20241016 23:34:05.148458  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/shared_mutex", line 722, in lock_shared
E20241016 23:34:05.148535  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/shared_mutex", line 421, in lock_shared
E20241016 23:34:05.148612  7904 backward.hpp:4200]     | Source "/usr/include/c++/10/shared_mutex", line 224, in __glibcxx_rwlock_rdlock
E20241016 23:34:05.148691  7904 backward.hpp:4200]       Source "/usr/include/c++/10/shared_mutex", line 73, in insert [0x55bff8984d10]
E20241016 23:34:05.148766  7904 backward.hpp:4200] #0    Object "/usr/lib/x86_64-linux-gnu/libpthread-2.31.so", at 0x7f22d71113f0, in __pthread_rwlock_rdlock