I'm getting this pretty consistently when attempti...
# community-help
j
I'm getting this pretty consistently when attempting to import/upsert records (full trace in thread) - Any thoughts?
ConnectionError: ('Connection aborted.', BadStatusLine('HTTP/1.1 0 \r\n'))
Copy code
/app/huxley/core/typesense/typesense.py in import_records(collection_id, records, action)
   40   resp = collection.documents.import_(
   41     records,
---> 42     {"action": action},
   43   )
   44   failures = list(filter(lambda x: x["success"] is False, resp))

/usr/local/lib/python3.7/site-packages/typesense/documents.py in import_(self, documents, params, batch_size)
   76 
   77         docs_import = '\n'.join(document_strs)
---> 78         api_response = <http://self.api_call.post|self.api_call.post>(self._endpoint_path('import'), docs_import, params, as_json=False)
   79         res_obj_strs = api_response.split('\n')
   80 

/usr/local/lib/python3.7/site-packages/typesense/api_call.py in post(self, endpoint, body, params, as_json)
  144     return self.make_request(<http://requests.post|requests.post>, endpoint, as_json,
  145                 params=params, data=body,
--> 146                 timeout=self.config.connection_timeout_seconds)
  147 
  148   def put(self, endpoint, body, params=None):

/usr/local/lib/python3.7/site-packages/typesense/api_call.py in make_request(self, fn, endpoint, as_json, **kwargs)
  128 
  129     logger.debug('No retries left. Raising last exception: {}'.format(last_exception))
--> 130     raise last_exception
  131 
  132   def set_node_healthcheck(self, node, is_healthy):

/usr/local/lib/python3.7/site-packages/typesense/api_call.py in make_request(self, fn, endpoint, as_json, **kwargs)
   99           kwargs['data'] = json.dumps(kwargs['data'])
  100 
--> 101         r = fn(url, headers={ApiCall.API_KEY_HEADER_NAME: self.config.api_key}, **kwargs)
  102 
  103         # Treat any status code > 0 and < 500 to be an indication that node is healthy

/usr/local/lib/python3.7/site-packages/requests/api.py in post(url, data, json, **kwargs)
  113   """
  114 
--> 115   return request("post", url, data=data, json=json, **kwargs)
  116 
  117 

/usr/local/lib/python3.7/site-packages/requests/api.py in request(method, url, **kwargs)
   57   # cases, and look like a memory leak in others.
   58   with sessions.Session() as session:
---> 59     return session.request(method=method, url=url, **kwargs)
   60 
   61 

/usr/local/lib/python3.7/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
  585     }
  586     send_kwargs.update(settings)
--> 587     resp = self.send(prep, **send_kwargs)
  588 
  589     return resp

/usr/local/lib/python3.7/site-packages/requests/sessions.py in send(self, request, **kwargs)
  699 
  700     # Send the request
--> 701     r = adapter.send(request, **kwargs)
  702 
  703     # Total elapsed time of the request (approximately)

/usr/local/lib/python3.7/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
  545 
  546     except (ProtocolError, OSError) as err:
--> 547       raise ConnectionError(err, request=request)
  548 
  549     except MaxRetryError as e:

ConnectionError: ('Connection aborted.', BadStatusLine('HTTP/1.1 0 \r\n'))
Clusters are on 0.24.1, started seeing this 2 days ago
Happy to DM a cluster id for an easier debugging/investigation
I've scheduled an instance and capacity upgrade to see if that resolves it. Only seems to happen on my prod instance 😬
j
I do see an OOM issue on one of the nodes - so that would have contributed to it
Let me know if you continue seeing it after the upgrade completes
j
Cool, thanks for that visibility! We have auto upgrade capacity enabled, any idea why it wasn't handled automatically? Also anything we can do to get the upgrade sped up? (i see its in progress) The issues affecting all of our production servers and causing an influx of bug reports 😅
j
auto-scaling only kicks-in when recommended thresholds are consistently crossed for 12 hours… In this case it was a brief spike, which is why it didn’t kick-in
Yeah, it’s already in progress and the reindexing time on new higher capacity nodes is dependent on the size of your dataset…
✅ 1
j
Makes sense! Thanks.