Hi All,
I am getting following error while Importing the NDJSON file.
File "C:\Users\VenkadeshMuthuraj\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\requests\models.py", line 974, in json
return complexjson.loads(self.text, **kwargs)
~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.13_3.13.496.0_x64__qbz5n2kfra8p0\Lib\json\__init__.py", line 346, in loads
return _default_decoder.decode(s)
~~~~~~~~~~~~~~~~~~~~~~~^^^
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.13_3.13.496.0_x64__qbz5n2kfra8p0\Lib\json\decoder.py", line 348, in decode
raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data: line 2 column 1 (char 17)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.13_3.13.496.0_x64__qbz5n2kfra8p0\Lib\runpy.py", line 198, in _run_module_as_main
return _run_code(code, main_globals, None,
"__main__", mod_spec)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.13_3.13.496.0_x64__qbz5n2kfra8p0\Lib\runpy.py", line 88, in _run_code
exec(code, run_globals)
~~~~^^^^^^^^^^^^^^^^^^^
File "c:\Users\VenkadeshMuthuraj\.vscode\extensions\ms-python.debugpy-2024.12.0-win32-x64\bundled\libs\debugpy\adapter/../..\debugpy\launcher/../..\debugpy\__main__.py", line 71, in <module>
cli.main()
~~~~~~~~^^
File "c:\Users\VenkadeshMuthuraj\.vscode\extensions\ms-python.debugpy-2024.12.0-win32-x64\bundled\libs\debugpy\adapter/../..\debugpy\launcher/../..\debugpy/..\debugpy\server\cli.py", line 501, in main
run()
~~~^^
File "c:\Users\VenkadeshMuthuraj\.vscode\extensions\ms-python.debugpy-2024.12.0-win32-x64\bundled\libs\debugpy\adapter/../..\debugpy\launcher/../..\debugpy/..\debugpy\server\cli.py", line 351, in run_file
runpy.run_path(target, run_name="__main__")
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "c:\Users\VenkadeshMuthuraj\.vscode\extensions\ms-python.debugpy-2024.12.0-win32-x64\bundled\libs\debugpy\_vendored\pydevd\_pydevd_bundle\pydevd_runpy.py", line 310, in run_path
return _run_module_code(code, init_globals, run_name, pkg_name=pkg_name, script_name=fname)
File "c:\Users\VenkadeshMuthuraj\.vscode\extensions\ms-python.debugpy-2024.12.0-win32-x64\bundled\libs\debugpy\_vendored\pydevd\_pydevd_bundle\pydevd_runpy.py", line 127, in _run_module_code
_run_code(code, mod_globals, init_globals, mod_name, mod_spec, pkg_name, script_name)
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "c:\Users\VenkadeshMuthuraj\.vscode\extensions\ms-python.debugpy-2024.12.0-win32-x64\bundled\libs\debugpy\_vendored\pydevd\_pydevd_bundle\pydevd_runpy.py", line 118, in _run_code
exec(code, run_globals)
~~~~^^^^^^^^^^^^^^^^^^^
File "c:\One Drive\OneDrive - Optx\Documents\TypeSensePOC\BulkLoadTotypeSense\NDJSONtoTypeSense.py", line 34, in <module>
bulk_import_to_typesense(data_batches)
~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^
File "c:\One Drive\OneDrive - Optx\Documents\TypeSensePOC\BulkLoadTotypeSense\NDJSONtoTypeSense.py", line 30, in bulk_import_to_typesense
print(response.status_code, response.json())
~~~~~~~~~~~~~^^
File "C:\Users\VenkadeshMuthuraj\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\requests\models.py", line 978, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Extra data: line 2 column 1 (char 17)
Here is Python Code I am using it :
import requests
import json
TypeSense_API_ENDPOINT = "
http://server:8108/collections/playerimport/documents/import?action=upsert"
TypeSense_API_KEY = "aaa"
def load_data_in_batches(file_path, batch_size=500):
with open(file_path, 'r') as file:
batch = []
for line in file:
batch.append(json.loads(line))
if len(batch) == batch_size:
yield batch
batch = []
if batch:
yield batch # Yield remaining record
def bulk_import_to_typesense(data_batches):
headers = {
"X-TYPESENSE-API-KEY": TypeSense_API_KEY,
"Content-Type": "text/plain"
}
for batch in data_batches:
dataj= "\n".join(json.dumps(doc) for doc in batch)
print(dataj)
response =
requests.post(TypeSense_API_ENDPOINT, headers=headers, data=dataj)
print(response.status_code, response.json())
# Load and send batches
data_batches = load_data_in_batches("PlayerTop10.ndjson")
bulk_import_to_typesense(data_batches)
Kindly Help me to resolve this issue.