https://typesense.org logo
Join Slack
Powered by
Does the upsert call in the import API trigger aut...
# community-help
d
Does the upsert call in the import API trigger autoembedding? I think i might have lost my embeddings due to it I get:
Copy code
Error: 400 - Field `embedding` does not have a vector query index.
    at TypesenseInstantsearchAdapter._validateTypesenseResult (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/typesense-instantsearch-adapter@2.9.0-3_@babel+runtime@7.26.0/node_modules/typesense-instantsearch-adapter/src/TypesenseInstantsearchAdapter.js:90:13)
    at map (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/typesense-instantsearch-adapter@2.9.0-3_@babel+runtime@7.26.0/node_modules/typesense-instantsearch-adapter/src/TypesenseInstantsearchAdapter.js:26:14)
    at Array.map (<anonymous>)
    at TypesenseInstantsearchAdapter._callee$ (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/typesense-instantsearch-adapter@2.9.0-3_@babel+runtime@7.26.0/node_modules/typesense-instantsearch-adapter/src/TypesenseInstantsearchAdapter.js:25:58)
    at tryCatch (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/@babel+runtime@7.26.0/node_modules/@babel/runtime/helpers/regeneratorRuntime.js:45:16)
    at Generator.<anonymous> (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/@babel+runtime@7.26.0/node_modules/@babel/runtime/helpers/regeneratorRuntime.js:133:17)
    at Generator.next (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/@babel+runtime@7.26.0/node_modules/@babel/runtime/helpers/regeneratorRuntime.js:74:21)
    at asyncGeneratorStep (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/@babel+runtime@7.26.0/node_modules/@babel/runtime/helpers/asyncToGenerator.js:3:17)
    at _next (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/@babel+runtime@7.26.0/node_modules/@babel/runtime/helpers/asyncToGenerator.js:17:9)
[17:05:52.697] ERROR (82066): Error in SSR for search page
    err: {
      "type": "Error",
      "message": "400 - Field `embedding` does not have a vector query index.",
      "stack":
          Error: 400 - Field `embedding` does not have a vector query index.
              at TypesenseInstantsearchAdapter._validateTypesenseResult (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/typesense-instantsearch-adapter@2.9.0-3_@babel+runtime@7.26.0/node_modules/typesense-instantsearch-adapter/src/TypesenseInstantsearchAdapter.js:90:13)
              at map (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/typesense-instantsearch-adapter@2.9.0-3_@babel+runtime@7.26.0/node_modules/typesense-instantsearch-adapter/src/TypesenseInstantsearchAdapter.js:26:14)
              at Array.map (<anonymous>)
              at TypesenseInstantsearchAdapter._callee$ (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/typesense-instantsearch-adapter@2.9.0-3_@babel+runtime@7.26.0/node_modules/typesense-instantsearch-adapter/src/TypesenseInstantsearchAdapter.js:25:58)
              at tryCatch (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/@babel+runtime@7.26.0/node_modules/@babel/runtime/helpers/regeneratorRuntime.js:45:16)
              at Generator.<anonymous> (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/@babel+runtime@7.26.0/node_modules/@babel/runtime/helpers/regeneratorRuntime.js:133:17)
              at Generator.next (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/@babel+runtime@7.26.0/node_modules/@babel/runtime/helpers/regeneratorRuntime.js:74:21)
              at asyncGeneratorStep (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/@babel+runtime@7.26.0/node_modules/@babel/runtime/helpers/asyncToGenerator.js:3:17)
              at _next (/Users/wayrapro/Documents/Projects/onde-vamos/node_modules/.pnpm/@babel+runtime@7.26.0/node_modules/@babel/runtime/helpers/asyncToGenerator.js:17:9)
    }
j
The upsert call triggers auto-embedding calls, only if the fields inside 
embed.from
change
Could you share the field definition for 
embedding
from your schema?
d
hmmmm…..
Copy code
{
  "created_at": 1726001707,
  "default_sorting_field": "",
  "enable_nested_fields": true,
  "fields": [
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "title",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "description",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "m2_size",
      "optional": false,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "float"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "parking",
      "optional": false,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "bool"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "city",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "state",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "location",
      "optional": false,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "geopoint"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "images",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string[]"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "updated_at_unix",
      "optional": false,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "int64"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "activities",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string[]"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "main_activities",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string[]"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "availability",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "object[]"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "bookings",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "object[]"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "availability.start_time_int",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "int64[]"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "availability.end_time_int",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "int64[]"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "availability.day_of_week",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "int64[]"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "amenities",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string[]"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "category",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "price_per_hour",
      "optional": true,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "float"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "max_capacity",
      "optional": false,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "int32"
    }
  ],
  "name": "onde_sites",
  "num_documents": 212,
  "symbols_to_index": [],
  "token_separators": []
}
I don’t see embeddings in it
maybe….I’m misremembering that we had autoembedding
our indexing pipeline was recently reimplemented….and perhaps we skipped to generate the embeddings themselves?
hmmm
but that’s wild
we have also onde_spaces_dev which was a clone of the main one
Copy code
{
  "created_at": 1730142736,
  "default_sorting_field": "",
  "enable_nested_fields": true,
  "fields": [
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "title",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "description",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "m2_size",
      "optional": false,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "float"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "parking",
      "optional": false,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "bool"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "city",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "state",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "location",
      "optional": false,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "geopoint"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "images",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string[]"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "updated_at_unix",
      "optional": false,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "int64"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "activities",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string[]"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "main_activities",
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string[]"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "availability",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "object[]"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "bookings",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "object[]"
    },
    {
      "embed": {
        "from": [
          "title",
          "description"
        ],
        "model_config": {
          "api_key": "sk-jY**********************************************",
          "model_name": "openai/text-embedding-3-small"
        }
      },
      "facet": false,
      "hnsw_params": {
        "M": 16,
        "ef_construction": 200
      },
      "index": true,
      "infix": false,
      "locale": "",
      "name": "embedding",
      "num_dim": 1536,
      "optional": false,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "float[]",
      "vec_dist": "cosine"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "availability.start_time_int",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "int64[]"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "availability.end_time_int",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "int64[]"
    },
    {
      "facet": false,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "availability.day_of_week",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "int64[]"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "amenities",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string[]"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "category",
      "optional": true,
      "sort": false,
      "stem": false,
      "store": true,
      "type": "string"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "price_per_hour",
      "optional": true,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "float"
    },
    {
      "facet": true,
      "index": true,
      "infix": false,
      "locale": "",
      "name": "max_capacity",
      "optional": false,
      "sort": true,
      "stem": false,
      "store": true,
      "type": "int32"
    }
  ],
  "name": "onde_sites_dev",
  "num_documents": 205,
  "symbols_to_index": [],
  "token_separators": []
}
and it does have the embed
so…I’m not sure what could have happened. That the clone keeps the embed but not the main one
j
So yeah, somehow the embedding field is missing from the other collection. May be it wasn't created during the reindexing process in the new schema, in your indexing pipeline?
d
…. but the _dev one schema was cloned from the main one….
I haven’t changed the schema programatically, so I don’t know what could have caused this.
is there a way to see history of a schema to see what/who changed it? or when it changed?
j
I just created a collection with open ai auto-embedding and tried cloning it to a new collection, and the auto-embedding field definitions were carried over
We have a history of API calls made to the cluster, but not the actual payload inside these requests
d
Sorry i must be confusing in my words. The first schema is the first one created: onde_vamos. I’m pretty sure this had the auto embed the second schema onde_vamos_dev is a clone of onde_vamos. So if it has the auto embed, I’m assuming at some point in the past 
onde_vamos
also had it. Currently the 
onde_vamos_dev
one has the embed field and the 
onde_vamos
does not. This is not something we desired to do and not sure how it would have been made. We only used the import api and upsert in our indexing pipeline
This is our code for indexing
Copy code
export const bulkIndexToTypesense = async ({
  transformedSpaces,
}: { transformedSpaces: SearchSpacesHit[] }) => {
  const spaceDocuments: SearchSpacesHit[] = transformedSpaces;
  const indexName =
    Config.STAGE === "prod" ? IndexNames.onde_sites : IndexNames.onde_sites_dev;

  <http://logger.info|logger.info>(
    spaceDocuments.map((e) => e.id),
    "Processing spaces",
  );

  const client = getAdminClient(Config.TYPESENSE_ADMIN_API_KEY);
  try {
    const res = await client
      .collections<SearchSpacesHit & { embedding?: number[] }>(indexName)
      .documents()
      .import(spaceDocuments, { action: "upsert" });
    return { res, error: null };
  } catch (err) {
    if (err instanceof ImportError) {
      logger.error(err, "Error indexing to Typesense");
      return {
        error: err.importResults.filter((e) => !e.success),
        res: null,
      };
    }
    return { error: err, res: null };
  }
};
j
the second schema onde_vamos_dev is a clone of onde_vamos
How was this clone created?
By clicking the "Clone Schema" button in the dashboard under Collection Settings?
d
yessir
j
So somehow that field was explicitly dropped after cloning
d
Well yeah. I’m not sure if I did something wrong on my end. How can I re-enable embedding? Would reindexing the whole corpus with upsert trigger the embedding afterwards?
j
Just adding an embedding field to the existing schema will trigger embedding generation
Or you could also just create a new collection (make sure the auto embedding field is present) and then reimport your dataset with upsert
d
what’s the syntax for embedding? should ijust copy the field here?
j
Yup it's what you have in the screenshot, but just need to replace the api key with your actual open ai api key
You want to remove lines 3-12
Open in Slack
PreviousNext
Powered by