Hello every one , we are facing some inconsistent result on large dataset with nested fields (i didn't notice problem up to 1M document; but now testing with 17M document i see issues below). we are using docker image typesense:29.0.rc20 (because we need this commit for stable operation). I m not sure how to debug, what shall i look for ? did i do something wroong in the schema? • the filter does return data that do not match: query:

{q:'*',filter_by:'on_sale:true && sale.type:EnglishAuction',include_fields: 'price,on_sale,sale.price,sale.type,sale.primary', per_page:20}

{"facet_counts" => [],
 "found" => 1640,
 "hits" =>
  [{"document" => {"on_sale" => true, "price" => 90, "sale" => {"price" => 90, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 90, "sale" => {"price" => 90, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 260, "sale" => {"price" => 260, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 400, "sale" => {"price" => 400, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 100, "sale" => {"price" => 100, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 220, "sale" => {"price" => 220, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 520, "sale" => {"price" => 520, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 90, "sale" => {"price" => 90, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 550, "sale" => {"price" => 550, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 90, "sale" => {"price" => 90, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 320, "sale" => {"price" => 320, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 90, "sale" => {"price" => 90, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 320, "sale" => {"price" => 320, "primary" => false, "type" => "SingleSaleOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 4180, "sale" => {"price" => 4180, "primary" => false, "type" => "SingleSaleOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 540, "sale" => {"price" => 540, "primary" => false, "type" => "SingleSaleOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 330, "sale" => {"price" => 330, "primary" => false, "type" => "SingleSaleOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 590, "sale" => {"price" => 590, "primary" => false, "type" => "SingleSaleOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 410, "sale" => {"price" => 410, "primary" => false, "type" => "SingleSaleOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 90, "sale" => {"price" => 90, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "price" => 90, "sale" => {"price" => 90, "primary" => true, "type" => "EnglishAuction"}}, "highlight" => {}, "highlights" => []}],
 "out_of" => 15712569,
 "page" => 1,
 "request_params" => {"collection_name" => "blockchain_cards-1", "first_q" => "*", "per_page" => 20, "q" => "*"},
 "search_cutoff" => false,
 "search_time_ms" => 9}

• Order is not respected query, data seems somewhat orderd and i noticed that

desc

order is way more consistent than

asc

order

{q:'*',filter_by:'on_sale:true && sale.type:PrimaryOffer && sale.price:>1000' ,sort_by:'sale.price:asc',include_fields: 'on_sale,sale.price,sale.type,sale.primary', per_page:20}

{"facet_counts" => [],
 "found" => 3581,
 "hits" =>
  [{"document" => {"on_sale" => true, "sale" => {"price" => 2290, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 5300, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 1220, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 1220, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 1220, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 5300, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 5300, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 1220, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 5300, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 1220, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 1220, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 5300, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 5300, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 5300, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 1220, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 6200, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 1220, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 6200, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 6200, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []},
   {"document" => {"on_sale" => true, "sale" => {"price" => 6200, "primary" => true, "type" => "PrimaryOffer"}}, "highlight" => {}, "highlights" => []}],
 "out_of" => 15712568,
 "page" => 1,
 "request_params" => {"collection_name" => "blockchain_cards-1", "first_q" => "*", "per_page" => 20, "q" => "*"},
 "search_cutoff" => false,
 "search_time_ms" => 13}

and here the relavant part of the schema (the schema has 139 fields i filtred the one that are involved)

{"created_at" => 1747317842,
 "default_sorting_field" => "",
 "enable_nested_fields" => true,
 "fields" =>
  [
   {"facet" => true, "index" => true, "infix" => false, "locale" => "", "name" => "on_sale", "optional" => true, "sort" => true, "stem" => false, "stem_dictionary" => "", "store" => true, "type" => "bool"},
   {"facet" => false, "index" => true, "infix" => false, "locale" => "", "name" => "sale", "optional" => true, "sort" => false, "stem" => false, "stem_dictionary" => "", "store" => true, "type" => "object"},
   {"facet" => false, "index" => true, "infix" => false, "locale" => "", "name" => "sale.end_date", "optional" => true, "sort" => true, "stem" => false, "stem_dictionary" => "", "store" => true, "type" => "int64"},
   {"facet" => true, "index" => true, "infix" => false, "locale" => "", "name" => "sale.id", "optional" => true, "sort" => false, "stem" => false, "stem_dictionary" => "", "store" => true, "type" => "string"},
   {"facet" => true, "index" => true, "infix" => false, "locale" => "", "name" => "sale.price", "optional" => true, "range_index" => true, "sort" => true, "stem" => false, "stem_dictionary" => "", "store" => true, "type" => "int64"},
   {"facet" => true, "index" => true, "infix" => false, "locale" => "", "name" => "sale.primary", "optional" => true, "sort" => false, "stem" => false, "stem_dictionary" => "", "store" => true, "type" => "bool"},   {"facet" => true, "index" => true, "infix" => false, "locale" => "", "name" => "sale.price_range", "optional" => true, "sort" => false, "stem" => false, "stem_dictionary" => "", "store" => true, "type" => "string"},
   {"facet" => true, "index" => true, "infix" => false, "locale" => "", "name" => "sale.type", "optional" => true, "sort" => false, "stem" => false, "stem_dictionary" => "", "store" => true, "type" => "string"},
   {"facet" => false, "index" => true, "infix" => false, "locale" => "", "name" => "sale.type_ranked", "optional" => true, "sort" => true, "stem" => false, "stem_dictionary" => "", "store" => true, "type" => "int64"}
]
 "name" => "blockchain_cards-1",
 "num_documents" => 15712567,
 "symbols_to_index" => [],
 "token_separators" => []}

Thanks!