#include <string.h>
#include <limits.h>
#include "search.h"
#include "array.h"
#include "internal.h"

/***************************************************************************
 *
 * Explanation - Used to give details for query scores
 *
 ***************************************************************************/

Explanation *expl_new(float value, const char *description, ...)
{
    Explanation *expl = ALLOC(Explanation);

    va_list args;
    va_start(args, description);
    expl->description = vstrfmt(description, args);
    va_end(args);

    expl->value = value;
    expl->details = ary_new_type_capa(Explanation *,
                                      EXPLANATION_DETAILS_START_SIZE);
    return expl;
}

void expl_destroy(Explanation *expl)
{
    ary_destroy((void **)expl->details, (free_ft)expl_destroy);
    free(expl->description);
    free(expl);
}

Explanation *expl_add_detail(Explanation *expl, Explanation *detail)
{
    ary_push(expl->details, detail);
    return expl;
}

char *expl_to_s_depth(Explanation *expl, int depth)
{
    int i;
    char *buffer = ALLOC_N(char, depth * 2 + 1);
    const int num_details = ary_size(expl->details);

    memset(buffer, ' ', sizeof(char) * depth * 2);
    buffer[depth*2] = 0;

    buffer = estrcat(buffer, strfmt("%f = %s\n",
                                    expl->value, expl->description));
    for (i = 0; i < num_details; i++) {
        buffer = estrcat(buffer, expl_to_s_depth(expl->details[i], depth + 1));
    }

    return buffer;
}

char *expl_to_html(Explanation *expl)
{
    int i;
    char *buffer;
    const int num_details = ary_size(expl->details);

    buffer = strfmt("<ul>\n<li>%f = %s</li>\n", expl->value, expl->description);

    for (i = 0; i < num_details; i++) {
        estrcat(buffer, expl_to_html(expl->details[i]));
    }

    REALLOC_N(buffer, char, strlen(buffer) + 10);
    return strcat(buffer, "</ul>\n");
}

/***************************************************************************
 *
 * Hit
 *
 ***************************************************************************/

static bool hit_less_than(const Hit *hit1, const Hit *hit2)
{
    if (hit1->score == hit2->score) {
        return hit1->doc > hit2->doc;
    }
    else {
        return hit1->score < hit1->score;
    }
}

static bool hit_lt(Hit *hit1, Hit *hit2)
{
    if (hit1->score == hit2->score) {
        return hit1->doc > hit2->doc;
    }
    else {
        return hit1->score < hit2->score;
    }
}

static void hit_pq_down(PriorityQueue *pq)
{
    register int i = 1;
    register int j = 2;     /* i << 1; */
    register int k = 3;     /* j + 1;  */
    Hit **heap = (Hit **)pq->heap;
    Hit *node = heap[i];    /* save top node */

    if ((k <= pq->size) && hit_lt(heap[k], heap[j])) {
        j = k;
    }

    while ((j <= pq->size) && hit_lt(heap[j], node)) {
        heap[i] = heap[j];  /* shift up child */
        i = j;
        j = i << 1;
        k = j + 1;
        if ((k <= pq->size) && hit_lt(heap[k], heap[j])) {
            j = k;
        }
    }
    heap[i] = node;
}

static Hit *hit_pq_pop(PriorityQueue *pq)
{
    if (pq->size > 0) {
        Hit **heap = (Hit **)pq->heap;
        Hit *result = heap[1];    /* save first value */
        heap[1] = heap[pq->size]; /* move last to first */
        heap[pq->size] = NULL;
        pq->size--;
        hit_pq_down(pq);          /* adjust heap */
        return result;
    }
    else {
        return NULL;
    }
}

static void hit_pq_up(PriorityQueue *pq)
{
    Hit **heap = (Hit **)pq->heap;
    Hit *node;
    int i = pq->size;
    int j = i >> 1;
    node = heap[i];

    while ((j > 0) && hit_lt(node, heap[j])) {
        heap[i] = heap[j];
        i = j;
        j = j >> 1;
    }
    heap[i] = node;
}

static void hit_pq_insert(PriorityQueue *pq, Hit *hit) 
{
    if (pq->size < pq->capa) {
        Hit *new_hit = ALLOC(Hit);
        memcpy(new_hit, hit, sizeof(Hit));
        pq->size++;
        if (pq->size >= pq->mem_capa) {
            pq->mem_capa <<= 1;
            REALLOC_N(pq->heap, void *, pq->mem_capa);
        }
        pq->heap[pq->size] = new_hit;
        hit_pq_up(pq);
    }
    else if (pq->size > 0 && hit_lt((Hit *)pq->heap[1], hit)) {
        memcpy(pq->heap[1], hit, sizeof(Hit));
        hit_pq_down(pq);
    }
}

static void hit_pq_multi_insert(PriorityQueue *pq, Hit *hit) 
{
    hit_pq_insert(pq, hit);
    free(hit);
}

/***************************************************************************
 *
 * TopDocs
 *
 ***************************************************************************/

TopDocs *td_new(int total_hits, int size, Hit **hits, float max_score)
{
    TopDocs *td = ALLOC(TopDocs);
    td->total_hits = total_hits;
    td->size = size;
    td->hits = hits;
    td->max_score = max_score;
    return td;
}

void td_destroy(TopDocs *td)
{
    int i;

    for (i = 0; i < td->size; i++) {
        free(td->hits[i]);
    }
    free(td->hits);
    free(td);
}

char *td_to_s(TopDocs *td)
{
    int i;
    Hit *hit;
    char *buffer = strfmt("%d hits sorted by <score, doc_num>\n",
                          td->total_hits);
    for (i = 0; i < td->size; i++) {
        hit = td->hits[i];
        estrcat(buffer, strfmt("\t%d:%f\n", hit->doc, hit->score));
    }
    return buffer;
}

/***************************************************************************
 *
 * Weight
 *
 ***************************************************************************/

Query *w_get_query(Weight *self)
{
    return self->query;
}

float w_get_value(Weight *self)
{
    return self->value;
}

float w_sum_of_squared_weights(Weight *self)
{
    self->qweight = self->idf * self->query->boost;
    return self->qweight * self->qweight;   /* square it */
}

void w_normalize(Weight *self, float normalization_factor)
{
    self->qnorm = normalization_factor;
    self->qweight *= normalization_factor;  /* normalize query weight */
    self->value = self->qweight * self->idf;/* idf for document */
}

void w_destroy(Weight *self)
{
    q_deref(self->query);
    free(self);
}

Weight *w_create(size_t size, Query *query)
{
    Weight *self                    = (Weight *)ecalloc(size);
#ifdef DEBUG
    if (size < sizeof(Weight)) {
        RAISE(FERRET_ERROR, "size of weight <%d> should be at least <%d>",
              (int)size, (int)sizeof(Weight));
    }
#endif
    REF(query);
    self->query                     = query;
    self->get_query                 = &w_get_query;
    self->get_value                 = &w_get_value;
    self->normalize                 = &w_normalize;
    self->destroy                   = &w_destroy;
    self->sum_of_squared_weights    = &w_sum_of_squared_weights;
    return self;
}

/***************************************************************************
 *
 * Query
 *
 ***************************************************************************/

static const char *QUERY_NAMES[] = {
    "TermQuery",
    "MultiTermQuery",
    "BooleanQuery",
    "PhraseQuery",
    "ConstantScoreQuery",
    "FilteredQuery",
    "MatchAllQuery",
    "RangeQuery",
    "WildCardQuery",
    "FuzzyQuery",
    "PrefixQuery",
    "SpanTermQuery",
    "SpanMultiTermQuery",
    "SpanPrefixQuery",
    "SpanFirstQuery",
    "SpanOrQuery",
    "SpanNotQuery",
    "SpanNearQuery"
};

static const char *UNKNOWN_QUERY_NAME = "UnkownQuery";
    
const char *q_get_query_name(QueryType type) {
    if (type >= NELEMS(QUERY_NAMES)) {
        return UNKNOWN_QUERY_NAME;
    }
    else {
        return QUERY_NAMES[type];
    }
}

static Query *q_rewrite(Query *self, IndexReader *ir)
{
    (void)ir;
    self->ref_cnt++;
    return self;
}

static void q_extract_terms(Query *self, HashSet *terms)
{
    /* do nothing by default */
    (void)self;
    (void)terms;
}

Similarity *q_get_similarity_i(Query *self, Searcher *searcher)
{
    (void)self;
    return searcher->get_similarity(searcher);
}

void q_destroy_i(Query *self)
{
    free(self);
}

void q_deref(Query *self)
{  
    if (--(self->ref_cnt) == 0) {
        self->destroy_i(self);
    }
}

Weight *q_create_weight_unsup(Query *self, Searcher *searcher)
{
    (void)self;
    (void)searcher;
    RAISE(UNSUPPORTED_ERROR,
          "Create weight is unsupported for this type of query");
    return NULL;
}

Weight *q_weight(Query *self, Searcher *searcher)
{
    Query      *query   = searcher->rewrite(searcher, self);
    Weight     *weight  = query->create_weight_i(query, searcher);
    float       sum     = weight->sum_of_squared_weights(weight);
    Similarity *sim     = query->get_similarity(query, searcher);
    float       norm    = sim_query_norm(sim, sum);
    q_deref(query);

    weight->normalize(weight, norm);
    return self->weight = weight;
}

#define BQ(query) ((BooleanQuery *)(query))
Query *q_combine(Query **queries, int q_cnt)
{
    int i;
    Query *q, *ret_q;
    HashSet *uniques = hs_new((hash_ft)&q_hash, (eq_ft)&q_eq, NULL);

    for (i = 0; i < q_cnt; i++) {
        q = queries[i];
        if (q->type == BOOLEAN_QUERY) {
            int j;
            bool splittable = true;
            if (BQ(q)->coord_disabled == false) {
                splittable = false;
            }
            else {
                for (j = 0; j < BQ(q)->clause_cnt; j++) {
                    if (BQ(q)->clauses[j]->occur != BC_SHOULD) {
                        splittable = false;
                        break;
                    }
                }
            }
            if (splittable) {
                for (j = 0; j < BQ(q)->clause_cnt; j++) {
                    Query *sub_q = BQ(q)->clauses[j]->query;
                    hs_add(uniques, sub_q);
                }
            }
            else {
                hs_add(uniques, q);
            }
        }
        else {
            hs_add(uniques, q);
        }
    }
    if (uniques->size == 1) {
        ret_q = (Query *)uniques->first->elem; 
        REF(ret_q);
    }
    else {
        HashSetEntry *hse;
        ret_q = bq_new(true);
        for (hse = uniques->first; hse; hse = hse->next) {
            q = (Query *)hse->elem;
            bq_add_query(ret_q, q, BC_SHOULD);
        }
    }
    hs_destroy(uniques);

    return ret_q;
}

unsigned long q_hash(Query *self)
{
    return (self->hash(self) << 5) | self->type;
}

int q_eq(Query *self, Query *o)
{
    return (self == o)
        || ((self->type == o->type)
            && (self->boost == o->boost)
            && self->eq(self, o));
}

static MatchVector *q_get_matchv_i(Query *self, MatchVector *mv, TermVector *tv)
{
    /* be default we don't add any matches */
    (void)self; (void)tv;
    return mv;
}

Query *q_create(size_t size)
{
    Query *self = (Query *)ecalloc(size);
#ifdef DEBUG
    if (size < sizeof(Query)) {
        RAISE(FERRET_ERROR, "Size of a query <%d> should never be smaller than "
              "the size of a Query struct <%d>", (int)size, (int)sizeof(Query));
    }
#endif
    self->boost             = 1.0;
    self->rewrite           = &q_rewrite;
    self->get_similarity    = &q_get_similarity_i;
    self->extract_terms     = &q_extract_terms;
    self->get_matchv_i      = &q_get_matchv_i;
    self->weight            = NULL;
    self->ref_cnt           = 1;
    return self;
}

/***************************************************************************
 *
 * Scorer
 *
 ***************************************************************************/

void scorer_destroy_i(Scorer *scorer)
{
    free(scorer);
}

Scorer *scorer_create(size_t size, Similarity *similarity)
{
    Scorer *self        = (Scorer *)ecalloc(size);
#ifdef DEBUG
    if (size < sizeof(Scorer)) {
        RAISE(FERRET_ERROR, "size of scorer <%d> should be at least <%d>",
              (int)size, (int)sizeof(Scorer));
    }
#endif
    self->destroy       = &scorer_destroy_i;
    self->similarity    = similarity;
    return self;
}

bool scorer_less_than(void *p1, void *p2)
{
    Scorer *s1 = (Scorer *)p1;
    Scorer *s2 = (Scorer *)p2;
    return s1->score(s1) < s2->score(s2);
}

bool scorer_doc_less_than(const Scorer *s1, const Scorer *s2)
{
    return s1->doc < s2->doc;
}

int scorer_doc_cmp(const void *p1, const void *p2)
{
    return (*(Scorer **)p1)->doc - (*(Scorer **)p2)->doc;
}

/***************************************************************************
 *
 * Highlighter
 *
 ***************************************************************************/

/* ** MatchRange ** */
static int match_range_cmp(const void *p1, const void *p2)
{
    int diff = ((MatchRange *)p1)->start - ((MatchRange *)p2)->start;
    if (diff != 0) {
        return diff;
    }
    else {
        return ((MatchRange *)p2)->end - ((MatchRange *)p1)->end;
    }
}



/* ** MatchVector ** */
MatchVector *matchv_new()
{
    MatchVector *matchv = ALLOC(MatchVector);

    matchv->size = 0;
    matchv->capa = MATCH_VECTOR_INIT_CAPA;
    matchv->matches = ALLOC_N(MatchRange, MATCH_VECTOR_INIT_CAPA);

    return matchv;
}

MatchVector *matchv_add(MatchVector *self, int start, int end)
{
    if (self->size >= self->capa) {
        self->capa <<= 1;
        REALLOC_N(self->matches, MatchRange, self->capa);
    }
    self->matches[self->size].start = start;
    self->matches[self->size].end   = end;
    self->matches[self->size].score = 1.0;
    self->size++;
    return self;
}

MatchVector *matchv_sort(MatchVector *self)
{
    qsort(self->matches, self->size, sizeof(MatchRange), &match_range_cmp);
    return self;
}

MatchVector *matchv_compact(MatchVector *self)
{
    int left, right;
    matchv_sort(self);
    for (right = left = 0; right < self->size; right++) {
        /* Note the end + 1. This compacts a range 3:5 and 6:8 inleft 3:8 */
        if (self->matches[right].start > self->matches[left].end + 1) {
            left++;
            self->matches[left].start = self->matches[right].start;
            self->matches[left].end = self->matches[right].end;
            self->matches[left].score = self->matches[right].score;
        }
        else if (self->matches[right].end > self->matches[left].end) {
            self->matches[left].end = self->matches[right].end;
        }
        else {
            self->matches[left].score += self->matches[right].score;
        }
    }
    self->size = left + 1;
    return self;
}

MatchVector *matchv_compact_with_breaks(MatchVector *self)
{
    int left, right;
    matchv_sort(self);
    for (right = left = 0; right < self->size; right++) {
        /* Note: no end + 1. Unlike above won't compact ranges 3:5 and 6:8 */
        if (self->matches[right].start > self->matches[left].end) {
            left++;
            self->matches[left].start = self->matches[right].start;
            self->matches[left].end = self->matches[right].end;
            self->matches[left].score = self->matches[right].score;
        }
        else if (self->matches[right].end > self->matches[left].end) {
            self->matches[left].end = self->matches[right].end;
            self->matches[left].score += self->matches[right].score;
        }
        else if (right > left) {
            self->matches[left].score += self->matches[right].score;
        }
    }
    self->size = left + 1;
    return self;
}


static MatchVector *matchv_set_offsets(MatchVector *mv, Offset *offsets)
{
    int i;
    for (i = 0; i < mv->size; i++) {
        mv->matches[i].start_offset = offsets[mv->matches[i].start].start;
        mv->matches[i].end_offset = offsets[mv->matches[i].end].end;
    }
    return mv;
}

void matchv_destroy(MatchVector *self)
{
    free(self->matches);
    free(self);
}

/***************************************************************************
 *
 * Searcher
 *
 ***************************************************************************/

MatchVector *searcher_get_match_vector(Searcher *self,
                                       Query *query,
                                       const int doc_num,
                                       Symbol field)
{
    MatchVector *mv = matchv_new();
    bool rewrite = query->get_matchv_i == q_get_matchv_i;
    TermVector *tv = self->get_term_vector(self, doc_num, field);
    if (rewrite) {
        query = self->rewrite(self, query);
    }
    if (tv && tv->term_cnt > 0 && tv->terms[0].positions != NULL) {
        mv = query->get_matchv_i(query, mv, tv);
        tv_destroy(tv);
    }
    if (rewrite) {
        q_deref(query);
    }
    return mv;
}

typedef struct Excerpt
{
    int start;
    int end;
    int start_pos;
    int end_pos;
    int start_offset;
    int end_offset;
    double score;
} Excerpt;

/*
static int excerpt_cmp(const void *p1, const void *p2)
{
    double score1 = (*((Excerpt **)p1))->score;
    double score2 = (*((Excerpt **)p2))->score;
    if (score1 > score2) return 1;
    if (score1 < score2) return -1;
    return 0;
}
*/

static int excerpt_start_cmp(const void *p1, const void *p2)
{
    return (*((Excerpt **)p1))->start - (*((Excerpt **)p2))->start;
}

static int excerpt_lt(Excerpt *e1, Excerpt *e2)
{
    return e1->score > e2->score; /* want the highest score at top */
}

static Excerpt *excerpt_new(int start, int end, double score)
{
    Excerpt *excerpt = ALLOC_AND_ZERO(Excerpt);
    excerpt->start = start;
    excerpt->end = end;
    excerpt->score = score;
    return excerpt;
}

static Excerpt *excerpt_recalc_score(Excerpt *e, MatchVector *mv)
{
    int i;
    double score = 0.0;
    for (i = e->start; i <= e->end; i++) {
        score += mv->matches[i].score;
    }
    e->score = score;
    return e;
}

/* expand an excerpt to it's largest possible size */
static Excerpt *excerpt_expand(Excerpt *e, const int len, TermVector *tv)
{
    Offset *offsets = tv->offsets;
    int offset_cnt = tv->offset_cnt;
    bool did_expansion = true;
    int i;
    /* fill in skipped offsets */
    for (i = 1; i < offset_cnt; i++) {
        if (offsets[i].start == 0) {
            offsets[i].start = offsets[i-1].start;
        }
        if (offsets[i].end == 0) {
            offsets[i].end = offsets[i-1].end;
        }
    }
    
    while (did_expansion) {
        did_expansion = false;
        if (e->start_pos > 0
            && (e->end_offset - offsets[e->start_pos - 1].start) < len) {
            e->start_pos--;
            e->start_offset = offsets[e->start_pos].start;
            did_expansion = true;
        }
        if (e->end_pos < (offset_cnt - 1)
            && (offsets[e->end_pos + 1].end - e->start_offset) < len) {
            e->end_pos++;
            e->end_offset = offsets[e->end_pos].end;
            did_expansion = true;
        }
    }
    return e;
}

static char *excerpt_get_str(Excerpt *e, MatchVector *mv,
                             LazyDocField *lazy_df,
                             const char *pre_tag, 
                             const char *post_tag,
                             const char *ellipsis)
{
    int i, len;
    int last_offset = e->start_offset;
    const int num_matches = e->end - e->start + 1;
    const int pre_tag_len = (int)strlen(pre_tag);
    const int post_tag_len = (int)strlen(post_tag);
    const int ellipsis_len = (int)strlen(ellipsis);
    char *excerpt_str = ALLOC_N(char,
                                10 + e->end_offset - e->start_offset
                                + (num_matches * (pre_tag_len + post_tag_len))
                                + (2 * ellipsis_len));
    char *e_ptr = excerpt_str;
    if (e->start_offset > 0) {
        memcpy(e_ptr, ellipsis, ellipsis_len);
        e_ptr += ellipsis_len;
    }
    for (i = e->start; i <= e->end; i++) {
        MatchRange *mr = mv->matches + i;
        len = mr->start_offset - last_offset;
        if (len) {
            lazy_df_get_bytes(lazy_df, e_ptr, last_offset, len);
            e_ptr += len;
        }
        memcpy(e_ptr, pre_tag, pre_tag_len);
        e_ptr += pre_tag_len;
        len = mr->end_offset - mr->start_offset;
        if (len) {
            lazy_df_get_bytes(lazy_df, e_ptr, mr->start_offset, len);
            e_ptr += len;
        }
        memcpy(e_ptr, post_tag, post_tag_len);
        e_ptr += post_tag_len;
        last_offset = mr->end_offset;
    }
    if ((lazy_df->len - e->end_offset) <= ellipsis_len) {
        /* no point using ellipsis if it takes up more space */
        e->end_offset = lazy_df->len;
    }
    len = e->end_offset - last_offset;
    if (len) {
        lazy_df_get_bytes(lazy_df, e_ptr, last_offset, len);
        e_ptr += len;
    }
    if (e->end_offset < lazy_df->len) {
        memcpy(e_ptr, ellipsis, ellipsis_len);
        e_ptr += ellipsis_len;
    }
    *e_ptr = '\0';
    return excerpt_str;
}

static char *highlight_field(MatchVector *mv,
                             LazyDocField *lazy_df,
                             TermVector *tv,
                             const char *pre_tag, 
                             const char *post_tag)
{
    const int pre_len = (int)strlen(pre_tag);
    const int post_len = (int)strlen(post_tag);
    char *excerpt_str =
        ALLOC_N(char, 10 + lazy_df->len + (mv->size * (pre_len + post_len)));
    if (mv->size > 0) {
        int last_offset = 0;
        int i, len;
        char *e_ptr = excerpt_str;
        matchv_compact_with_breaks(mv);
        matchv_set_offsets(mv, tv->offsets);
        for (i = 0; i < mv->size; i++) {
            MatchRange *mr = mv->matches + i;
            len = mr->start_offset - last_offset;
            if (len) {
                lazy_df_get_bytes(lazy_df, e_ptr, last_offset, len);
                e_ptr += len;
            }
            memcpy(e_ptr, pre_tag, pre_len);
            e_ptr += pre_len;
            len = mr->end_offset - mr->start_offset;
            if (len) {
                lazy_df_get_bytes(lazy_df, e_ptr, mr->start_offset, len);
                e_ptr += len;
            }
            memcpy(e_ptr, post_tag, post_len);
            e_ptr += post_len;
            last_offset = mr->end_offset;
        }
        len = lazy_df->len - last_offset;
        if (len) {
            lazy_df_get_bytes(lazy_df, e_ptr, last_offset, len);
            e_ptr += len;
        }
        *e_ptr = '\0';
    }
    else {
        lazy_df_get_bytes(lazy_df, excerpt_str, 0, lazy_df->len);
        excerpt_str[lazy_df->len] = '\0';
    }
    return excerpt_str;
}

char **searcher_highlight(Searcher *self,
                          Query *query,
                          const int doc_num,
                          Symbol field,
                          const int excerpt_len,
                          const int num_excerpts,
                          const char *pre_tag,
                          const char *post_tag,
                          const char *ellipsis)
{
    char **excerpt_strs = NULL;
    TermVector *tv = self->get_term_vector(self, doc_num, field);
    LazyDoc *lazy_doc = self->get_lazy_doc(self, doc_num);
    LazyDocField *lazy_df = NULL;
    if (lazy_doc) {
        lazy_df = lazy_doc_get(lazy_doc, field);
    }
    if (tv && lazy_df && tv->term_cnt > 0 && tv->terms[0].positions != NULL
        && tv->offsets != NULL) {
        MatchVector *mv;
        query = self->rewrite(self, query);
        mv = query->get_matchv_i(query, matchv_new(), tv);
        q_deref(query);
        if (lazy_df->len < (excerpt_len * num_excerpts)) {
            excerpt_strs = ary_new_type_capa(char *, 1);
            ary_push(excerpt_strs,
                     highlight_field(mv, lazy_df, tv, pre_tag, post_tag));
        }
        else if (mv->size > 0) {
            Excerpt **excerpts = ALLOC_AND_ZERO_N(Excerpt *, num_excerpts);
            int e_start, e_end, i, j;
            MatchRange *matches = mv->matches;
            double running_score = 0.0;
            Offset *offsets = tv->offsets;
            PriorityQueue *excerpt_pq;

            matchv_compact_with_breaks(mv);
            matchv_set_offsets(mv, offsets);
            excerpt_pq = pq_new(mv->size, (lt_ft)&excerpt_lt, &free);
            /* add all possible excerpts to the priority queue */
            
            for (e_start = e_end = 0; e_start < mv->size; e_start++) {
                const int start_offset = matches[e_start].start_offset;
                if (e_start > e_end) {
                    running_score = 0.0;
                    e_end = e_start;
                }
                while (e_end < mv->size && (matches[e_end].end_offset
                                             <= start_offset + excerpt_len)) {
                    running_score += matches[e_end].score;
                    e_end++;
                }
                pq_push(excerpt_pq,
                        excerpt_new(e_start, e_end - 1, running_score));
                /* - 0.1 so that earlier matches take priority */
                running_score -= matches[e_start].score;
            }

            for (i = 0; i < num_excerpts && excerpt_pq->size > 0; i++) {
                excerpts[i] = (Excerpt *)pq_pop(excerpt_pq);
                if (i < num_excerpts - 1) {
                    /* set match ranges alread included to 0 */
                    Excerpt *e = excerpts[i];
                    for (j = e->start; j <= e->end; j++) {
                        matches[j].score = 0.0;
                    }
                    e = NULL;
                    while (e != (Excerpt *)pq_top(excerpt_pq)) {
                        e = (Excerpt *)pq_top(excerpt_pq);
                        excerpt_recalc_score(e, mv);
                        pq_down(excerpt_pq);
                    }
                }
            }

            qsort(excerpts, i, sizeof(Excerpt *), &excerpt_start_cmp);
            for (j = 0; j < i; j++) {
                Excerpt *e = excerpts[j];
                e->start_pos = matches[e->start].start;
                e->end_pos = matches[e->end].end;
                e->start_offset = offsets[e->start_pos].start;
                e->end_offset = offsets[e->end_pos].end;
            }

            if (i < num_excerpts) {
                const int diff = num_excerpts - i;
                memmove(excerpts + (diff), excerpts,
                        i * sizeof(Excerpt *));
                for (j = 0; j < diff; j++) {
                    /* these new excerpts will grow into one long excerpt at
                     * the start */
                    excerpts[j] = ALLOC_AND_ZERO(Excerpt);
                    excerpts[j]->end = -1;
                }
            }

            excerpt_strs = ary_new_type_capa(char *, num_excerpts);
            /* merge excerpts where possible */
            for (i = 0; i < num_excerpts;) {
                Excerpt *ei = excerpts[i];
                int merged = 1; /* 1 means a single excerpt, ie no merges */
                for (j = i + 1; j < num_excerpts; j++) {
                    Excerpt *ej = excerpts[j];
                    if ((ej->end_offset - ei->start_offset)
                        < (j - i + 1) * excerpt_len) {
                        ei->end = ej->end;
                        ei->end_pos = ej->end_pos;
                        ei->end_offset = ej->end_offset;
                        merged = j - i + 1;
                    }
                }
                excerpt_expand(ei, merged * excerpt_len, tv);
                ary_push(excerpt_strs,
                         excerpt_get_str(ei, mv, lazy_df,
                                         pre_tag, post_tag, ellipsis));
                i += merged;
            }
            for (i = 0; i < num_excerpts; i++) {
                free(excerpts[i]);
            }
            free(excerpts);
            pq_destroy(excerpt_pq);
        }
        matchv_destroy(mv);
    }
    if (tv) tv_destroy(tv);
    if (lazy_doc) lazy_doc_close(lazy_doc);
    return excerpt_strs;
}

static Weight *sea_create_weight(Searcher *self, Query *query)
{
    return q_weight(query, self);
}

static void sea_check_args(int num_docs, int first_doc)
{
    if (num_docs <= 0) {
        RAISE(ARG_ERROR, ":num_docs was set to %d but should be greater "
              "than 0 : %d <= 0", num_docs, num_docs);
    }

    if (first_doc < 0) {
        RAISE(ARG_ERROR, ":first_doc was set to %d but should be greater "
              "than or equal to 0 : %d < 0", first_doc, first_doc);
    }
}

static Similarity *sea_get_similarity(Searcher *self)
{
    return self->similarity;
}

/***************************************************************************
 *
 * IndexSearcher
 *
 ***************************************************************************/

#define ISEA(searcher) ((IndexSearcher *)(searcher))

int isea_doc_freq(Searcher *self, Symbol field, const char *term)
{
    return ir_doc_freq(ISEA(self)->ir, field, term);
}

static Document *isea_get_doc(Searcher *self, int doc_num)
{
    IndexReader *ir = ISEA(self)->ir;
    return ir->get_doc(ir, doc_num);
}

static LazyDoc *isea_get_lazy_doc(Searcher *self, int doc_num)
{
    IndexReader *ir = ISEA(self)->ir;
    return ir->get_lazy_doc(ir, doc_num);
}

static int isea_max_doc(Searcher *self)
{
    IndexReader *ir = ISEA(self)->ir;
    return ir->max_doc(ir);
}

#define IS_FILTERED(bits, post_filter, scorer, searcher) \
((bits && !bv_get(bits, scorer->doc))\
 || (post_filter \
     && !(filter_factor = \
          post_filter->filter_func(scorer->doc, scorer->score(scorer),\
                                   searcher, post_filter->arg))))

static TopDocs *isea_search_w(Searcher *self,
                              Weight *weight,
                              int first_doc,
                              int num_docs,
                              Filter *filter,
                              Sort *sort,
                              PostFilter *post_filter,
                              bool load_fields)
{
    int max_size = num_docs + (num_docs == INT_MAX ? 0 : first_doc);
    int i;
    Scorer *scorer;
    Hit **score_docs = NULL;
    Hit hit;
    int total_hits = 0;
    float score, max_score = 0.0;
    float filter_factor = 1.0;
    BitVector *bits = (filter
                       ? filt_get_bv(filter, ISEA(self)->ir)
                       : NULL);
    Hit *(*hq_pop)(PriorityQueue *pq);
    void (*hq_insert)(PriorityQueue *pq, Hit *hit);
    void (*hq_destroy)(PriorityQueue *self);
    PriorityQueue *hq;

    sea_check_args(num_docs, first_doc);

    scorer = weight->scorer(weight, ISEA(self)->ir);
    if (!scorer || 0 == ISEA(self)->ir->num_docs(ISEA(self)->ir)) {
        if (scorer) scorer->destroy(scorer);
        return td_new(0, 0, NULL, 0.0);
    }

    if (sort) {
        hq = fshq_pq_new(max_size, sort, ISEA(self)->ir);
        hq_insert = &fshq_pq_insert;
        hq_destroy = &fshq_pq_destroy;
        if (load_fields) {
            hq_pop = &fshq_pq_pop_fd;
        }
        else {
            hq_pop = &fshq_pq_pop;
        }
    }
    else {
        hq = pq_new(max_size, (lt_ft)&hit_less_than, &free);
        hq_pop = &hit_pq_pop;
        hq_insert = &hit_pq_insert;
        hq_destroy = &pq_destroy;
    }

    while (scorer->next(scorer)) {
        if (bits && !bv_get(bits, scorer->doc)) continue;
        score = scorer->score(scorer);
        if (post_filter &&
            !(filter_factor = post_filter->filter_func(scorer->doc,
                                                       score,
                                                       self,
                                                       post_filter->arg))) {
            continue;
        }
        total_hits++;
        if (filter_factor < 1.0) score *= filter_factor;
        if (score > max_score) max_score = score;
        hit.doc = scorer->doc; hit.score = score;
        hq_insert(hq, &hit);
    }
    scorer->destroy(scorer);

    if (hq->size > first_doc) {
        if ((hq->size - first_doc) < num_docs) {
            num_docs = hq->size - first_doc;
        }
        score_docs = ALLOC_N(Hit *, num_docs);
        for (i = num_docs - 1; i >= 0; i--) {
            score_docs[i] = hq_pop(hq);
            /*
            printf("score_docs[i][%d] = [%ld] => %d-->%f\n", i,
                   score_docs[i], score_docs[i]->doc, score_docs[i]->score);
            */
        }
    }
    else {
        num_docs = 0;
    }
    pq_clear(hq);
    hq_destroy(hq);

    return td_new(total_hits, num_docs, score_docs, max_score);
}

static TopDocs *isea_search(Searcher *self,
                            Query *query,
                            int first_doc,
                            int num_docs,
                            Filter *filter,
                            Sort *sort,
                            PostFilter *post_filter,
                            bool load_fields)
{
    TopDocs *td;
    Weight *weight = q_weight(query, self);
    td = isea_search_w(self, weight, first_doc, num_docs, filter,
                         sort, post_filter, load_fields);
    weight->destroy(weight);
    return td;
}

static void isea_search_each_w(Searcher *self, Weight *weight, Filter *filter,
                               PostFilter *post_filter,
                               void (*fn)(Searcher *, int, float, void *),
                               void *arg)
{
    Scorer *scorer;
    float filter_factor = 1.0;
    BitVector *bits = (filter
                       ? filt_get_bv(filter, ISEA(self)->ir)
                       : NULL);

    scorer = weight->scorer(weight, ISEA(self)->ir);
    if (!scorer) {
        return;
    }

    while (scorer->next(scorer)) {
        float score;
        if (bits && !bv_get(bits, scorer->doc)) continue;
        score = scorer->score(scorer);
        if (post_filter &&
            !(filter_factor = post_filter->filter_func(scorer->doc,
                                                       score,
                                                       self,
                                                       post_filter->arg))) {
            continue;
        }
        fn(self, scorer->doc, filter_factor * score, arg);
    }
    scorer->destroy(scorer);
}

static void isea_search_each(Searcher *self, Query *query, Filter *filter,
                             PostFilter *post_filter,
                             void (*fn)(Searcher *, int, float, void *),
                             void *arg)
{
    Weight *weight = q_weight(query, self);
    isea_search_each_w(self, weight, filter, post_filter, fn, arg);
    weight->destroy(weight);
}

/*
 * Scan the index for all documents that match a query and write the results
 * to a buffer. It will stop scanning once the limit is reached and it starts
 * scanning from offset_docnum. 
 *
 * Note: Unlike the offset_docnum in other search methods, this offset_docnum
 * refers to document number and not hit.
 */
static int isea_search_unscored_w(Searcher *self,
                                  Weight *weight,
                                  int *buf,
                                  int limit,
                                  int offset_docnum)
{
    int count = 0;
    Scorer *scorer = weight->scorer(weight, ISEA(self)->ir);
    if (scorer) {
        if (scorer->skip_to(scorer, offset_docnum)) {
            do {
                buf[count++] = scorer->doc;
            } while (count < limit && scorer->next(scorer));
        }
        scorer->destroy(scorer);
    }
    return count;
}

static int isea_search_unscored(Searcher *self,
                                Query *query,
                                int *buf,
                                int limit,
                                int offset_docnum)
{
    int count;
    Weight *weight = q_weight(query, self);
    count = isea_search_unscored_w(self, weight, buf, limit, offset_docnum);
    weight->destroy(weight);
    return count;
}

static Query *isea_rewrite(Searcher *self, Query *original)
{
    int q_is_destroyed = false;
    Query *query = original;
    Query *rewritten_query = query->rewrite(query, ISEA(self)->ir);
    while (q_is_destroyed || (query != rewritten_query)) {
        query = rewritten_query;
        rewritten_query = query->rewrite(query, ISEA(self)->ir);
        q_is_destroyed = (query->ref_cnt <= 1);
        q_deref(query); /* destroy intermediate queries */
    }
    return query;
}

static Explanation *isea_explain(Searcher *self,
                                 Query *query,
                                 int doc_num)
{
    Weight *weight = q_weight(query, self);
    Explanation *e =  weight->explain(weight, ISEA(self)->ir, doc_num);
    weight->destroy(weight);
    return e;
}

static Explanation *isea_explain_w(Searcher *self, Weight *w, int doc_num)
{
    return w->explain(w, ISEA(self)->ir, doc_num);
}

static TermVector *isea_get_term_vector(Searcher *self,
                                          const int doc_num,
                                          Symbol field)
{
    IndexReader *ir = ISEA(self)->ir;
    return ir->term_vector(ir, doc_num, field);
}

static void isea_close(Searcher *self)
{
    if (ISEA(self)->ir && ISEA(self)->close_ir) {
        ir_close(ISEA(self)->ir);
    }
    free(self);
}

Searcher *isea_new(IndexReader *ir)
{
    Searcher *self          = (Searcher *)ALLOC(IndexSearcher);

    ISEA(self)->ir          = ir;
    ISEA(self)->close_ir    = true;

    self->similarity        = sim_create_default();
    self->doc_freq          = &isea_doc_freq;
    self->get_doc           = &isea_get_doc;
    self->get_lazy_doc      = &isea_get_lazy_doc;
    self->max_doc           = &isea_max_doc;
    self->create_weight     = &sea_create_weight;
    self->search            = &isea_search;
    self->search_w          = &isea_search_w;
    self->search_each       = &isea_search_each;
    self->search_each_w     = &isea_search_each_w;
    self->search_unscored   = &isea_search_unscored;
    self->search_unscored_w = &isea_search_unscored_w;
    self->rewrite           = &isea_rewrite;
    self->explain           = &isea_explain;
    self->explain_w         = &isea_explain_w;
    self->get_term_vector   = &isea_get_term_vector;
    self->get_similarity    = &sea_get_similarity;
    self->close             = &isea_close;

    return self;
}

/***************************************************************************
 *
 * CachedDFSearcher
 *
 ***************************************************************************/

#define CDFSEA(searcher) ((CachedDFSearcher *)(searcher))
typedef struct CachedDFSearcher
{
    Searcher super;
    Hash     *df_map;
    int      max_doc;
} CachedDFSearcher;

static int cdfsea_doc_freq(Searcher *self, Symbol field, const char *text)
{
    Term term;
    int *df;
    term.field = field;
    term.text = (char *)text;
    df = (int *)h_get(CDFSEA(self)->df_map, &term);
    return df ? *df : 0;
}

static Document *cdfsea_get_doc(Searcher *self, int doc_num)
{
    (void)self; (void)doc_num;
    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
    return NULL;
}

static int cdfsea_max_doc(Searcher *self)
{
    (void)self;
    return CDFSEA(self)->max_doc;
}

static Weight *cdfsea_create_weight(Searcher *self, Query *query)
{
    (void)self; (void)query;
    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
    return NULL;
}

static TopDocs *cdfsea_search_w(Searcher *self, Weight *w, int fd, int nd,
                                Filter *f, Sort *s, PostFilter *pf, bool load)
{
    (void)self; (void)w; (void)fd; (void)nd;
    (void)f; (void)s; (void)pf; (void)load;
    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
    return NULL;
}

static TopDocs *cdfsea_search(Searcher *self, Query *q, int fd, int nd,
                              Filter *f, Sort *s, PostFilter *pf, bool load)
{
    (void)self; (void)q; (void)fd; (void)nd;
    (void)f; (void)s; (void)pf; (void)load;
    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
    return NULL;
}

static void cdfsea_search_each(Searcher *self, Query *query, Filter *filter,
                               PostFilter *pf, 
                               void (*fn)(Searcher *, int, float, void *),
                               void *arg)
{
    (void)self; (void)query; (void)filter; (void)pf; (void)fn; (void)arg;
    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
}

static void cdfsea_search_each_w(Searcher *self, Weight *w, Filter *filter,
                                 PostFilter *pf, 
                                 void (*fn)(Searcher *, int, float, void *),
                                 void *arg)
{
    (void)self; (void)w; (void)filter; (void)pf; (void)fn; (void)arg;
    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
}

static Query *cdfsea_rewrite(Searcher *self, Query *original)
{
    (void)self;
    original->ref_cnt++;
    return original;
}

static Explanation *cdfsea_explain(Searcher *self, Query *query, int doc_num)
{
    (void)self; (void)query; (void)doc_num;
    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
    return NULL;
}

static Explanation *cdfsea_explain_w(Searcher *self, Weight *w, int doc_num)
{
    (void)self; (void)w; (void)doc_num;
    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
    return NULL;
}

static TermVector *cdfsea_get_term_vector(Searcher *self, const int doc_num,
                                          Symbol field)
{
    (void)self; (void)doc_num; (void)field;
    RAISE(UNSUPPORTED_ERROR, "%s", UNSUPPORTED_ERROR_MSG);
    return NULL;
}

static Similarity *cdfsea_get_similarity(Searcher *self)
{
    return self->similarity;
}

static void cdfsea_close(Searcher *self)
{
    h_destroy(CDFSEA(self)->df_map);
    free(self);
}

static Searcher *cdfsea_new(Hash *df_map, int max_doc)
{
    Searcher *self          = (Searcher *)ALLOC(CachedDFSearcher);

    CDFSEA(self)->df_map    = df_map;
    CDFSEA(self)->max_doc   = max_doc;

    self->similarity        = sim_create_default();
    self->doc_freq          = &cdfsea_doc_freq;
    self->get_doc           = &cdfsea_get_doc;
    self->max_doc           = &cdfsea_max_doc;
    self->create_weight     = &cdfsea_create_weight;
    self->search            = &cdfsea_search;
    self->search_w          = &cdfsea_search_w;
    self->search_each       = &cdfsea_search_each;
    self->search_each_w     = &cdfsea_search_each_w;
    self->rewrite           = &cdfsea_rewrite;
    self->explain           = &cdfsea_explain;
    self->explain_w         = &cdfsea_explain_w;
    self->get_term_vector   = &cdfsea_get_term_vector;
    self->get_similarity    = &cdfsea_get_similarity;
    self->close             = &cdfsea_close;
    return self;
}

/***************************************************************************
 *
 * MultiSearcher
 *
 ***************************************************************************/

#define MSEA(searcher) ((MultiSearcher *)(searcher))
static INLINE int msea_get_searcher_index(Searcher *self, int n)
{
    MultiSearcher *msea = MSEA(self);
    int lo = 0;                 /* search starts array */
    int hi = msea->s_cnt - 1;   /* for 1st element < n, return its index */
    int mid, mid_val;

    while (hi >= lo) {
        mid = (lo + hi) >> 1;
        mid_val = msea->starts[mid];
        if (n < mid_val) {
            hi = mid - 1;
        }
        else if (n > mid_val) {
            lo = mid + 1;
        }
        else {                  /* found a match */
            while (((mid+1) < msea->s_cnt)
                   && (msea->starts[mid+1] == mid_val)) {
                mid++;          /* scan to last match */
            }
            return mid;
        }
    }
    return hi;
}

static int msea_doc_freq(Searcher *self, Symbol field, const char *term)
{
    int i;
    int doc_freq = 0;
    MultiSearcher *msea = MSEA(self);
    for (i = 0; i < msea->s_cnt; i++) {
        Searcher *s = msea->searchers[i];
        doc_freq += s->doc_freq(s, field, term);
    }

    return doc_freq;
}

static Document *msea_get_doc(Searcher *self, int doc_num)
{
    MultiSearcher *msea = MSEA(self);
    int i = msea_get_searcher_index(self, doc_num);
    Searcher *s = msea->searchers[i];
    return s->get_doc(s, doc_num - msea->starts[i]);
}

static LazyDoc *msea_get_lazy_doc(Searcher *self, int doc_num)
{
    MultiSearcher *msea = MSEA(self);
    int i = msea_get_searcher_index(self, doc_num);
    Searcher *s = msea->searchers[i];
    return s->get_lazy_doc(s, doc_num - msea->starts[i]);
}

static int msea_max_doc(Searcher *self)
{
    return MSEA(self)->max_doc;
}

static int *msea_get_doc_freqs(Searcher *self, HashSet *terms)
{
    int i;
    HashSetEntry *hse;
    int *doc_freqs = ALLOC_N(int, terms->size);
    for (i = 0, hse = terms->first; hse; ++i, hse = hse->next) {
        Term *t = (Term *)hse->elem;
        doc_freqs[i] = msea_doc_freq(self, t->field, t->text);
    }
    return doc_freqs;
}

static Weight *msea_create_weight(Searcher *self, Query *query)
{
    int i, *doc_freqs;
    Searcher *cdfsea;
    Weight *w;
    Hash *df_map = h_new((hash_ft)&term_hash,
                         (eq_ft)&term_eq,
                         (free_ft)term_destroy,
                         free);
    Query *rewritten_query = self->rewrite(self, query);
    /* terms get copied directly to df_map so no need to free here */
    HashSet *terms = hs_new((hash_ft)&term_hash,
                            (eq_ft)&term_eq,
                            (free_ft)NULL);
    HashSetEntry *hse;

    rewritten_query->extract_terms(rewritten_query, terms);
    doc_freqs = msea_get_doc_freqs(self, terms);

    for (hse = terms->first, i = 0; hse; ++i, hse = hse->next) {
        h_set(df_map, hse->elem, imalloc(doc_freqs[i])); 
    }
    hs_destroy(terms);
    free(doc_freqs);

    cdfsea = cdfsea_new(df_map, MSEA(self)->max_doc);

    w = q_weight(rewritten_query, cdfsea);
    q_deref(rewritten_query);
    cdfsea->close(cdfsea);

    return w;
}

struct MultiSearchEachArg {
    int start;
    void *arg;
    void (*fn)(Searcher *, int, float, void *);
};

static void msea_search_each_i(Searcher *self, int doc_num, float score, void *arg)
{
    struct MultiSearchEachArg *mse_arg = (struct MultiSearchEachArg *)arg;

    mse_arg->fn(self, doc_num + mse_arg->start, score, mse_arg->arg);
}

static void msea_search_each_w(Searcher *self, Weight *w, Filter *filter,
                               PostFilter *post_filter,
                               void (*fn)(Searcher *, int, float, void *),
                               void *arg)
{
    int i;
    struct MultiSearchEachArg mse_arg;
    MultiSearcher *msea = MSEA(self);
    Searcher *s;

    mse_arg.fn = fn;
    mse_arg.arg = arg;
    for (i = 0; i < msea->s_cnt; i++) {
        s = msea->searchers[i];
        mse_arg.start = msea->starts[i];
        s->search_each_w(s, w, filter, post_filter,
                         &msea_search_each_i, &mse_arg);
    }
}

static void msea_search_each(Searcher *self, Query *query, Filter *filter,
                             PostFilter *post_filter,
                             void (*fn)(Searcher *, int, float, void *),
                             void *arg)
{
    Weight *weight = q_weight(query, self);
    msea_search_each_w(self, weight, filter, post_filter, fn, arg);
    weight->destroy(weight);
}

static int msea_search_unscored_w(Searcher *self,
                                  Weight *w,
                                  int *buf,
                                  int limit,
                                  int offset_docnum)
{
    int i, count = 0;
    MultiSearcher *msea = MSEA(self);

    for (i = 0; count < limit && i < msea->s_cnt; i++) {
        /* if offset_docnum falls in this or previous indexes */
        if (offset_docnum < msea->starts[i+1]) {
            Searcher *searcher = msea->searchers[i];
            const int index_offset = msea->starts[i];
            int current_limit = limit - count;
            /* if offset_docnum occurs in the current index then adjust,
             * otherwise set it to zero as it occurred in a previous index */
            int current_offset_docnum = offset_docnum > index_offset
                ? offset_docnum - index_offset
                : 0;

            /* record current count as we'll need to update docnums by the
             * index's offset */
            int j = count;
            count += searcher->search_unscored_w(searcher, w, buf + count,
                                                 current_limit,
                                                 current_offset_docnum);
            /* update doc nums with the current index's offsets */
            for (; j < count; j++) {
                buf[j] += index_offset;
            }
        }
    }
    return count;
}

static int msea_search_unscored(Searcher *self,
                                Query *query,
                                int *buf,
                                int limit,
                                int offset_docnum)
{
    int count;
    Weight *weight = q_weight(query, self);
    count = msea_search_unscored_w(self, weight, buf, limit, offset_docnum);
    weight->destroy(weight);
    return count;
}

struct MultiSearchArg {
    int total_hits, max_size;
    PriorityQueue *hq;
    void (*hq_insert)(PriorityQueue *pq, Hit *hit);
};

/*
 * FIXME Not used anywhere. Is it needed?
static void msea_search_i(Searcher *self, int doc_num, float score, void *arg)
{
    struct MultiSearchArg *ms_arg = (struct MultiSearchArg *)arg;
    Hit hit;
    (void)self;

    ms_arg->total_hits++;
    hit.doc = doc_num;
    hit.score = score;
    ms_arg->hq_insert(ms_arg->hq, &hit);
}
*/

static TopDocs *msea_search_w(Searcher *self,
                              Weight *weight,
                              int first_doc,
                              int num_docs,
                              Filter *filter,
                              Sort *sort,
                              PostFilter *post_filter,
                              bool load_fields)
{
    int max_size = num_docs + (num_docs == INT_MAX ? 0 : first_doc);
    int i;
    int total_hits = 0;
    Hit **score_docs = NULL;
    Hit *(*hq_pop)(PriorityQueue *pq);
    void (*hq_insert)(PriorityQueue *pq, Hit *hit);
    PriorityQueue *hq;
    float max_score = 0.0;
    (void)load_fields; /* does it automatically */

    sea_check_args(num_docs, first_doc);

    if (sort) {
        hq = pq_new(max_size, (lt_ft)fdshq_lt, &free);
        hq_insert = (void (*)(PriorityQueue *pq, Hit *hit))&pq_insert;
        hq_pop = (Hit *(*)(PriorityQueue *pq))&pq_pop;
    }
    else {
        hq = pq_new(max_size, (lt_ft)&hit_less_than, &free);
        hq_insert = &hit_pq_multi_insert;
        hq_pop = &hit_pq_pop;
    }

    /*if (sort) printf("sort = %s\n", sort_to_s(sort)); */
    for (i = 0; i < MSEA(self)->s_cnt; i++) {
        Searcher *s = MSEA(self)->searchers[i];
        TopDocs *td = s->search_w(s, weight, 0, max_size,
                                  filter, sort, post_filter, true);
        /*if (sort) printf("sort = %s\n", sort_to_s(sort)); */
        if (td->size > 0) {
            /*printf("td->size = %d %d\n", td->size, num_docs); */
            int j;
            int start = MSEA(self)->starts[i];
            for (j = 0; j < td->size; j++) {
                Hit *hit = td->hits[j];
                hit->doc += start;
                /*
                printf("adding hit = %d:%f\n", hit->doc, hit->score);
                */
                hq_insert(hq, hit);
            }
            td->size = 0;
            if (td->max_score > max_score) max_score = td->max_score;
        }
        total_hits += td->total_hits;
        td_destroy(td);
    }

    if (hq->size > first_doc) {
        if ((hq->size - first_doc) < num_docs) {
            num_docs = hq->size - first_doc;
        }
        score_docs = ALLOC_N(Hit *, num_docs);
        for (i = num_docs - 1; i >= 0; i--) {
            score_docs[i] = hq_pop(hq);
            /*
            Hit *hit = score_docs[i] = hq_pop(hq);
            printf("popped hit = %d-->%f\n", hit->doc, hit->score);
            */
        }
    }
    else {
        num_docs = 0;
    }
    pq_clear(hq);
    pq_destroy(hq);

    return td_new(total_hits, num_docs, score_docs, max_score);
}

static TopDocs *msea_search(Searcher *self,
                            Query *query,
                            int first_doc,
                            int num_docs,
                            Filter *filter,
                            Sort *sort,
                            PostFilter *post_filter,
                            bool load_fields)
{
    TopDocs *td;
    Weight *weight = q_weight(query, self);
    td = msea_search_w(self, weight, first_doc, num_docs, filter,
                       sort, post_filter, load_fields);
    weight->destroy(weight);
    return td;
}

static Query *msea_rewrite(Searcher *self, Query *original)
{
    int i;
    Searcher *s;
    MultiSearcher *msea = MSEA(self);
    Query **queries = ALLOC_N(Query *, msea->s_cnt), *rewritten;

    for (i = 0; i < msea->s_cnt; i++) {
        s = msea->searchers[i];
        queries[i] = s->rewrite(s, original);
    }
    rewritten = q_combine(queries, msea->s_cnt);

    for (i = 0; i < msea->s_cnt; i++) {
        q_deref(queries[i]);
    }
    free(queries);
    return rewritten;
}

static Explanation *msea_explain(Searcher *self, Query *query, int doc_num)
{
    MultiSearcher *msea = MSEA(self);
    int i = msea_get_searcher_index(self, doc_num);
    Weight *w = q_weight(query, self);
    Searcher *s = msea->searchers[i];
    Explanation *e = s->explain_w(s, w, doc_num - msea->starts[i]);
    w->destroy(w);
    return e;
}

static Explanation *msea_explain_w(Searcher *self, Weight *w, int doc_num)
{
    MultiSearcher *msea = MSEA(self);
    int i = msea_get_searcher_index(self, doc_num);
    Searcher *s = msea->searchers[i];
    Explanation *e = s->explain_w(s, w, doc_num - msea->starts[i]);
    return e;
}

static TermVector *msea_get_term_vector(Searcher *self, const int doc_num,
                                        Symbol field)
{
    MultiSearcher *msea = MSEA(self);
    int i = msea_get_searcher_index(self, doc_num);
    Searcher *s = msea->searchers[i];
    return s->get_term_vector(s, doc_num - msea->starts[i], field);
}

static Similarity *msea_get_similarity(Searcher *self)
{
    return self->similarity;
}

static void msea_close(Searcher *self)
{
    int i;
    Searcher *s;
    MultiSearcher *msea = MSEA(self);
    if (msea->close_subs) {
        for (i = 0; i < msea->s_cnt; i++) {
            s = msea->searchers[i];
            s->close(s);
        }
    }
    free(msea->searchers);
    free(msea->starts);
    free(self);
}

Searcher *msea_new(Searcher **searchers, int s_cnt, bool close_subs)
{
    int i, max_doc = 0;
    Searcher *self = (Searcher *)ALLOC(MultiSearcher);
    int *starts = ALLOC_N(int, s_cnt + 1);
    for (i = 0; i < s_cnt; i++) {
        starts[i] = max_doc;
        max_doc += searchers[i]->max_doc(searchers[i]);
    }
    starts[i] = max_doc;

    MSEA(self)->s_cnt           = s_cnt;
    MSEA(self)->searchers       = searchers;
    MSEA(self)->starts          = starts;
    MSEA(self)->max_doc         = max_doc;
    MSEA(self)->close_subs      = close_subs;

    self->similarity            = sim_create_default();
    self->doc_freq              = &msea_doc_freq;
    self->get_doc               = &msea_get_doc;
    self->get_lazy_doc          = &msea_get_lazy_doc;
    self->max_doc               = &msea_max_doc;
    self->create_weight         = &msea_create_weight;
    self->search                = &msea_search;
    self->search_w              = &msea_search_w;
    self->search_each           = &msea_search_each;
    self->search_each_w         = &msea_search_each_w;
    self->search_unscored       = &msea_search_unscored;
    self->search_unscored_w     = &msea_search_unscored_w;
    self->rewrite               = &msea_rewrite;
    self->explain               = &msea_explain;
    self->explain_w             = &msea_explain_w;
    self->get_term_vector       = &msea_get_term_vector;
    self->get_similarity        = &msea_get_similarity;
    self->close                 = &msea_close;
    return self;
}
