GitHub PullRequestReviewCommentEvent at 2014-04-02 09:11:05
Repository: elasticsearch
Actor: s1monw
Payload: {"comment":{"url":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/comments/11197325","id":11197325,"diff_hunk":"@@ -0,0 +1,217 @@\n+/*\n+ * Licensed to Elasticsearch under one or more contributor\n+ * license agreements. See the NOTICE file distributed with\n+ * this work for additional information regarding copyright\n+ * ownership. Elasticsearch licenses this file to you under\n+ * the Apache License, Version 2.0 (the \"License\"); you may\n+ * not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing,\n+ * software distributed under the License is distributed on an\n+ * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+ * KIND, either express or implied.  See the License for the\n+ * specific language governing permissions and limitations\n+ * under the License.\n+ */\n+\n+package org.elasticsearch.common.lucene.index;\n+\n+import com.google.common.collect.Lists;\n+import org.apache.lucene.index.*;\n+import org.apache.lucene.search.DocIdSet;\n+import org.apache.lucene.search.DocIdSetIterator;\n+import org.apache.lucene.search.Filter;\n+import org.apache.lucene.util.Bits;\n+import org.apache.lucene.util.BytesRef;\n+import org.elasticsearch.ElasticsearchIllegalArgumentException;\n+import org.elasticsearch.common.Nullable;\n+import org.elasticsearch.common.lucene.docset.DocIdSets;\n+import org.elasticsearch.common.lucene.search.ApplyAcceptedDocsFilter;\n+import org.elasticsearch.common.lucene.search.Queries;\n+\n+import java.io.IOException;\n+import java.util.Comparator;\n+import java.util.List;\n+\n+/**\n+ * A frequency TermsEnum that returns frequencies derived from a collection of cached leaf termEnums. \n+ * It also allows to provide a filter to explicitly compute frequencies only for docs that match the filter (heavier!).\n+ */\n+public class FilterableTermsEnum extends TermsEnum  {\n+\n+    static class Holder {\n+        final TermsEnum termsEnum;\n+        @Nullable\n+        DocsEnum docsEnum;\n+        @Nullable\n+        final Bits bits;\n+\n+        Holder(TermsEnum termsEnum, Bits bits) {\n+            this.termsEnum = termsEnum;\n+            this.bits = bits;\n+        }\n+    }\n+    protected final static int NOT_FOUND = -2;\n+    private final Holder[] enums;\n+    protected int currentDocFreq = 0;\n+    protected long currentTotalTermFreq = 0;\n+    protected BytesRef current;\n+    protected final int docsEnumFlag;\n+    protected int numDocs;\n+\n+    public FilterableTermsEnum(IndexReader reader, String field, int docsEnumFlag, @Nullable Filter filter) throws IOException {\n+        if ((docsEnumFlag != DocsEnum.FLAG_FREQS) && (docsEnumFlag != DocsEnum.FLAG_NONE)) {\n+            throw new ElasticsearchIllegalArgumentException(\"invalid docsEnumFlag of \" + docsEnumFlag);\n+        }\n+        this.docsEnumFlag = docsEnumFlag;\n+        if (filter == null) {\n+            numDocs = reader.numDocs();\n+        }\n+        \n+        List<AtomicReaderContext> leaves = reader.leaves();\n+        List<Holder> enums = Lists.newArrayListWithExpectedSize(leaves.size());\n+        for (AtomicReaderContext context : leaves) {\n+            Terms terms = context.reader().terms(field);\n+            if (terms == null) {\n+                continue;\n+            }\n+            TermsEnum termsEnum = terms.iterator(null);\n+            if (termsEnum == null) {\n+                continue;\n+            }\n+            Bits bits = null;\n+            if (filter != null) {\n+                if (filter == Queries.MATCH_ALL_FILTER) {\n+                    bits = context.reader().getLiveDocs();\n+                } else {\n+                    // we want to force apply deleted docs\n+                    filter = new ApplyAcceptedDocsFilter(filter);\n+                    DocIdSet docIdSet = filter.getDocIdSet(context, context.reader().getLiveDocs());\n+                    if (DocIdSets.isEmpty(docIdSet)) {\n+                        // fully filtered, none matching, no need to iterate on this\n+                        continue;\n+                    }\n+                    bits = DocIdSets.toSafeBits(context.reader(), docIdSet);\n+                    // Count how many docs are in our filtered set \n+                    // TODO make this lazy-loaded only for those that need it?\n+                    DocIdSetIterator iterator = docIdSet.iterator();\n+                    if (iterator != null) {\n+                        while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {\n+                            numDocs++;\n+                        }\n+                    }\n+                }\n+            }\n+            enums.add(new Holder(termsEnum, bits));\n+        }\n+        this.enums = enums.toArray(new Holder[enums.size()]);\n+    }\n+    \n+    public int getNumDocs() {\n+        return numDocs;\n+    }\n+\n+    @Override\n+    public BytesRef term() throws IOException {\n+        return current;\n+    }\n+\n+    @Override\n+    public boolean seekExact(BytesRef text) throws IOException {\n+        boolean found = false;\n+        int docFreq = 0;\n+        long totalTermFreq = 0;\n+        for (Holder anEnum : enums) {\n+            if (!anEnum.termsEnum.seekExact(text)) {\n+                continue;\n+            }\n+            found = true;\n+            if (anEnum.bits == null) {\n+                docFreq += anEnum.termsEnum.docFreq();\n+                if (docsEnumFlag == DocsEnum.FLAG_FREQS) {\n+                    long leafTotalTermFreq = anEnum.termsEnum.totalTermFreq();\n+                    if (totalTermFreq == -1 || leafTotalTermFreq == -1) {\n+                        totalTermFreq = -1;\n+                        continue;\n+                    }                    \n+                    totalTermFreq += leafTotalTermFreq;\n+                }\n+            } else {\n+                DocsEnum docsEnum = anEnum.docsEnum = anEnum.termsEnum.docs(anEnum.bits, anEnum.docsEnum, docsEnumFlag);\n+                // 2 choices for performing same heavy loop - one attempts to calculate totalTermFreq and other does not\n+                if (docsEnumFlag == DocsEnum.FLAG_FREQS) {\n+                    for (int docId = docsEnum.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {\n+                        docFreq++;\n+                        // docsEnum.freq() returns 1 if doc indexed with IndexOptions.DOCS_ONLY so no way of knowing if value\n+                        // is really 1 or unrecorded when filtering like this\n+                        totalTermFreq += docsEnum.freq();\n+                    }\n+                } else {\n+                    for (int docId = docsEnum.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {\n+                        //docsEnum.freq() behaviour is undefined if docsEnumFlag==DocsEnum.FLAG_NONE so don't bother with call \n+                        docFreq++;\n+                    }\n+                }\n+            }\n+        }\n+\n+        current = found ? text : null;\n+        if(found){","path":"src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java","position":162,"original_position":162,"commit_id":"717efd1d215e9c724ae2c94d92105a27c048870a","original_commit_id":"717efd1d215e9c724ae2c94d92105a27c048870a","user":{"login":"s1monw","id":973334,"avatar_url":"https://avatars.githubusercontent.com/u/973334?","gravatar_id":"8c12b7030c2431ff54d71891a722ff9a","url":"https://api.github.com/users/s1monw","html_url":"https://github.com/s1monw","followers_url":"https://api.github.com/users/s1monw/followers","following_url":"https://api.github.com/users/s1monw/following{/other_user}","gists_url":"https://api.github.com/users/s1monw/gists{/gist_id}","starred_url":"https://api.github.com/users/s1monw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s1monw/subscriptions","organizations_url":"https://api.github.com/users/s1monw/orgs","repos_url":"https://api.github.com/users/s1monw/repos","events_url":"https://api.github.com/users/s1monw/events{/privacy}","received_events_url":"https://api.github.com/users/s1monw/received_events","type":"User","site_admin":false},"body":"there are some minor formatting glitches here like whitespaces - can you reformat the files?","created_at":"2014-04-02T09:11:05Z","updated_at":"2014-04-02T09:11:05Z","html_url":"https://github.com/elasticsearch/elasticsearch/pull/5597#discussion_r11197325","pull_request_url":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/5597","_links":{"self":{"href":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/comments/11197325"},"html":{"href":"https://github.com/elasticsearch/elasticsearch/pull/5597#discussion_r11197325"},"pull_request":{"href":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/5597"}}}}