#NEWLINE# def json_corpus( json_file , doc_key , label_key , encoding = 'utf8' , nltk_stop = False , stop_freq = 0 , add_stop = None , tokenizer = word_tokenize ) : #NEWLINE# #INDENT# "\n    `json_corpus` is a convenience function for generating Corpus\n    objects from a json file. It construct a corpus, document labels\n    and metadata respectively from the specified fields in the json file.\n\n    `json_corpus` will perform word-level tokenization. \n    It will also strip punctuation and arabic numerals\n    outside the range 1-29. All letters are made lowercase.\n\n    :param json_file: Json file name containing documents and metadata.\n    :type json_file: string-like\n    \n    :param doc_key: Name of the key for documents.\n    :type doc_key: string-like\n\n    :param label_key: Name of the key used for document labels. Labels are \n    used when a viewer function outputs a list of documents. Any field other\n    than `doc_key` and `label_key` is stored as metadata.\n    :type label_key: string-like\n\n    :param encoding: A string indicating the file encoding or 'detect',\n        in which case `chardet` is used to automatically guess the encoding.\n        Default is `utf8`.\n    :type encoding: string, optional\n    \n    :param nltk_stop: If `True` then the corpus object is masked using\n        the NLTK English stop words. Default is `False`.\n    :type nltk_stop: boolean, optional\n\n    :param stop_freq: The upper bound for a word to be masked on the basis of its\n        collection frequency. Default is 0.\n    :type stop_freq: int, optional\n\n    :param add_stop: A list of stop words. Default is `None`.\n    :type add_stop: array-like, optional\n    \n    :param tokenizer: word tokenization function. Defaults to `vsm.extensions.corpusbuilders.util.word_tokenize`.\n    :type tokenizer: lambda s -> tokens\n\n    :returns: c : a Corpus object\n        Contains the tokenized corpus built from the input plain-text\n        corpus. Document tokens are named `documents`.\n\n    :See Also: :class:`vsm.corpus.Corpus`, \n        :meth:`vsm.corpus.util.paragraph_tokenize`, \n        :meth:`vsm.corpus.util.apply_stoplist`\n    " #NEWLINE# import json #NEWLINE# if ( encoding == 'detect' ) : #NEWLINE# #INDENT# encoding = detect_encoding ( json_file ) #NEWLINE# #UNINDENT# with open ( json_file , 'r' , encoding = encoding ) as f : #NEWLINE# #INDENT# json_data = json . load ( f ) #NEWLINE# #UNINDENT# docs = [ ] #NEWLINE# label = [ ] #NEWLINE# metadata = [ ] #NEWLINE# for i in json_data : #NEWLINE# #INDENT# docs . append ( i . pop ( doc_key , None ) . encode ( 'ascii' , 'ignore' ) ) #NEWLINE# label . append ( i . pop ( label_key , None ) ) #NEWLINE# metadata . append ( i ) #NEWLINE# #UNINDENT# docs = [ tokenizer ( d ) for d in docs ] #NEWLINE# corpus = sum ( docs , [ ] ) #NEWLINE# tok = np . cumsum ( np . array ( [ len ( d ) for d in docs ] ) ) #NEWLINE# dtype = [ ( 'idx' , np . array ( tok ) . dtype ) , ( 'document_label' , np . array ( label ) . dtype ) , ( 'metadata' , np . array ( metadata ) . dtype ) ] #NEWLINE# tok = np . array ( zip ( tok , label , metadata ) , dtype = dtype ) #NEWLINE# c = Corpus ( corpus , context_data = [ tok ] , context_types = [ 'document' ] ) #NEWLINE# if ( nltk_stop or stop_freq or add_stop ) : #NEWLINE# #INDENT# c = apply_stoplist ( c , nltk_stop = nltk_stop , freq = stop_freq , add_stop = add_stop ) #NEWLINE# #UNINDENT# return c