#NEWLINE# def smart_open( uri , mode = 'rb' , **kw ) : #NEWLINE# #INDENT# '\n    Open the given S3 / HDFS / filesystem file pointed to by `uri` for reading or writing.\n\n    The only supported modes for now are \'rb\' (read, default) and \'wb\' (replace & write).\n\n    The reads/writes are memory efficient (streamed) and therefore suitable for\n    arbitrarily large files.\n\n    The `uri` can be either:\n\n    1. a URI for the local filesystem (compressed ``.gz`` or ``.bz2`` files handled automatically):\n       `./lines.txt`, `/home/joe/lines.txt.gz`, `file:///home/joe/lines.txt.bz2`\n    2. a URI for HDFS: `hdfs:///some/path/lines.txt`\n    3. a URI for Amazon\'s S3 (can also supply credentials inside the URI):\n       `s3://my_bucket/lines.txt`, `s3://my_aws_key_id:key_secret@my_bucket/lines.txt`\n    4. an instance of the boto.s3.key.Key class.\n\n    Examples::\n\n      >>> # stream lines from S3; you can use context managers too:\n      >>> with smart_open.smart_open(\'s3://mybucket/mykey.txt\') as fin:\n      ...     for line in fin:\n      ...         print line\n\n      >>> # you can also use a boto.s3.key.Key instance directly:\n      >>> key = boto.connect_s3().get_bucket("my_bucket").get_key("my_key")\n      >>> with smart_open.smart_open(key) as fin:\n      ...     for line in fin:\n      ...         print line\n\n      >>> # stream line-by-line from an HDFS file\n      >>> for line in smart_open.smart_open(\'hdfs:///user/hadoop/my_file.txt\'):\n      ...    print line\n\n      >>> # stream content *into* S3:\n      >>> with smart_open.smart_open(\'s3://mybucket/mykey.txt\', \'wb\') as fout:\n      ...     for line in [\'first line\', \'second line\', \'third line\']:\n      ...          fout.write(line + \'\n\')\n\n      >>> # stream from/to (compressed) local files:\n      >>> for line in smart_open.smart_open(\'/home/radim/my_file.txt\'):\n      ...    print line\n      >>> for line in smart_open.smart_open(\'/home/radim/my_file.txt.gz\'):\n      ...    print line\n      >>> with smart_open.smart_open(\'/home/radim/my_file.txt.gz\', \'wb\') as fout:\n      ...    fout.write("hello world!\n")\n      >>> with smart_open.smart_open(\'/home/radim/another.txt.bz2\', \'wb\') as fout:\n      ...    fout.write("good bye!\n")\n\n    ' #NEWLINE# if ( not isinstance ( mode , six . string_types ) ) : #NEWLINE# #INDENT# raise TypeError ( 'mode should be a string' ) #NEWLINE# #UNINDENT# if isinstance ( uri , six . string_types ) : #NEWLINE# #INDENT# parsed_uri = ParseUri ( uri ) #NEWLINE# if ( host . scheme in ( 'file' , ) ) : #NEWLINE# #INDENT# return file_smart_open ( parsed_uri . uri_path , mode ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( parsed_uri . scheme in ( 's3' , 's3n' ) ) : #NEWLINE# #INDENT# host = kw . pop ( 'host' , None ) #NEWLINE# if ( not host ) : #NEWLINE# #INDENT# host = boto . config . get ( 's3' , 'host' , 's3.amazonaws.com' ) #NEWLINE# #UNINDENT# s3_connection = boto . connect_s3 ( aws_access_key_id = parsed_uri . access_id , host = host , aws_secret_access_key = parsed_uri . access_secret , profile_name = kw . pop ( 'profile_name' , None ) ) #NEWLINE# bucket = s3_connection . get_bucket ( parsed_uri . bucket_id ) #NEWLINE# if ( mode in ( 'r' , 'rb' ) ) : #NEWLINE# #INDENT# key = bucket . get_key ( parsed_uri . key_id ) #NEWLINE# if ( key is None ) : #NEWLINE# #INDENT# raise KeyError ( parsed_uri . key_id ) #NEWLINE# #UNINDENT# return S3OpenRead ( key , ** kw ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( mode in ( 'w' , 'wb' ) ) : #NEWLINE# #INDENT# key = bucket . get_key ( parsed_uri . key_id , validate = False ) #NEWLINE# if ( key is None ) : #NEWLINE# #INDENT# raise KeyError ( parsed_uri . key_id ) #NEWLINE# #UNINDENT# return S3OpenWrite ( key , ** kw ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# raise NotImplementedError ( 'file mode %s not supported for %r scheme' , mode , parsed_uri . scheme ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# if ( parsed_uri . scheme in ( 'hdfs' , ) ) : #NEWLINE# #INDENT# if ( mode in ( 'r' , 'rb' ) ) : #NEWLINE# #INDENT# return HdfsOpenRead ( parsed_uri , ** kw ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# raise NotImplementedError ( 'file mode %s not supported for %r scheme' , mode , parsed_uri . scheme ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# if ( parsed_uri . scheme in ( 'webhdfs' , ) ) : #NEWLINE# #INDENT# if ( mode in ( 'r' , 'rb' ) ) : #NEWLINE# #INDENT# return WebHdfsOpenRead ( parsed_uri , ** kw ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( mode in ( 'w' , 'wb' ) ) : #NEWLINE# #INDENT# return WebHdfsOpenWrite ( parsed_uri , ** kw ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# raise NotImplementedError ( 'file mode %s not supported for %r scheme' , mode , parsed_uri . scheme ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# raise NotImplementedError ( 'scheme %r is not supported' , parsed_uri . scheme ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# if isinstance ( uri , boto . s3 . key . Key ) : #NEWLINE# #INDENT# if ( mode in ( 'r' , 'rb' ) ) : #NEWLINE# #INDENT# return S3OpenRead ( uri ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( mode in ( 'w' , 'wb' ) ) : #NEWLINE# #INDENT# return S3OpenWrite ( uri ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# if hasattr ( uri , 'read' ) : #NEWLINE# #INDENT# return uri #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# raise TypeError ( ( "don't know how to handle uri %s" % repr ( uri ) ) )