Issue 1:
Code Sample, a copy-pastable example if possible

# Your code here
import pandas as pd
tables = pd.read_html('D:\\myhtml.html', header=0)
print (tables)

Problem description

[The html code looks like below. Note simple table scrapping looks good but on this one exactly it is not working as expected

The html code

I have a html file locally with multiple tables. Sometimes the content of the table is exactly the same including headers. While reading it through pandas I have noticed that when two tables are exactly identical it will drop the second one as if it is not there. When I change one Value value in the second table it will read the second table also and display.

How can i stop pandas doing that and read every table. Or is it a feature or bug

Attaching the exact HTML file in link. If you see there are 4 tables. Still I get only 3 tables values. The big 2 two tables have exact same data and it is publishing only first one

environment : Anaconda 3.6

]

Note: We receive a lot of issues on our GitHub tracker, so it is very possible that your issue has been posted before.

Please check first before submitting so that we do not have to handle and close duplicates!

Note: Many problems can be resolved by simply upgrading pandas to the latest version. Before submitting, please check

if that solution works for you. If possible, you may want to check if master addresses this issue, but that is not necessary.

Expected Output

Ideally it should show 4 tables. Showing only 3

Output of pd.show_versions()

[paste the output of pd.show_versions() here below this line]

INSTALLED VERSIONS

commit: None

python: 3.6.0.final.0

python-bits: 64

OS: Windows

OS-release: 7

machine: AMD64

processor: Intel64 Family 6 Model 60 Stepping 3, GenuineIntel

byteorder: little

LC_ALL: None

LANG: en

LOCALE: None.None

pandas: 0.19.2

nose: 1.3.7

pip: 9.0.1

setuptools: 27.2.0

Cython: 0.25.2

numpy: 1.11.3

scipy: 0.18.1

statsmodels: 0.6.1

xarray: None

IPython: 5.1.0

sphinx: 1.5.1

patsy: 0.4.1

dateutil: 2.6.0

pytz: 2016.10

blosc: None

bottleneck: 1.2.0

tables: 3.2.2

numexpr: 2.6.1

matplotlib: 2.0.0

openpyxl: 2.4.1

xlrd: 1.0.0

xlwt: 1.2.0

xlsxwriter: 0.9.6

lxml: 3.7.2

bs4: 4.6.0

html5lib: 0.9999999

httplib2: None

apiclient: None

sqlalchemy: 1.1.5

pymysql: None

psycopg2: None

jinja2: 2.9.4

boto: 2.45.0

pandas_datareader: None

Issue 2:
Code Sample

def get_user_account_credentials(): # https://github.com/pydata/pandas/blob/master/pandas/io/gbq.py#L160
from oauth2client.client import OAuth2WebServerFlow
from oauth2client.file import Storage
from oauth2client.tools import run_flow, argparser

reauth = False

flow = OAuth2WebServerFlow(
client_id=('495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd'
'.apps.googleusercontent.com'),
client_secret='kOc9wMptUtxkcIFbtZCcrEAc',
scope=['https://www.googleapis.com/auth/bigquery'],
redirect_uri='urn:ietf:wg:oauth:2.0:oob')

storage = Storage('bigquery_credentials.dat')
credentials = storage.get()

if credentials is None or credentials.invalid or reauth:
credentials = run_flow(flow, storage, argparser.parse_args([]))

return credentials

def _generate_bq_schema(df, default_type='STRING'): # https://github.com/pydata/pandas/blob/master/pandas/io/gbq.py#L735
""" Given a passed df, generate the associated Google BigQuery schema.
Parameters
----------
df : DataFrame
default_type : string
The default big query type in case the type of the column
does not exist in the schema.
"""

type_mapping = {
'i': 'INTEGER',
'b': 'BOOLEAN',
'f': 'FLOAT',
'O': 'STRING',
'S': 'STRING',
'U': 'STRING',
'M': 'TIMESTAMP'
}

fields = []
for column_name, dtype in df.dtypes.iteritems():
fields.append({'name': column_name,
'type': type_mapping.get(dtype.kind, default_type)})

return {'fields': fields}

def get_service(): # https://github.com/pydata/pandas/blob/master/pandas/io/gbq.py#L267
from apiclient.discovery import build
import httplib2

http = httplib2.Http()
credentials = get_user_account_credentials()
http = credentials.authorize(http)
return build('bigquery', 'v2', http=http)

def verify_schema(dataset_id, table_id, schema): # https://github.com/pydata/pandas/blob/master/pandas/io/gbq.py#L476
from apiclient.errors import HttpError
bigquery_service = get_service()

try:
return (bigquery_service.tables().get(
projectId='publicdata',
datasetId=dataset_id,
tableId=table_id
).execute()['schema']) == schema

except HttpError as ex:
pass
# self.process_http_error(ex)

def main():
shakespeare_df = pd.DataFrame({'word': 'foo', 'word_count': [1], 'corpus': 'bar', 'corpus_date': [2016]},
columns=['word', 'word_count', 'corpus', 'corpus_date'])
table_schema = _generate_bq_schema(shakespeare_df)

print 'Is the schema verified?'
print verify_schema('samples', 'shakespeare', table_schema)

if __name__ == '__main__':
main()

Expected Output

We want verify_schema to return True since the dataframe's columns are in the correct order, are named correctly, and have the correct types. The BigQuery table schema, however, has description and mode in addition to name and type.

Therefore verify_schema should remove description and mode from the BigQuery table schema when comparing it to the dataframe's schema.

Solution:

def get_user_account_credentials(): # https://github.com/pydata/pandas/blob/master/pandas/io/gbq.py#L160
from oauth2client.client import OAuth2WebServerFlow
from oauth2client.file import Storage
from oauth2client.tools import run_flow, argparser

reauth = False

flow = OAuth2WebServerFlow(
client_id=('495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd'
'.apps.googleusercontent.com'),
client_secret='kOc9wMptUtxkcIFbtZCcrEAc',
scope=['https://www.googleapis.com/auth/bigquery'],
redirect_uri='urn:ietf:wg:oauth:2.0:oob')

storage = Storage('bigquery_credentials.dat')
credentials = storage.get()

if credentials is None or credentials.invalid or reauth:
credentials = run_flow(flow, storage, argparser.parse_args([]))

return credentials

def _generate_bq_schema(df, default_type='STRING'): # https://github.com/pydata/pandas/blob/master/pandas/io/gbq.py#L735
""" Given a passed df, generate the associated Google BigQuery schema.
Parameters
----------
df : DataFrame
default_type : string
The default big query type in case the type of the column
does not exist in the schema.
"""

type_mapping = {
'i': 'INTEGER',
'b': 'BOOLEAN',
'f': 'FLOAT',
'O': 'STRING',
'S': 'STRING',
'U': 'STRING',
'M': 'TIMESTAMP'
}

fields = []
for column_name, dtype in df.dtypes.iteritems():
fields.append({'name': column_name,
'type': type_mapping.get(dtype.kind, default_type)})

return {'fields': fields}

def get_service(): # https://github.com/pydata/pandas/blob/master/pandas/io/gbq.py#L267
from apiclient.discovery import build
import httplib2

http = httplib2.Http()
credentials = get_user_account_credentials()
http = credentials.authorize(http)
return build('bigquery', 'v2', http=http)

def verify_schema(dataset_id, table_id, schema): # https://github.com/pydata/pandas/blob/master/pandas/io/gbq.py#L476
from apiclient.errors import HttpError
bigquery_service = get_service()

try:
return (bigquery_service.tables().get(
projectId='publicdata',
datasetId=dataset_id,
tableId=table_id
).execute()['schema']) == schema

except HttpError as ex:
pass
# self.process_http_error(ex)

def verify_schema_modified(dataset_id, table_id, schema):
from apiclient.errors import HttpError
bigquery_service = get_service()
try:
original_schema = bigquery_service.tables().get(
projectId='publicdata',
datasetId=dataset_id,
tableId=table_id
).execute()['schema']

modified_schema_list = [{key:field[key] for key in field if key != 'mode' and key != 'description'}
for field in original_schema['fields']]

return {'fields': modified_schema_list} == schema
except HttpError as ex:
pass
#self.process_http_error(ex)

def main():
shakespeare_df = pd.DataFrame({'word': 'foo', 'word_count': [1], 'corpus': 'bar', 'corpus_date': [2016]},
columns=['word', 'word_count', 'corpus', 'corpus_date'])
table_schema = _generate_bq_schema(shakespeare_df)

print 'Is the schema verified?'
print verify_schema('samples', 'shakespeare', table_schema)
print verify_schema_modified('samples', 'shakespeare', table_schema)

if __name__ == '__main__':
main()

output of pd.show_versions()

INSTALLED VERSIONS
------------------
commit: 5fd6ed036324f5de28816e2b66348e8a56b96f04
python: 2.7.10.final.0
python-bits: 64
OS: Darwin
OS-release: 15.3.0
machine: x86_64
processor: i386
byteorder: little
LC_ALL: None
LANG: None

pandas: 0.18.1+8.g5fd6ed0
nose: 1.3.6
pip: 8.1.1
setuptools: 20.8.1
Cython: 0.22
numpy: 1.11.0
scipy: 0.13.0b1
statsmodels: None
xarray: None
IPython: 4.0.0
sphinx: 1.3.1
patsy: None
dateutil: 2.5.2
pytz: 2016.3
blosc: None
bottleneck: None
tables: None
numexpr: None
matplotlib: 1.3.1
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml: None
bs4: None
html5lib: None
httplib2: 0.9.2
apiclient: 1.5.0
sqlalchemy: None
pymysql: None
psycopg2: None
jinja2: 2.8
boto: 2.38.0
pandas_datareader: None