Added ability to read footer to read_html, and a test

Author: mjsu

doc/source/whatsnew/v0.15.2.txt

@@ -66,6 +66,7 @@ Enhancements
 - Added support for ``utcfromtimestamp()``, ``fromtimestamp()``, and ``combine()`` on `Timestamp` class (:issue:`5351`).
 - Added Google Analytics (`pandas.io.ga`) basic documentation (:issue:`8835`). See :ref:`here<remote_data.ga>`.
 - Added flag ``order_categoricals`` to ``StataReader`` and ``read_stata`` to select whether to order imported categorical data (:issue:`8836`).  See :ref:`here <io.stata-categorical>` for more information on importing categorical variables from Stata data files.
+- Added ability to read table footer to read_html (:issue:`8552`)
 
 .. _whatsnew_0152.performance:
 

pandas/io/html.py

@@ -577,7 +577,7 @@ def _parse_raw_thead(self, table):
                 table.xpath(expr)]
 
     def _parse_raw_tfoot(self, table):
-        expr = './/tfoot//th'
+        expr = './/tfoot//th|//tfoot//td'
         return [_remove_whitespace(x.text_content()) for x in
                 table.xpath(expr)]
 
@@ -594,14 +594,17 @@ def _expand_elements(body):
 
 def _data_to_frame(data, header, index_col, skiprows, infer_types,
                    parse_dates, tupleize_cols, thousands):
-    head, body, _ = data  # _ is footer which is rarely used: ignore for now
+    head, body, foot = data
 
     if head:
         body = [head] + body
 
         if header is None:  # special case when a table has <th> elements
             header = 0
 
+    if foot:
+        body += [foot]
+
     # fill out elements of body that are "ragged"
     _expand_elements(body)
 

pandas/io/tests/test_html.py

@@ -426,6 +426,38 @@ def test_empty_tables(self):
         res1 = self.read_html(StringIO(data1))
         res2 = self.read_html(StringIO(data2))
         assert_framelist_equal(res1, res2)
+    
+    def test_tfoot_read(self):
+        """
+        Make sure that read_html reads tfoot, containing td or th. 
+        Ignores empty tfoot
+        """
+        data_template = '''<table>
+            <thead>
+                <tr>
+                    <th>A</th>
+                    <th>B</th>
+                </tr>
+            </thead>
+            <tbody>
+                <tr>
+                    <td>bodyA</td>
+                    <td>bodyB</td>
+                </tr>
+            </tbody>
+            <tfoot>
+                {footer}
+            </tfoot>
+        </table>'''
+
+        data1 = data_template.format(footer = "")
+        data2 = data_template.format(footer ="<tr><td>footA</td><th>footB</th></tr>")
+    
+        d1 = {'A': ['bodyA'], 'B': ['bodyB']}
+        d2 = {'A': ['bodyA', 'footA'], 'B': ['bodyB', 'footB']}
+    
+        tm.assert_frame_equal(self.read_html(data1)[0], DataFrame(d1))
+        tm.assert_frame_equal(self.read_html(data2)[0], DataFrame(d2))
 
     def test_countries_municipalities(self):
         # GH5048