How can I parse html table inside a script respecting merged cells - google-apps-script

Question

This header is contained in a js file https://www.portaldefinancas.com/js-tx-ctb/th-cdib.js
document.write(""),document.write('</p></caption><thead><tr><th rowspan="4">Mês de<br>Referência</th><th colspan="7">Taxas - %</th></tr><tr> <th rowspan="3">Mensal</th><th colspan="4">Anualizada</th><th colspan="2">Acumulada</th></tr><tr> <th colspan="2">Ano de<br>252 dias<br> úteis</th><th colspan="2">Ano de<br>365/366 dias<br>corridos</th><th rowspan="2">No ano</th><th rowspan="2">Em <br>12 meses</th></tr><tr><th>Dias</th><th> Taxa</th><th>Dias</th><th> Taxa</th></tr></thead><tbody>');
How can I parse the headers respecting the merged rows and merged columns. The script I use today is
function getHeaders(url) {
var source = UrlFetchApp.fetch(url).getContentText()
source = source.split('document')[2]
var table = '<table><tr><th ' + source.match(/(?<=<th ).*(?=th>)/g) + 'th></tr></table>'
table=table.replace(/ê/g,'ê').replace(/ú/g,'ú').replace(/<br>/g,'\n')
var doc = XmlService.parse(table);
var rows = doc.getDescendants().filter(function(c) {
var element = c.asElement();
return element && element.getName() == "tr";
});
var data = rows.slice(0).map(function(row) {
return row.getChildren("th").map(function(cell) {
return cell.getValue();
});
});
return data;
}
but it doesn't respect merged areas. Thanks for any help !

Answer 1

Since intellectual exercises is my my drug of choice... I can't help it. Here is the possible solution. It works to a degree but it shows little the traits of lofty style of coding:
function main() {
var sheet = SpreadsheetApp.getActiveSheet();
var data = getHeaders();
data = handle_rowspans(data);
sheet.getRange(1, 1, data.length, data[0].length).setValues(data);
}
function getHeaders(url) {
// var source = UrlFetchApp.fetch(url).getContentText()
// source = source.split('document')[2]
var source = `<thead><tr><th rowspan="4">Mês de<br>Referência</th><th colspan="7">Taxas - %</th></tr><tr> <th rowspan="3">Mensal</th><th colspan="4">Anualizada</th><th colspan="2">Acumulada</th></tr><tr> <th colspan="2">Ano de<br>252 dias<br> úteis</th><th colspan="2">Ano de<br>365/366 dias<br>corridos</th><th rowspan="2">No ano</th><th rowspan="2">Em <br>12 meses</th></tr><tr><th>Dias</th><th> Taxa</th><th>Dias</th><th> Taxa</th></tr></thead><tbody>`;
source = handle_colspans(source);
table = '<table><tr><th ' + source.match(/(?<=<th ).*(?=th>)/g) + 'th></tr></table>';
table = table.replace(/ê/g, 'ê').replace(/ú/g, 'ú').replace(/<br>/g, '\n');
var doc = XmlService.parse(table);
var rows = doc.getDescendants().filter(function (c) {
var element = c.asElement();
return element && element.getName() == "tr";
});
var data = rows.slice(0).map(function (row) {
return row.getChildren("th").map(function (cell) {
return cell.getValue();
});
});
return data;
}
function handle_colspans(table) {
return table.split('</tr>').map(r => add_cells_in_row(r)).join('</tr>');
function add_cells_in_row(row) {
var cells = row.split('</th>');
for (var i in cells) {
if (/colspan/.test(cells[i])) {
var colspan = cells[i].replace(/.*colspan="(\d+).*/, '$1');
cells[i] += '{col' + colspan + '}';
cells[i] = [cells[i], ...(new Array(+colspan - 1).fill('<th>'))];
}
if (/rowspan/.test(cells[i])) {
var rowspan = cells[i].replace(/.*rowspan="(\d+).*/, '$1');
cells[i] += '{row' + rowspan + '}';
}
}
return cells.flat().join('</th>')
}
}
function handle_rowspans(array) {
for (var row in array) {
for (var col in array[row]) {
if (/\{row/.test(array[row][col])) {
var rowspan = array[row][col].replace(/.*\{row(\d+).*/s, '$1');
for (var r = 1; r < rowspan; r++) array[+row + r].splice(col, 0, '')
}
}
}
return array;
}
It will get you the table like this:
Whrere {row#} and {col#} means how many cells or rows to the left or to the bottom you need to join to the current cell to recreate the original design. It could be the next dose of the intellectual exercises. :)

Answer 2

A solution, example with url = https://www.portaldefinancas.com/js-tx-ctb/th-cdib.js
function getHeaders(url) {
var source = UrlFetchApp.fetch(url).getContentText()
source = source.split('document')[2]
var table = '<table><tr><th ' + source.match(/(?<=<th ).*(?=th>)/g) + 'th></tr></table>'
table=table.replace(/ê/g,'ê').replace(/ú/g,'ú').replace(/<br>/g,'\n')
var doc = XmlService.parse(table);
var rows = doc.getDescendants().filter(function(c) {
var element = c.asElement();
return element && element.getName() == "tr";
});
var n=0
var data=[]
rows.slice(0).map(function(row) {data[n++]=[]})
n=0
rows.slice(0).map(function(row) {
row.getChildren("th").map(function(cell) {
try{nbcols = cell.getAttribute('colspan').getValue()}catch(e){nbcols = 1}
try{nbrows = cell.getAttribute('rowspan').getValue()}catch(e){nbrows = 1}
var value = cell.getValue()
r=0
var free=0
while(r<nbrows*1){
c=0
while(c<nbcols*1){
while(data[n+r][free]!=null){free++}
data[n+r][free]=(value)
value=''
c++
}
r++
}
});
n++
});
return (data);
}