I Have this table;
that was created using this code;
let
Source = Xml.Tables(Web.Contents("https://www.edmunds.com/sitemap_web54-mmy-cost-to-own.xml")),
Table0 = Source{0}[Table],
#"Kept First Rows" = Table.FirstN(Table0,10),
#"Added Custom" = Table.AddColumn(#"Kept First Rows", "Custom", each Web.BrowserContents([loc])),
#"Added Custom3" = Table.AddColumn(#"Added Custom", "Custom.3", each try Text.Range([Custom],Text.PositionOf([Custom],"<optgroup"),Text.PositionOf([Custom],"</optgroup>")-Text.PositionOf([Custom],"<optgroup")+11) otherwise "<optgroup/>"),
#"Parsed XML" = Table.TransformColumns(#"Added Custom3",{{"Custom.3", Xml.Tables}}),
#"Expanded Custom.3" = Table.ExpandTableColumn(#"Parsed XML", "Custom.3", {"option"}, {"option"}),
#"Expanded option" = Table.ExpandTableColumn(#"Expanded Custom.3", "option", {"Element:Text", "Attribute:value"}, {"Model", "Style"})
in
#"Expanded option"
If you look at the loc column you will see that there is the same link for multiple models.
Ultimately I want the Cost to Own data for each model.
So I created this code in a new query and tied it to a custom column in the above query.
(PageMake as text)=>
let
Source = Web.BrowserContents(PageMake),
#"Extracted Table From Html" = Html.Table(Source, {{"Column1", "SECTION:nth-child(2) > DIV.table-responsive > TABLE.costs-table.text-gray-darker.table.table-borderless > * > TR > :nth-child(1)"}, {"Column2", "SECTION:nth-child(2) > DIV.table-responsive > TABLE.costs-table.text-gray-darker.table.table-borderless > * > TR > :nth-child(2)"}, {"Column3", "SECTION:nth-child(2) > DIV.table-responsive > TABLE.costs-table.text-gray-darker.table.table-borderless > * > TR > :nth-child(3)"}, {"Column4", "SECTION:nth-child(2) > DIV.table-responsive > TABLE.costs-table.text-gray-darker.table.table-borderless > * > TR > :nth-child(4)"}, {"Column5", "SECTION:nth-child(2) > DIV.table-responsive > TABLE.costs-table.text-gray-darker.table.table-borderless > * > TR > :nth-child(5)"}, {"Column6", "SECTION:nth-child(2) > DIV.table-responsive > TABLE.costs-table.text-gray-darker.table.table-borderless > * > TR > :nth-child(6)"}, {"Column7", "SECTION:nth-child(2) > DIV.table-responsive > TABLE.costs-table.text-gray-darker.table.table-borderless > * > TR > :nth-child(7)"}}, [RowSelector="SECTION:nth-child(2) > DIV.table-responsive > TABLE.costs-table.text-gray-darker.table.table-borderless > * > TR"]),
#"Promoted Headers" = Table.PromoteHeaders(#"Extracted Table From Html", [PromoteAllScalars=true]),
#"Changed Type" = Table.TransformColumnTypes(#"Promoted Headers",{{"", type text}, {"Year 1", Currency.Type}, {"Year 2", Currency.Type}, {"Year 3", Currency.Type}, {"Year 4", Currency.Type}, {"Year 5", Currency.Type}, {"Total", Currency.Type}})
in
#"Changed Type"
The problem is that the link does not have any tie to the different models so when it pulls the Cost to Own data table it simply pulls the first one for all models associated with that link.
The problem you are facing is that there is not a distinct table for each vehicle. There is one table shared by several. If you do a new web query (Query -> Web) and enter a URL like https://www.edmunds.com/lexus/ls-460/2016/cost-to-own/?style=401580678 you'll see the collection of tables. (I'm quite sure you already know this from your info above.) But if you look within the Suggested Tables, you'll see they all contain information for multiple 2016 models. I think you'll need to pull the entire table with the information you want (I think you want Table 1) and then parse that table once you get it. You can use any one of the vehicles from that table to get to the table. (It appears to me, from a quick glance, that each table is all of the vehicles from the year group).
Call your function from a new column in your first query with = Table.AddColumn(#"Expanded option", "Custom.1", each fnGetEdmunds(Text.From([loc])&"?style="&Text.From([Style]))). You'll get some errors because some rows don't have Styles to find pages with the Cost to Own and some of the pages don't have tables with Cost to Own. So you'll have to handle those errors.
Here's M code:
//The base query:
let
Source = Xml.Tables(Web.Contents("https://www.edmunds.com/sitemap_web54-mmy-cost-to-own.xml")),
Table0 = Source{0}[Table],
#"Kept First Rows" = Table.FirstN(Table0,10),
#"Added Custom" = Table.AddColumn(#"Kept First Rows", "Custom", each Web.BrowserContents([loc])),
#"Added Custom3" = Table.AddColumn(#"Added Custom", "Custom.3", each try Text.Range([Custom],Text.PositionOf([Custom],"<optgroup"),Text.PositionOf([Custom],"</optgroup>")-Text.PositionOf([Custom],"<optgroup")+11) otherwise "<optgroup/>"),
#"Parsed XML" = Table.TransformColumns(#"Added Custom3",{{"Custom.3", Xml.Tables}}),
#"Expanded Custom.3" = Table.ExpandTableColumn(#"Parsed XML", "Custom.3", {"option"}, {"option"}),
#"Expanded option" = Table.ExpandTableColumn(#"Expanded Custom.3", "option", {"Element:Text", "Attribute:value"}, {"Model", "Style"}),
#"Invoked Custom Function" = Table.AddColumn(#"Expanded option", "Custom.1", each fnGetEdmunds(Text.From([loc])&"?style="&Text.From([Style])))
in
#"Invoked Custom Function"
//The function named fnGetEdmunds
(PageMake as text)=>
let
Source = Web.BrowserContents(PageMake),
#"Extracted Table From Html" = Html.Table(Source, {{"Column1", ".col-fixed"}, {"Column2", ".col-padding-left"}, {"Column3", ".col-padding-left + *"}, {"Column4", ".d-none TD:nth-child(4)"}, {"Column5", ".d-none TD:nth-child(5)"}, {"Column6", ".d-none TD:nth-child(6)"}, {"Column7", ".d-none .font-weight-bold"}, {"Column8", ".d-none:nth-child(3) TD:nth-child(4)"}, {"Column9", ".d-none:nth-child(3) TD:nth-child(5)"}, {"Column10", ".d-none:nth-child(3) TD:nth-child(6)"}, {"Column11", ".d-none:nth-child(4) TD:nth-child(4)"}, {"Column12", ".d-none:nth-child(4) TD:nth-child(5)"}, {"Column13", ".d-none:nth-child(4) TD:nth-child(6)"}, {"Column14", ".d-none:nth-child(5) TD:nth-child(4)"}, {"Column15", ".d-none:nth-child(5) TD:nth-child(5)"}, {"Column16", ".d-none:nth-child(5) TD:nth-child(6)"}, {"Column17", ".d-inline"}, {"Column18", ".p-0.heading-4"}, {"Column19", ".mb-1 SPAN"}, {"Column20", "CAPTION"}, {"Column21", "TH:nth-child(4)"}, {"Column22", "TH:nth-child(5)"}, {"Column23", "TH:nth-child(6)"}, {"Column24", "TH:nth-child(7)"}}, [RowSelector=".col-fixed"]),
#"Promoted Headers" = Table.PromoteHeaders(#"Extracted Table From Html", [PromoteAllScalars=true]),
#"Changed Type" = Table.TransformColumnTypes(#"Promoted Headers",{{"", type text}, {"Year 1", type text}, {"Year 2", type text}, {"Column4", Currency.Type}, {"Column5", Currency.Type}, {"Column6", Currency.Type}, {"Column7", Currency.Type}, {"Column8", Currency.Type}, {"Column9", Currency.Type}, {"Column10", Currency.Type}, {"Column11", Currency.Type}, {"Column12", Currency.Type}, {"Column13", Currency.Type}, {"Column14", Currency.Type}, {"Column15", Currency.Type}, {"Column16", Currency.Type}, {"Column17", type text}, {"Column18", type text}, {"Column19", type text}, {"Column20", type text}, {"Year 3", type text}, {"Year 4", type text}, {"Year 5", type text}, {"Total", type text}})
in
#"Changed Type"
Like I said above: You'll still need to parse the tables to extract the info for the specific vehicles, and to handle the errors.
it is my first use of power query and I need your help
I'd like to retrieve from web some compagny values base on their identification number, the french SIRET
for that I'd like to enter the SIRET in a cell (named entersiret) in my excel and then pass it to the url and in power query. I have tried this but it is not working :
let
siret = Excel.CurrentWorkbook(){[Name="entersiret"]}[Content],
entersiret = siret[Column1]{0},
Source = Json.Document(Web.Contents("https://entreprise.data.gouv.fr/api/sirene/v3/etablissements/" & siret)),
#"Converti en table" = Record.ToTable(Source),
#"Value développé" = Table.ExpandRecordColumn(#"Converti en table", "Value", {"id", "siren", "nic", "siret", "statut_diffusion", "date_creation", "tranche_effectifs", "annee_effectifs", "activite_principale_registre_metiers", "date_dernier_traitement", "etablissement_siege", "nombre_periodes", "complement_adresse", "numero_voie", "indice_repetition", "type_voie", "libelle_voie", "code_postal", "libelle_commune", "libelle_commune_etranger", "distribution_speciale", "code_commune", "code_cedex", "libelle_cedex", "code_pays_etranger", "libelle_pays_etranger", "complement_adresse_2", "numero_voie_2", "indice_repetition_2", "type_voie_2", "libelle_voie_2", "code_postal_2", "libelle_commune_2", "libelle_commune_etranger_2", "distribution_speciale_2", "code_commune_2", "code_cedex_2", "libelle_cedex_2", "code_pays_etranger_2", "libelle_pays_etranger_2", "date_debut", "etat_administratif", "enseigne_1", "enseigne_2", "enseigne_3", "denomination_usuelle", "activite_principale", "nomenclature_activite_principale", "caractere_employeur", "longitude", "latitude", "geo_score", "geo_type", "geo_adresse", "geo_id", "geo_ligne", "geo_l4", "geo_l5", "unite_legale_id", "created_at", "updated_at", "unite_legale"}, {"Value.id", "Value.siren", "Value.nic", "Value.siret", "Value.statut_diffusion", "Value.date_creation", "Value.tranche_effectifs", "Value.annee_effectifs", "Value.activite_principale_registre_metiers", "Value.date_dernier_traitement", "Value.etablissement_siege", "Value.nombre_periodes", "Value.complement_adresse", "Value.numero_voie", "Value.indice_repetition", "Value.type_voie", "Value.libelle_voie", "Value.code_postal", "Value.libelle_commune", "Value.libelle_commune_etranger", "Value.distribution_speciale", "Value.code_commune", "Value.code_cedex", "Value.libelle_cedex", "Value.code_pays_etranger", "Value.libelle_pays_etranger", "Value.complement_adresse_2", "Value.numero_voie_2", "Value.indice_repetition_2", "Value.type_voie_2", "Value.libelle_voie_2", "Value.code_postal_2", "Value.libelle_commune_2", "Value.libelle_commune_etranger_2", "Value.distribution_speciale_2", "Value.code_commune_2", "Value.code_cedex_2", "Value.libelle_cedex_2", "Value.code_pays_etranger_2", "Value.libelle_pays_etranger_2", "Value.date_debut", "Value.etat_administratif", "Value.enseigne_1", "Value.enseigne_2", "Value.enseigne_3", "Value.denomination_usuelle", "Value.activite_principale", "Value.nomenclature_activite_principale", "Value.caractere_employeur", "Value.longitude", "Value.latitude", "Value.geo_score", "Value.geo_type", "Value.geo_adresse", "Value.geo_id", "Value.geo_ligne", "Value.geo_l4", "Value.geo_l5", "Value.unite_legale_id", "Value.created_at", "Value.updated_at", "Value.unite_legale"})
in
#"Value développé"
You drill down the value "siret" in "entersiret" but you don't use it. This will work:
let
siret = Excel.CurrentWorkbook(){[Name="entersiret"]}[Content],
entersiret = siret[Column1]{0},
Source = Json.Document(Web.Contents("https://entreprise.data.gouv.fr/api/sirene/v3/etablissements/" & entersiret)),
#"Converti en table" = Record.ToTable(Source),
#"Value développé" = Table.ExpandRecordColumn(#"Converti en table", "Value", {"id", "siren", "nic", "siret", "statut_diffusion", "date_creation", "tranche_effectifs", "annee_effectifs", "activite_principale_registre_metiers", "date_dernier_traitement", "etablissement_siege", "nombre_periodes", "complement_adresse", "numero_voie", "indice_repetition", "type_voie", "libelle_voie", "code_postal", "libelle_commune", "libelle_commune_etranger", "distribution_speciale", "code_commune", "code_cedex", "libelle_cedex", "code_pays_etranger", "libelle_pays_etranger", "complement_adresse_2", "numero_voie_2", "indice_repetition_2", "type_voie_2", "libelle_voie_2", "code_postal_2", "libelle_commune_2", "libelle_commune_etranger_2", "distribution_speciale_2", "code_commune_2", "code_cedex_2", "libelle_cedex_2", "code_pays_etranger_2", "libelle_pays_etranger_2", "date_debut", "etat_administratif", "enseigne_1", "enseigne_2", "enseigne_3", "denomination_usuelle", "activite_principale", "nomenclature_activite_principale", "caractere_employeur", "longitude", "latitude", "geo_score", "geo_type", "geo_adresse", "geo_id", "geo_ligne", "geo_l4", "geo_l5", "unite_legale_id", "created_at", "updated_at", "unite_legale"}, {"Value.id", "Value.siren", "Value.nic", "Value.siret", "Value.statut_diffusion", "Value.date_creation", "Value.tranche_effectifs", "Value.annee_effectifs", "Value.activite_principale_registre_metiers", "Value.date_dernier_traitement", "Value.etablissement_siege", "Value.nombre_periodes", "Value.complement_adresse", "Value.numero_voie", "Value.indice_repetition", "Value.type_voie", "Value.libelle_voie", "Value.code_postal", "Value.libelle_commune", "Value.libelle_commune_etranger", "Value.distribution_speciale", "Value.code_commune", "Value.code_cedex", "Value.libelle_cedex", "Value.code_pays_etranger", "Value.libelle_pays_etranger", "Value.complement_adresse_2", "Value.numero_voie_2", "Value.indice_repetition_2", "Value.type_voie_2", "Value.libelle_voie_2", "Value.code_postal_2", "Value.libelle_commune_2", "Value.libelle_commune_etranger_2", "Value.distribution_speciale_2", "Value.code_commune_2", "Value.code_cedex_2", "Value.libelle_cedex_2", "Value.code_pays_etranger_2", "Value.libelle_pays_etranger_2", "Value.date_debut", "Value.etat_administratif", "Value.enseigne_1", "Value.enseigne_2", "Value.enseigne_3", "Value.denomination_usuelle", "Value.activite_principale", "Value.nomenclature_activite_principale", "Value.caractere_employeur", "Value.longitude", "Value.latitude", "Value.geo_score", "Value.geo_type", "Value.geo_adresse", "Value.geo_id", "Value.geo_ligne", "Value.geo_l4", "Value.geo_l5", "Value.unite_legale_id", "Value.created_at", "Value.updated_at", "Value.unite_legale"})
in
#"Value développé"
solved...my mistake:
let
siret = Excel.CurrentWorkbook(){[Name="entersiret"]}[Content],
paramsiret = Number.ToText(siret[Column1]{0}),
Source = Json.Document(Web.Contents("https://entreprise.data.gouv.fr/api/sirene/v3/etablissements/" & paramsiret)),
#"Converti en table" = Record.ToTable(Source),
#"Value développé" = Table.ExpandRecordColumn(#"Converti en table", "Value", {"id", "siren", "nic", "siret", "statut_diffusion", "date_creation", "tranche_effectifs", "annee_effectifs", "activite_principale_registre_metiers", "date_dernier_traitement", "etablissement_siege", "nombre_periodes", "complement_adresse", "numero_voie", "indice_repetition", "type_voie", "libelle_voie", "code_postal", "libelle_commune", "libelle_commune_etranger", "distribution_speciale", "code_commune", "code_cedex", "libelle_cedex", "code_pays_etranger", "libelle_pays_etranger", "complement_adresse_2", "numero_voie_2", "indice_repetition_2", "type_voie_2", "libelle_voie_2", "code_postal_2", "libelle_commune_2", "libelle_commune_etranger_2", "distribution_speciale_2", "code_commune_2", "code_cedex_2", "libelle_cedex_2", "code_pays_etranger_2", "libelle_pays_etranger_2", "date_debut", "etat_administratif", "enseigne_1", "enseigne_2", "enseigne_3", "denomination_usuelle", "activite_principale", "nomenclature_activite_principale", "caractere_employeur", "longitude", "latitude", "geo_score", "geo_type", "geo_adresse", "geo_id", "geo_ligne", "geo_l4", "geo_l5", "unite_legale_id", "created_at", "updated_at", "unite_legale"}, {"Value.id", "Value.siren", "Value.nic", "Value.siret", "Value.statut_diffusion", "Value.date_creation", "Value.tranche_effectifs", "Value.annee_effectifs", "Value.activite_principale_registre_metiers", "Value.date_dernier_traitement", "Value.etablissement_siege", "Value.nombre_periodes", "Value.complement_adresse", "Value.numero_voie", "Value.indice_repetition", "Value.type_voie", "Value.libelle_voie", "Value.code_postal", "Value.libelle_commune", "Value.libelle_commune_etranger", "Value.distribution_speciale", "Value.code_commune", "Value.code_cedex", "Value.libelle_cedex", "Value.code_pays_etranger", "Value.libelle_pays_etranger", "Value.complement_adresse_2", "Value.numero_voie_2", "Value.indice_repetition_2", "Value.type_voie_2", "Value.libelle_voie_2", "Value.code_postal_2", "Value.libelle_commune_2", "Value.libelle_commune_etranger_2", "Value.distribution_speciale_2", "Value.code_commune_2", "Value.code_cedex_2", "Value.libelle_cedex_2", "Value.code_pays_etranger_2", "Value.libelle_pays_etranger_2", "Value.date_debut", "Value.etat_administratif", "Value.enseigne_1", "Value.enseigne_2", "Value.enseigne_3", "Value.denomination_usuelle", "Value.activite_principale", "Value.nomenclature_activite_principale", "Value.caractere_employeur", "Value.longitude", "Value.latitude", "Value.geo_score", "Value.geo_type", "Value.geo_adresse", "Value.geo_id", "Value.geo_ligne", "Value.geo_l4", "Value.geo_l5", "Value.unite_legale_id", "Value.created_at", "Value.updated_at", "Value.unite_legale"})
in
#"Value développé"
I'm trying to merge several columns in a table to columns of another table. Each column in the primary table contains texts while the PrimaryAnalysis table contains indexes for the texts. I'd like to create columns of indexes for the primary table but I'm having to do that one at a time for each table thus:
#"Merged Queries" = Table.NestedJoin(#"Changed Type2",{"Text.1"},PrimaryAnalysis,{"Letter"},"NewColumn"),
#"Expanded NewColumn" = Table.ExpandTableColumn(#"Merged Queries", "NewColumn", {"Index"}, {"Index"}),
#"Renamed Columns2" = Table.RenameColumns(#"Expanded NewColumn",{{"Index", "First"}}),
#"Merged Queries1" = Table.NestedJoin(#"Renamed Columns2",{"Text.2"},PrimaryAnalysis,{"Letter"},"NewColumn"),
#"Expanded NewColumn1" = Table.ExpandTableColumn(#"Merged Queries1", "NewColumn", {"Index"}, {"Index"}),
#"Renamed Columns3" = Table.RenameColumns(#"Expanded NewColumn1",{{"Index", "2nd"}}),
#"Merged Queries2" = Table.NestedJoin(#"Renamed Columns3",{"Text.3"},PrimaryAnalysis,{"Letter"},"NewColumn"),
#"Expanded NewColumn2" = Table.ExpandTableColumn(#"Merged Queries2", "NewColumn", {"Index"}, {"Index"}),
#"Renamed Columns4" = Table.RenameColumns(#"Expanded NewColumn2",{{"Index", "3rd"}}),
Now I have to do that for 23 columns. Is there a way to implement DO...Repeat or any other loop in Power Query to perform this task?
Thanks in advance.
One way to loop in Power Query is to use a recursive function.
In the code below I read an Excel file with a table that should be similar to your primary table (so the step #"Changed Type2" in the code below should be similar to your step #"Changed Type2").
Next a function AddIndices is defined in which 1 column with an Index is added in each iteration. After 23 iterations the function stops, otherwise it calls itself.
An important point of attention with such recursive functions is that it MUST include a Table.Buffer (see step "Expanded"), otherwise in each iteration the code tries to evaluate all former iterations again and gets stuck. Table.Buffer prevents this.
In the last step of the query, the function is invoked.
let
Source = Excel.Workbook(File.Contents("C:\Users\Marcel\Documents\Forum bijdragen\StackOverflow Power Query\Loop Computation in Power Query.xlsx"), null, true),
Tabel1_Table = Source{[Item="Tabel1",Kind="Table"]}[Data],
#"Changed Type2" = Table.TransformColumnTypes(Tabel1_Table,{{"Text.1", type text}, {"Text.2", type text}, {"Text.3", type text}, {"Text.4", type text}, {"Text.5", type text}, {"Text.6", type text}, {"Text.7", type text}, {"Text.8", type text}, {"Text.9", type text}, {"Text.10", type text}, {"Text.11", type text}, {"Text.12", type text}, {"Text.13", type text}, {"Text.14", type text}, {"Text.15", type text}, {"Text.16", type text}, {"Text.17", type text}, {"Text.18", type text}, {"Text.19", type text}, {"Text.20", type text}, {"Text.21", type text}, {"Text.22", type text}, {"Text.23", type text}}),
// Recursive function:
AddIndices = (TableSoFar as table, optional Iteration as number) as table =>
let
CurrentIteration = if Iteration = null then 1 else Iteration,
CurrentColumn = "Text."&Text.From(CurrentIteration),
NewIndexColumn = "Index."&Text.From(CurrentIteration),
MergedTable = Table.NestedJoin(TableSoFar,{CurrentColumn},PrimaryAnalysis,{"Letter"},"NewColumn"),
Expanded = Table.Buffer(Table.ExpandTableColumn(MergedTable, "NewColumn", {"Index"}, {NewIndexColumn})),
Result = if CurrentIteration = 23 then Expanded else #AddIndices(Expanded, CurrentIteration + 1)
in
Result,
// Call recursive function:
AddedIndices = AddIndices(#"Changed Type2")
in
AddedIndices