it is my first use of power query and I need your help
I'd like to retrieve from web some compagny values base on their identification number, the french SIRET
for that I'd like to enter the SIRET in a cell (named entersiret) in my excel and then pass it to the url and in power query. I have tried this but it is not working :
let
siret = Excel.CurrentWorkbook(){[Name="entersiret"]}[Content],
entersiret = siret[Column1]{0},
Source = Json.Document(Web.Contents("https://entreprise.data.gouv.fr/api/sirene/v3/etablissements/" & siret)),
#"Converti en table" = Record.ToTable(Source),
#"Value développé" = Table.ExpandRecordColumn(#"Converti en table", "Value", {"id", "siren", "nic", "siret", "statut_diffusion", "date_creation", "tranche_effectifs", "annee_effectifs", "activite_principale_registre_metiers", "date_dernier_traitement", "etablissement_siege", "nombre_periodes", "complement_adresse", "numero_voie", "indice_repetition", "type_voie", "libelle_voie", "code_postal", "libelle_commune", "libelle_commune_etranger", "distribution_speciale", "code_commune", "code_cedex", "libelle_cedex", "code_pays_etranger", "libelle_pays_etranger", "complement_adresse_2", "numero_voie_2", "indice_repetition_2", "type_voie_2", "libelle_voie_2", "code_postal_2", "libelle_commune_2", "libelle_commune_etranger_2", "distribution_speciale_2", "code_commune_2", "code_cedex_2", "libelle_cedex_2", "code_pays_etranger_2", "libelle_pays_etranger_2", "date_debut", "etat_administratif", "enseigne_1", "enseigne_2", "enseigne_3", "denomination_usuelle", "activite_principale", "nomenclature_activite_principale", "caractere_employeur", "longitude", "latitude", "geo_score", "geo_type", "geo_adresse", "geo_id", "geo_ligne", "geo_l4", "geo_l5", "unite_legale_id", "created_at", "updated_at", "unite_legale"}, {"Value.id", "Value.siren", "Value.nic", "Value.siret", "Value.statut_diffusion", "Value.date_creation", "Value.tranche_effectifs", "Value.annee_effectifs", "Value.activite_principale_registre_metiers", "Value.date_dernier_traitement", "Value.etablissement_siege", "Value.nombre_periodes", "Value.complement_adresse", "Value.numero_voie", "Value.indice_repetition", "Value.type_voie", "Value.libelle_voie", "Value.code_postal", "Value.libelle_commune", "Value.libelle_commune_etranger", "Value.distribution_speciale", "Value.code_commune", "Value.code_cedex", "Value.libelle_cedex", "Value.code_pays_etranger", "Value.libelle_pays_etranger", "Value.complement_adresse_2", "Value.numero_voie_2", "Value.indice_repetition_2", "Value.type_voie_2", "Value.libelle_voie_2", "Value.code_postal_2", "Value.libelle_commune_2", "Value.libelle_commune_etranger_2", "Value.distribution_speciale_2", "Value.code_commune_2", "Value.code_cedex_2", "Value.libelle_cedex_2", "Value.code_pays_etranger_2", "Value.libelle_pays_etranger_2", "Value.date_debut", "Value.etat_administratif", "Value.enseigne_1", "Value.enseigne_2", "Value.enseigne_3", "Value.denomination_usuelle", "Value.activite_principale", "Value.nomenclature_activite_principale", "Value.caractere_employeur", "Value.longitude", "Value.latitude", "Value.geo_score", "Value.geo_type", "Value.geo_adresse", "Value.geo_id", "Value.geo_ligne", "Value.geo_l4", "Value.geo_l5", "Value.unite_legale_id", "Value.created_at", "Value.updated_at", "Value.unite_legale"})
in
#"Value développé"
You drill down the value "siret" in "entersiret" but you don't use it. This will work:
let
siret = Excel.CurrentWorkbook(){[Name="entersiret"]}[Content],
entersiret = siret[Column1]{0},
Source = Json.Document(Web.Contents("https://entreprise.data.gouv.fr/api/sirene/v3/etablissements/" & entersiret)),
#"Converti en table" = Record.ToTable(Source),
#"Value développé" = Table.ExpandRecordColumn(#"Converti en table", "Value", {"id", "siren", "nic", "siret", "statut_diffusion", "date_creation", "tranche_effectifs", "annee_effectifs", "activite_principale_registre_metiers", "date_dernier_traitement", "etablissement_siege", "nombre_periodes", "complement_adresse", "numero_voie", "indice_repetition", "type_voie", "libelle_voie", "code_postal", "libelle_commune", "libelle_commune_etranger", "distribution_speciale", "code_commune", "code_cedex", "libelle_cedex", "code_pays_etranger", "libelle_pays_etranger", "complement_adresse_2", "numero_voie_2", "indice_repetition_2", "type_voie_2", "libelle_voie_2", "code_postal_2", "libelle_commune_2", "libelle_commune_etranger_2", "distribution_speciale_2", "code_commune_2", "code_cedex_2", "libelle_cedex_2", "code_pays_etranger_2", "libelle_pays_etranger_2", "date_debut", "etat_administratif", "enseigne_1", "enseigne_2", "enseigne_3", "denomination_usuelle", "activite_principale", "nomenclature_activite_principale", "caractere_employeur", "longitude", "latitude", "geo_score", "geo_type", "geo_adresse", "geo_id", "geo_ligne", "geo_l4", "geo_l5", "unite_legale_id", "created_at", "updated_at", "unite_legale"}, {"Value.id", "Value.siren", "Value.nic", "Value.siret", "Value.statut_diffusion", "Value.date_creation", "Value.tranche_effectifs", "Value.annee_effectifs", "Value.activite_principale_registre_metiers", "Value.date_dernier_traitement", "Value.etablissement_siege", "Value.nombre_periodes", "Value.complement_adresse", "Value.numero_voie", "Value.indice_repetition", "Value.type_voie", "Value.libelle_voie", "Value.code_postal", "Value.libelle_commune", "Value.libelle_commune_etranger", "Value.distribution_speciale", "Value.code_commune", "Value.code_cedex", "Value.libelle_cedex", "Value.code_pays_etranger", "Value.libelle_pays_etranger", "Value.complement_adresse_2", "Value.numero_voie_2", "Value.indice_repetition_2", "Value.type_voie_2", "Value.libelle_voie_2", "Value.code_postal_2", "Value.libelle_commune_2", "Value.libelle_commune_etranger_2", "Value.distribution_speciale_2", "Value.code_commune_2", "Value.code_cedex_2", "Value.libelle_cedex_2", "Value.code_pays_etranger_2", "Value.libelle_pays_etranger_2", "Value.date_debut", "Value.etat_administratif", "Value.enseigne_1", "Value.enseigne_2", "Value.enseigne_3", "Value.denomination_usuelle", "Value.activite_principale", "Value.nomenclature_activite_principale", "Value.caractere_employeur", "Value.longitude", "Value.latitude", "Value.geo_score", "Value.geo_type", "Value.geo_adresse", "Value.geo_id", "Value.geo_ligne", "Value.geo_l4", "Value.geo_l5", "Value.unite_legale_id", "Value.created_at", "Value.updated_at", "Value.unite_legale"})
in
#"Value développé"
solved...my mistake:
let
siret = Excel.CurrentWorkbook(){[Name="entersiret"]}[Content],
paramsiret = Number.ToText(siret[Column1]{0}),
Source = Json.Document(Web.Contents("https://entreprise.data.gouv.fr/api/sirene/v3/etablissements/" & paramsiret)),
#"Converti en table" = Record.ToTable(Source),
#"Value développé" = Table.ExpandRecordColumn(#"Converti en table", "Value", {"id", "siren", "nic", "siret", "statut_diffusion", "date_creation", "tranche_effectifs", "annee_effectifs", "activite_principale_registre_metiers", "date_dernier_traitement", "etablissement_siege", "nombre_periodes", "complement_adresse", "numero_voie", "indice_repetition", "type_voie", "libelle_voie", "code_postal", "libelle_commune", "libelle_commune_etranger", "distribution_speciale", "code_commune", "code_cedex", "libelle_cedex", "code_pays_etranger", "libelle_pays_etranger", "complement_adresse_2", "numero_voie_2", "indice_repetition_2", "type_voie_2", "libelle_voie_2", "code_postal_2", "libelle_commune_2", "libelle_commune_etranger_2", "distribution_speciale_2", "code_commune_2", "code_cedex_2", "libelle_cedex_2", "code_pays_etranger_2", "libelle_pays_etranger_2", "date_debut", "etat_administratif", "enseigne_1", "enseigne_2", "enseigne_3", "denomination_usuelle", "activite_principale", "nomenclature_activite_principale", "caractere_employeur", "longitude", "latitude", "geo_score", "geo_type", "geo_adresse", "geo_id", "geo_ligne", "geo_l4", "geo_l5", "unite_legale_id", "created_at", "updated_at", "unite_legale"}, {"Value.id", "Value.siren", "Value.nic", "Value.siret", "Value.statut_diffusion", "Value.date_creation", "Value.tranche_effectifs", "Value.annee_effectifs", "Value.activite_principale_registre_metiers", "Value.date_dernier_traitement", "Value.etablissement_siege", "Value.nombre_periodes", "Value.complement_adresse", "Value.numero_voie", "Value.indice_repetition", "Value.type_voie", "Value.libelle_voie", "Value.code_postal", "Value.libelle_commune", "Value.libelle_commune_etranger", "Value.distribution_speciale", "Value.code_commune", "Value.code_cedex", "Value.libelle_cedex", "Value.code_pays_etranger", "Value.libelle_pays_etranger", "Value.complement_adresse_2", "Value.numero_voie_2", "Value.indice_repetition_2", "Value.type_voie_2", "Value.libelle_voie_2", "Value.code_postal_2", "Value.libelle_commune_2", "Value.libelle_commune_etranger_2", "Value.distribution_speciale_2", "Value.code_commune_2", "Value.code_cedex_2", "Value.libelle_cedex_2", "Value.code_pays_etranger_2", "Value.libelle_pays_etranger_2", "Value.date_debut", "Value.etat_administratif", "Value.enseigne_1", "Value.enseigne_2", "Value.enseigne_3", "Value.denomination_usuelle", "Value.activite_principale", "Value.nomenclature_activite_principale", "Value.caractere_employeur", "Value.longitude", "Value.latitude", "Value.geo_score", "Value.geo_type", "Value.geo_adresse", "Value.geo_id", "Value.geo_ligne", "Value.geo_l4", "Value.geo_l5", "Value.unite_legale_id", "Value.created_at", "Value.updated_at", "Value.unite_legale"})
in
#"Value développé"
Related
The data has HTML values inside text:
col1
-------------------------------------------------------------
Drell-Yan Process Background Estimation Using eμ Method
Expressions of constant π
Computational Analysis of Protein β-Structure
δ13C and 14C Measurements in Aerosol Particles
I need to get actual symbols instead of all the HTML decimal values.
Html.Table in powerquery-m can decode HTML decimal values into visible symbols:
let
Source = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("PY7BCsIwDIZfJcyrg03nUHaa06MwEA8ydyg1arFrR5KCvr1l4C7JId/3/+m65EBobXpVDlryGplhr/T7ST64OxxZzKDEeAcXNu4JeAtZtioXu01ZwQnl5e9Jv+yS42ek6EaQwT9Axy3KCfzxMqsmrvHDGGRKVBbqOL5sJiW2Cxo3G0VZpWehoCUQTu582Vb5ugFDkBdNfEJxJAZ0whD9Gsmzt9AqEqMtctL3Pw==", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [col1 = _t]),
#"Changed Type" = Table.TransformColumnTypes(Source,{{"col1", type text}}),
#"Added Custom" = Table.AddColumn(#"Changed Type", "HtmlTable", each Html.Table([col1],{{"HtmlDecoded",":root"}})),
#"Expanded HtmlTable" = Table.ExpandTableColumn(#"Added Custom", "HtmlTable", {"HtmlDecoded"}, {"HtmlDecoded"})
in
#"Expanded HtmlTable"
Or in place:
let
Source = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("PY7BCsIwDIZfJcyrg03nUHaa06MwEA8ydyg1arFrR5KCvr1l4C7JId/3/+m65EBobXpVDlryGplhr/T7ST64OxxZzKDEeAcXNu4JeAtZtioXu01ZwQnl5e9Jv+yS42ek6EaQwT9Axy3KCfzxMqsmrvHDGGRKVBbqOL5sJiW2Cxo3G0VZpWehoCUQTu582Vb5ugFDkBdNfEJxJAZ0whD9Gsmzt9AqEqMtctL3Pw==", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [col1 = _t]),
#"Changed Type" = Table.TransformColumnTypes(Source,{{"col1", type text}}),
Decoded = Table.TransformColumns(#"Changed Type", {{"col1", each Table.FirstValue(Html.Table(_,{{"HtmlDecoded",":root"}})) }} )
in
Decoded
SSIS supports Power Query as a Source, but do not seem to support M code using Html.Table. I'm using only SQL Server Data Tools (SSDT)
M code below=
let Scrapper =
(Page as number) as table =>
let
Source = Web.BrowserContents("https://www.zerohedge.com/?page=" & Number.ToText(Page)),
#"Extracted Table From Html" = Html.Table(Source, {{"Title", ".teaser-title:nth-last-child(4)"}, {"Date", ".extras__created:nth-last-child(1)"}, {"Views", ".extras__views:nth-last-child(2)"}}, [RowSelector=".view-content:nth-last-child(2) > DIV.views-row"]),
#"Changed Type" = Table.TransformColumnTypes(#"Extracted Table From Html",{{"Title", type text}, {"Date", type datetime}, {"Views", Int64.Type}})
in
#"Changed Type",
Source = {0..1},
#"Converted to Table" = Table.FromList(Source, Splitter.SplitByNothing(), null, null, ExtraValues.Error),
#"Renamed Columns" = Table.RenameColumns(#"Converted to Table",{{"Column1", "Pages"}}),
#"Invoked Custom Function" = Table.AddColumn(#"Renamed Columns", "Scrapper", each Scrapper([Pages])),
#"Expanded Scrapper" = Table.ExpandTableColumn(#"Invoked Custom Function", "Scrapper", {"Title", "Date", "Views"}, {"Title", "Date", "Views"})
in
#"Expanded Scrapper"
Error: 0x0 at Data Flow Task, Power Query Source: The import
Html.Table matches no exports. Did you miss a module reference?
I don't see any other solution that wait for Microsoft to support Html.table
error image in SSDT
I have already studied the case
Quantmod: Error loading symbols from MySQL DB
and already try to fix the getSymbols.MySQL function in R
However, I found that my database just contain
date, open, high, low, close, volume (without the close.adj column).
So, if I want to further modify the getSymbols.MySQL function, what can I do?
I have tried to use 'fix(getSymbols.MySQL)' to fix the function. However, it returns
Error in colnames<-(*tmp*, value = c("H0001.Open", "H0001.High", "H0001.Low", : length of 'dimnames' [2] not equal to array extent
when I connect to my database.
function (Symbols, env, return.class = "xts", db.fields = c("date",
"o", "h", "l", "c", "v", "a"), field.names = NULL, user = NULL,
password = NULL, dbname = NULL, host = "localhost", port = 3306,
...)
{
importDefaults("getSymbols.MySQL")
this.env <- environment()
for (var in names(list(...))) {
assign(var, list(...)[[var]], this.env)
}
if (!hasArg(verbose))
verbose <- FALSE
if (!hasArg(auto.assign))
auto.assign <- TRUE
if (!requireNamespace("DBI", quietly = TRUE))
stop("package:", dQuote("DBI"), "cannot be loaded.")
if (!requireNamespace("RMySQL", quietly = TRUE))
stop("package:", dQuote("RMySQL"), "cannot be loaded.")
if (is.null(user) || is.null(password) || is.null(dbname)) {
stop(paste("At least one connection argument (", sQuote("user"),
sQuote("password"), sQuote("dbname"), ") is not set"))
}
con <- DBI::dbConnect("MySQL", user = user, password = password,
dbname = dbname, host = host, port = port)
db.Symbols <- DBI::dbListTables(con)
if (length(Symbols) != sum(Symbols %in% db.Symbols)) {
missing.db.symbol <- Symbols[!Symbols %in% db.Symbols]
warning(paste("could not load symbol(s): ", paste(missing.db.symbol,
collapse = ", ")))
Symbols <- Symbols[Symbols %in% db.Symbols]
}
for (i in 1:length(Symbols)) {
if (verbose) {
cat(paste("Loading ", Symbols[[i]], paste(rep(".",
10 - nchar(Symbols[[i]])), collapse = ""), sep = ""))
}
query <- paste("SELECT ", paste(db.fields, collapse = ","),
" FROM ", Symbols[[i]], " ORDER BY date")
rs <- DBI::dbSendQuery(con, query)
fr <- DBI::fetch(rs, n = -1)
fr <- xts(as.matrix(fr[, -1]), order.by = as.Date(fr[,
1], origin = "1970-01-01"), src = dbname, updated = Sys.time())
colnames(fr) <- paste(Symbols[[i]], c("Open", "High",
"Low", "Close", "Volume", "Adjusted"), sep = ".")
fr <- convert.time.series(fr = fr, return.class = return.class)
if (auto.assign)
assign(Symbols[[i]], fr, env)
if (verbose)
cat("done\n")
}
DBI::dbDisconnect(con)
if (auto.assign)
return(Symbols)
return(fr)
}
I think the problem is the function was designed to read 7 column of data rather than 6 column of data. Hope someone can help.
Here's a patch that should allow you to do what you want. I'm unable to test because I don't have a MySQL installation to test against. Please let me know whether or not it works.
diff --git a/R/getSymbols.R b/R/getSymbols.R
index 0a2e814..7a9be66 100644
--- a/R/getSymbols.R
+++ b/R/getSymbols.R
## -634,9 +634,9 ## function(Symbols,env,return.class='xts',
fr <- xts(as.matrix(fr[,-1]),
order.by=as.Date(fr[,1],origin='1970-01-01'),
src=dbname,updated=Sys.time())
- colnames(fr) <- paste(Symbols[[i]],
- c('Open','High','Low','Close','Volume','Adjusted'),
- sep='.')
+ if(is.null(field.names))
+ field.names <- c('Open','High','Low','Close','Volume','Adjusted')
+ colnames(fr) <- paste(Symbols[[i]], field.names, sep='.')
fr <- convert.time.series(fr=fr,return.class=return.class)
if(auto.assign)
assign(Symbols[[i]],fr,env)
Then your function call should be:
getSymbols.MySQL("H0001", env, return.class = 'xts',
db.fields = c("date", "open", "high", "low", "close", "volume"),
field.names = c("date", "open", "high", "low", "close", "volume"),
user = 'xxxx', password = 'xxxx', host='xxxx', dbname = 'xxxx')
I am trying to extract some data that is being returned in a nested hash from Mailchimp. Here is a trimmed down version of the results I am getting. For each email there are multiple GROUPINGS and for each GROUPING there are multiple GROUPS.
My objective is to get this into a mysql table with a layout like: email_addr, list, grouping1_id, grouping1_name, group1_name, group1_interest, group2_name, group2_interest, grouping2_id, grouping2_name, etc. So there is one row per subscriber with all the grouping and group information.
{"email"=>"dummy#gmail.com", "merges"=>{"EMAIL"=>"dummy#gmail.com",
"GROUPINGS"=>[{"id"=>1, "name"=>"Grouping One", "groups"=>[{"name"=>"Group One",
"interested"=>false}, {"name"=>"Group", "interested"=>true},
{"name"=>"Group Three", "interested"=>true}]}, {"id"=>2, "name"=>"Grouping Two",
"groups"=>[{"name"=>"Group Four", "interested"=>false},
{"name"=>"Group Five", "interested"=>false}]}]}}
Right now, the code I have below runs and inserts the results of the nested blocks into the table but there is one row for each pass through the groups.each_with_index statement. My approach so far seems overly complicated but I am not sure of how to approach this to process the data correctly.
Any help is appreciated.
UPDATED:
I cleaned up the logic a bit and separated the database writes into each level of the hash processing. Now the data is inserted and updated in the database correctly. Although this still feels very inelegant.
def organize_members_subs
#members_data = #members_subs["data"]
#members_data.each do |member|
#email_addr = member["email"]
#db.query("INSERT INTO db.details
(email_addr, list)
VALUES ('#{#email_addr}', '#{#list}' ) ")
groupings = member["merges"]["GROUPINGS"]
groupings.each_with_index do |grouping, index|
#groupings_name = grouping["name"]
#groupings_id = grouping["id"]
#groupings_label = "grp#{index}_"
#db.query("UPDATE db.details
SET grouping#{index}_id = '#{#groupings_id}'
, grouping#{index}_name = '#{#groupings_name}'
WHERE email_addr = '#{#email_addr}' ")
groups = member["merges"]["GROUPINGS"][index]["groups"]
groups.each_with_index do |group, index|
#group_name = group["name"]
#group_interested = group["interested"]
#db.query("UPDATE db.details
SET #{#groupings_label}group#{index}_name = '#{#group_name}'
, #{#groupings_label}group#{index}_int = '#{#group_interested}'
WHERE email_addr = '#{#email_addr}' ")
break if index == groups.length
end
break if index == groupings.length
end
end
end
To start, I wanted to take a closer at look your hash. Rather than reformatting it myself, I did this:
require "awesome_print"
h = `{"email"=>..., "interested"=>false}]}]}}`
ap h
Scroll down to the bottom of my answer to see ap's formatting of the hash.
I will answer your question assuming the db structure is a given, but would like to make a few points:
If "id" is unique for each grouping record, could you make that the key, and dispense with index?
If "name" is unique for each grouping record, could you dispense with both "id" and index?
If "name" is unique for each group record (for a given grouping), could you just have group["name"] => group["interested"] for each group?
Moving on to your code, I will also assume the structure of your hash is given. Later, I will revisit that assumption.
The changes I propose to your code are fairly minor and some are purely stylistic:
Make all instance variables local variables, meaning that two additional arguments must be passed to def organize_members_subs.
With two possible exceptions, eliminate local variables that are only used once after being defined. For example, rather than
groupings_id = grouping["id"], then SET grouping#{index}_id = '#{#groupings_id}', just have SET grouping#{index}_id = '#{grouping["id"]}'.
The two possible exceptions are groupings and groups. For example, you could get rid of the former by writing
member["merges"]["GROUPINGS"].each_with_index do |grouping, index_1|. I'd keep them as variables (so I could easily check their values),
but that's a stylistic decision.
The variable index in groupings.each_with_index do |grouping, index| is in scope within the inner block, which uses an iterator variable with the same name.
I presume that the latter takes precedence, but they should be named differently. I've changed them to index_out and index_in, respectively.
index_out ranges from 0 to groupings.length-1, so break if index_out == groupings.length will never be executed, and therefore may be removed. Ditto for break if index_in == groups.length.
I moved groupings_label = "grp#{index}_" down to draw attention to the fact that it is needed only later, not in the preceding SET expression.
These changes result in the following:
def organize_members_subs(db, list, #members_subs["data"])
members_data.each do |member|
email_addr = member["email"]
db.query("INSERT INTO db.details
(email_addr, list)
VALUES ('#{email_addr}', '#{list}' ) ")
groupings = member["merges"]["GROUPINGS"]
groupings.each_with_index do |grouping, index_out|
db.query("UPDATE db.details
SET grouping#{index_out}_id = '#{grouping["id"]}'
, grouping#{index_out}_name = '#{grouping["name"]}'
WHERE email_addr = '#{email_addr}' ")
groupings_label = "grp#{index_out}_"
groups = member["merges"]["GROUPINGS"][index_out]["groups"]
groups.each_with_index do |group, index_in|
db.query("UPDATE db.details
SET #{groupings_label}group#{index_in}_name = '#{group["name"]}'
, #{groupings_label}group#{index_in}_int = '#{group["interested"]}'
WHERE email_addr = '#{email_addr}' ")
end
end
end
end
Looking at your hash, I am wondering if you could simplify it to the following (formatting courtesy of awesome print):
{
"email" => "dummy#gmail.com",
"merges" => {
"EMAIL" => "dummy#gmail.com",
"GROUPINGS" => {
1 => {
"name" => "Grouping One",
"groups" => {
"Group One" => false,
"Group Two" => true,
"Group Three" => true
}
},
2 => {
"name" => "Grouping Two",
"groups" => {
"Group Four" => false,
"Group Five" => false
}
}
}
}
}
or even
{
"email" => "dummy#gmail.com",
"merges" => {
"EMAIL" => "dummy#gmail.com",
"GROUPINGS" => {
"Grouping One" => {
"Group One" => false,
"Group Two" => true,
"Group Three" => true
},
"Grouping Two" => {
"Group Four" => false,
"Group Five" => false
}
}
}
}
These are not so much as suggestions, but just food for thought.
Awesome print applied to your hash:
ap h # =>
{
"email" => "dummy#gmail.com",
"merges" => {
"EMAIL" => "dummy#gmail.com",
"GROUPINGS" => [
[0] {
"id" => 1,
"name" => "Grouping One",
"groups" => [
[0] {
"name" => "Group One",
"interested" => false
},
[1] {
"name" => "Group",
"interested" => true
},
[2] {
"name" => "Group Three",
"interested" => true
}
]
},
[1] {
"id" => 2,
"name" => "Grouping Two",
"groups" => [
[0] {
"name" => "Group Four",
"interested" => false
},
[1] {
"name" => "Group Five",
"interested" => false
}
]
}
]
}
}
First, maybe extra, but I like to work with symbols since I do a lot of my work in Rails. So let's steal a method from here: How do I convert a Ruby hash so that all of its keys are symbols?
def recursive_symbolize_keys(h)
case h
when Hash
Hash[
h.map do |k, v|
[ k.respond_to?(:to_sym) ? k.to_sym : k, recursive_symbolize_keys(v) ]
end
]
when Enumerable
h.map { |v| recursive_symbolize_keys(v) }
else
h
end
end
OK, lets build a class to make this easier to manipulate and extend as our needs change:
class MemberSub
attr_accessor :email, :groupings, :data_hash, :list, :data_row, :db_sql
def initialize(data_hash)
#convert all keys to symbols
#data_hash = recursive_symbolize_keys(data_hash)
#email = #data_hash[:email]
#list = 'Members'
#groupings = #data_hash[:merges][:GROUPINGS]
#data_row = data_row
#db_sql = db_insert
end
def data_row
#returns a data row for DB
row_hash = {}
row_hash['email'] = #email
row_hash['list'] = #list
gc = 1
#iterate through groupings
#groupings.each_with_index do |grouping, index|
row_hash["grouping#{index + 1}_id"] = grouping[:id]
row_hash["grouping#{index + 1}_name"] = grouping[:name]
#iterate through the groups
grouping[:groups].each do |group|
row_hash["group#{gc}_name"] = group[:name]
row_hash["group#{gc}_interest"] = group[:interested]
gc += 1
end
end
row_hash
end
def db_insert
"INSERT INTO db.details (#{#data_row.keys}) VALUES (#{#data_row.values})".tr('[]','')
end
end
Now you can do feed it a row using what ever iteration method and make a new object:
row = MemberSub.new({"email"=>"dummy#gmail.com", "list"=>"Members", "merges"=>
{"EMAIL"=>"dummy#gmail.com", "GROUPINGS"=>[{"id"=>1, "name"=>"Grouping One", "groups"=>
[{"name"=>"Group One", "interested"=>false}, {"name"=>"Group Two", "interested"=>true},
{"name"=>"Group Three", "interested"=>true}]}, {"id"=>2, "name"=>"Grouping Two", "groups"=>
[{"name"=>"Group Four", "interested"=>false}, {"name"=>"Group Five", "interested"=>false}]}]}})
and make a query:
db.query(row.db_sql)
db.query(INSERT INTO db.details ("email", "list", "grouping1_id", "grouping1_name",
"group1_name", "group1_interest", "group2_name", "group2_interest", "group3_name",
"group3_interest", "grouping2_id", "grouping2_name", "group4_name", "group4_interest",
"group5_name", "group5_interest") VALUES ("dummy#gmail.com", "Members", 1, "Grouping One",
"Group One", false, "Group Two", true, "Group Three", true, 2, "Grouping Two", "Group Four",
false, "Group Five", false))
The other methods should be self explanatory. You don't have to have them all available as attar_accessor but I just did that for example.
I checked out the questions that have already been asked on this subject, "There are many", but I do not find a solution.
I have a fairly large task and the files name is 'sample_employee_data.rake'... so here goes:
namespace :db do
desc "Fill database with sample Employee data"
task populate: :environment do
#gender = ["Male", "Female"]
#role = ["Staff", "Support", "Team_Leader", "Manager", "Director"]
#marital_status = ["Single", "Married"]
#primary_position = ["Household", "Job Developer", "Job Coach",
"Job Support", "Personal Care"]
#trained_position = ["Household", "Job Developer", "Job Coach",
"Job Support", "Personal Care"]
#emer_contact_relationship = ["Friend", "Brother", "Sister", "Aunt",
"Uncle", "Cousin", "Nephew", "Father",
"Mother", "Spouse"]
def randomDate(params={})
years_back = params[:year_range] || 5
latest_year = params[:year_latest] || 0
year = (rand * (years_back)).ceil +
(Time.now.year - latest_year - years_back)
month = (rand * 12).ceil
day = (rand * 31).ceil
series = [date = Time.local(year, month, day)]
if params[:series]
params[:series].each do |some_time_after|
series << series.last + (rand * some_time_after).ceil
end
return series
end
date
end
Employee.create!(first_name: "Shelly",
last_name: "Houghton",
mi: "M",
full_name: "Shelly M Houghton",
marital_status: "Single",
gender: "Female",
hire_date: "2000-04-16",
primary_position: "Manager",
trained_position: "Job Developer",
email: "shoughton#example.com",
active: true,
address1: "76th Ave",
city: "Frave",
zip_code: "54806",
state: "WI",
emp_home_ph: "1-111-111-1111",
emp_mobile_ph: "1-222-222-2222",
emer_contact_first_name: "Kenneth",
emer_contact_last_name: "Koening",
emer_contact_relationship: "Friend",
emer_contact_ph: "1-333-333-3333",
role: "Manager",
birth_date: "1982-08-21",
admin: true,
password: "90nothguoh",
password_confirmation: "90nothguoh")
99.times do |n|
first_name = Faker::Name.first_name
last_name = Faker::Name.last_name
mi = ("A".."Z").to_a[rand(26)]
full_name = Faker::Name.full_name
marital_status = #marital_status[rand(2)].to_s
gender = #gender[rand(2)].to_s
hire_date = randomDate(:year_range => 60, :year_latest => 12)
birth_date = randomDate(:year_range => 60, :year_latest => 22)
primary_position = #primary_position[rand(5)].to_s
trained_position = #trained_position[rand(5)].to_s
email = "emp-#{n+1}#example.org"
active = [true, false][rand(2)]
admin = (1 == rand(2) ? true : false)
role = #role[rand(5)].to_s
address1 = "Seaview-#{n+5}Way"
city = Faker::Lorem.words(1).to_s.capitalize
state = Faker::Address.us_state()
zip_code = Faker::Address.zip_code
emp_home_ph = Faker::PhoneNumber.phone_number
emp_mobile_ph = Faker::PhoneNumber.phone_number
emer_contact_first_name = Faker::Name.first_name
emer_contact_last_name = Faker::Name.last_name
emer_contact_relationship = #emer_contact_relationship[rand(10)].to_s
emer_contact_ph = Faker::PhoneNumber.phone_number
password = "uniqueone"
Employee.create!(first_name: first_name, mi: mi, last_name: last_name,
full_name: full_name, marital_status: marital_status,
gender: gender, birth_date: birth_date, hire_date: hire_date,
primary_position: primary_position, trained_position:
trained_position, email: email, role: role, address1:
address1, city: city, state: state, zip_code: zip_code,
emp_home_ph: emp_home_ph, emp_mobile_ph: emp_mobile_ph,
emer_contact_first_name: emer_contact_first_name,
emer_contact_last_name: emer_contact_last_name,
emer_contact_relationship: emer_contact_relationship,
emer_contact_ph: emer_contact_ph, password: password,
password_confirmation: password)
end
end
end
I ran:
rake sample_employee_data.rake
and I got this flag:
rake aborted!
Don't know how to build task 'sample_employee_data.rake'
home/me/.rvm/gems/ruby-1.9.3-p385#rails3212/bin/ruby_noexec_wrapper:14:in 'eval'
home/me/.rvm/gems/ruby-1.9.3-p385#rails3212/bin/ruby_noexec_wrapper:14:in '<main>'
Can anyone spot the problem ... my apologies for such a long file.
Thanks.
There's a couple things going on here.
When you run $ rake sample_employee_data.rake you're asking rake to rake your file - but rake responds to tasks:
$ rake some_namespace:some_task_in_that_namespace
You're also using a custom .rake file rather than the conventional Rakefile - which is fine, but this means you need to tell rake explicitly that you're not using its default Rakefile.
$ rake --rakefile your_custom_rake_file.rake some_task
So in your case, you need to issue a rake command like so:
$ rake --rakefile sample_employee_data.rake db:populate