Faker "Don't Know How to Build Task? - rake-task

I checked out the questions that have already been asked on this subject, "There are many", but I do not find a solution.
I have a fairly large task and the files name is 'sample_employee_data.rake'... so here goes:
namespace :db do
desc "Fill database with sample Employee data"
task populate: :environment do
#gender = ["Male", "Female"]
#role = ["Staff", "Support", "Team_Leader", "Manager", "Director"]
#marital_status = ["Single", "Married"]
#primary_position = ["Household", "Job Developer", "Job Coach",
"Job Support", "Personal Care"]
#trained_position = ["Household", "Job Developer", "Job Coach",
"Job Support", "Personal Care"]
#emer_contact_relationship = ["Friend", "Brother", "Sister", "Aunt",
"Uncle", "Cousin", "Nephew", "Father",
"Mother", "Spouse"]
def randomDate(params={})
years_back = params[:year_range] || 5
latest_year = params[:year_latest] || 0
year = (rand * (years_back)).ceil +
(Time.now.year - latest_year - years_back)
month = (rand * 12).ceil
day = (rand * 31).ceil
series = [date = Time.local(year, month, day)]
if params[:series]
params[:series].each do |some_time_after|
series << series.last + (rand * some_time_after).ceil
end
return series
end
date
end
Employee.create!(first_name: "Shelly",
last_name: "Houghton",
mi: "M",
full_name: "Shelly M Houghton",
marital_status: "Single",
gender: "Female",
hire_date: "2000-04-16",
primary_position: "Manager",
trained_position: "Job Developer",
email: "shoughton#example.com",
active: true,
address1: "76th Ave",
city: "Frave",
zip_code: "54806",
state: "WI",
emp_home_ph: "1-111-111-1111",
emp_mobile_ph: "1-222-222-2222",
emer_contact_first_name: "Kenneth",
emer_contact_last_name: "Koening",
emer_contact_relationship: "Friend",
emer_contact_ph: "1-333-333-3333",
role: "Manager",
birth_date: "1982-08-21",
admin: true,
password: "90nothguoh",
password_confirmation: "90nothguoh")
99.times do |n|
first_name = Faker::Name.first_name
last_name = Faker::Name.last_name
mi = ("A".."Z").to_a[rand(26)]
full_name = Faker::Name.full_name
marital_status = #marital_status[rand(2)].to_s
gender = #gender[rand(2)].to_s
hire_date = randomDate(:year_range => 60, :year_latest => 12)
birth_date = randomDate(:year_range => 60, :year_latest => 22)
primary_position = #primary_position[rand(5)].to_s
trained_position = #trained_position[rand(5)].to_s
email = "emp-#{n+1}#example.org"
active = [true, false][rand(2)]
admin = (1 == rand(2) ? true : false)
role = #role[rand(5)].to_s
address1 = "Seaview-#{n+5}Way"
city = Faker::Lorem.words(1).to_s.capitalize
state = Faker::Address.us_state()
zip_code = Faker::Address.zip_code
emp_home_ph = Faker::PhoneNumber.phone_number
emp_mobile_ph = Faker::PhoneNumber.phone_number
emer_contact_first_name = Faker::Name.first_name
emer_contact_last_name = Faker::Name.last_name
emer_contact_relationship = #emer_contact_relationship[rand(10)].to_s
emer_contact_ph = Faker::PhoneNumber.phone_number
password = "uniqueone"
Employee.create!(first_name: first_name, mi: mi, last_name: last_name,
full_name: full_name, marital_status: marital_status,
gender: gender, birth_date: birth_date, hire_date: hire_date,
primary_position: primary_position, trained_position:
trained_position, email: email, role: role, address1:
address1, city: city, state: state, zip_code: zip_code,
emp_home_ph: emp_home_ph, emp_mobile_ph: emp_mobile_ph,
emer_contact_first_name: emer_contact_first_name,
emer_contact_last_name: emer_contact_last_name,
emer_contact_relationship: emer_contact_relationship,
emer_contact_ph: emer_contact_ph, password: password,
password_confirmation: password)
end
end
end
I ran:
rake sample_employee_data.rake
and I got this flag:
rake aborted!
Don't know how to build task 'sample_employee_data.rake'
home/me/.rvm/gems/ruby-1.9.3-p385#rails3212/bin/ruby_noexec_wrapper:14:in 'eval'
home/me/.rvm/gems/ruby-1.9.3-p385#rails3212/bin/ruby_noexec_wrapper:14:in '<main>'
Can anyone spot the problem ... my apologies for such a long file.
Thanks.

There's a couple things going on here.
When you run $ rake sample_employee_data.rake you're asking rake to rake your file - but rake responds to tasks:
$ rake some_namespace:some_task_in_that_namespace
You're also using a custom .rake file rather than the conventional Rakefile - which is fine, but this means you need to tell rake explicitly that you're not using its default Rakefile.
$ rake --rakefile your_custom_rake_file.rake some_task
So in your case, you need to issue a rake command like so:
$ rake --rakefile sample_employee_data.rake db:populate

Related

How to configure CloudFunctionInvokeFunctionOperator to call cloud function on composer(airflow)

I ran into the error to invoke cloud function by using CloudFunctionInvokeFunctionOperator like this.
line 915, in execute raise HttpError(resp, content, uri=self.uri) googleapiclient.errors.HttpError: <HttpError 404 when requesting https://cloudfunctions.googleapis.com/v1/projects/pongthorn/locations/asia-southeast1/functions/crypto-trading-to-bq:call?alt=json returned "Function crypto-trading-to-bq in region asia-southeast1 in project pongthorn does not exist". Details: "Function crypto-trading-to-bq in region asia-southeast1 in project pongthorn does not exist">
I assump that I my made mistake at function id , What is functoin id , the figure below is my cloud functoin and function name is crypto-trading-to-bq
function id and function name , they are the same??????
I set 3 variable names on JSON file and upload it to airflow as the following values
{
"project_id": "pongthorn",
"region_name": "asia-southeast1",
"function_name": "crypto-trading-to-bq"
}
This is my code
import datetime
import airflow
from airflow.providers.google.cloud.operators.functions import (
CloudFunctionDeleteFunctionOperator,
CloudFunctionDeployFunctionOperator,
CloudFunctionInvokeFunctionOperator,
)
YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1)
XProjdctID=airflow.models.Variable.get('project_id')
XRegion=airflow.models.Variable.get('region_name')
XFunction=airflow.models.Variable.get('function_name')
default_args = {
'owner': 'Binance Trading Transaction',
'depends_on_past': False,
'email': [''],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': datetime.timedelta(minutes=5),
'start_date': YESTERDAY,
}
with airflow.DAG(
'bn_trading_flow',
catchup=False,
default_args=default_args,
schedule_interval=datetime.timedelta(days=1)) as dag:
call_crypto_trading_to_bq = CloudFunctionInvokeFunctionOperator(
task_id = "load_crypto_trading_to_bq",
location = XRegion,
function_id = XFunction,
input_data = {},
)

RoR, need to create recursive relationships in table associations

I'm very new to rails, and am a little stuck on the logic for this problem.
I have one table (using mysql) of employees, each of them with a manager_id key which refers to the employee they report to. So for example the employee with the title of "CEO" with an id of 1, has a manager_id of nil, and the employee with title of "CTO" has a manager_id of 1. So my records look like this
id: 1, first_name: "Bob", last_name: "Boss", title: "CEO", manager_id: null
id: 2, first_name: "Pat", last_name: "Guy", title: "CTO", manager_id: 1
id: 3, first_name: "John", last_name: "Dude", title: "VP of engineering", manager_id: 2
and my JSON structure should look like this
[
{id: 1, first_name: "Bob", last_name: "Boss", title: "CEO", manager_id: null, descendents: [
{id: 2, first_name: "Pat", last_name: "Guy", title: "CTO", manager_id: 1, descendents: [
{id: 3, first_name: "John", last_name: "Dude", title: "VP of engineering", manager_id: 2, descendents: [....]}
]},
{..more CEO descendents...}
]
I'm trying to create a nested JSON structure that starts at CEO, lists all employees that report to them, and each of those employees descendants. I was trying to write a script that creates this but I keep getting infinite recursive calls. This is what I have
#start at some root
#root = Employee.find_by title: 'CEO'
#convert to hash table
#results[0] = #root.attributes
#add direct_reports key
#results[0]["direct_reports"] = []
def getBelow(root=#root)
#reports = Employee.where("manager_id = ?", #root[:id])
if #reports.blank?
return []
else
#reports = #reports.map(&:attributes)
#reports.each do |person|
person["direct_reports"] = []
getBelow(person)
end
#reports = Employee.where("manager_id = ?", #root[:id])
root["direct_reports"] = #reports
end
return #root
end
#list = getBelow(#results[0])
If I'm passing in each new person object, shouldn't they all eventually end when #reports.blank? becomes true?
An alternative I was thinking of was to use table associations inspired by this blog post
https://hashrocket.com/blog/posts/recursive-sql-in-activerecord
but that seems a little too complicated.
Some issues in the getBelow method
You are always using #root, instead of using the param (root). So you are always starting again from the 'CEO'.
You are calling getBelow recursively but you are not using the result.
You call #reports = Employee.where("manager_id = ?", #root[:id]) twice.
You return #root.
As Jorge Najera said, there are gems that handle a tree structure easily. If you want to build it on your own, this is my suggestion:
#start at some root
#root = Employee.find_by manager_id: nil
#convert to hash table
#tree = #root.attributes
#add direct_reports key
#tree["direct_reports"] = getBelow(#root)
def getBelow(manager)
branch = []
Employee.where("manager_id = ?", manager.id).each do |employee|
node = employee.attributes
node["direct_reports"] = getBelow(employee)
branch << node
end
branch
end
This was not tested so I think you´ll get some errors, but I believe the idea is fine.

after_update callback won't update attribute

Good day... I have implemented an after_update callback for a specific column that will be updated, and after that column is updated the callback is meant to add the newly updated value to another column i.e
class Product < ActiveRecord::Base
after_update :update_stock, :if => :produced_changed?
private
def update_stock
self.stock = self.stock + self.produced
end
end
When i run rails console and run "Product.update produced:450" the stock column will automatically add the new value. i.e it works perfectly but when I try updating from the "view/controller" it doesn't work at all.
please is there a reason why?
2.2.4 :004 > h
=> Product id: 1, name: "MAC Air Filter", partnumber: 33440, description: "Air filter", **stock: 3440**, created_at: "2016-04-08 11:38:58", updated_at: "2016-04-19 20:33:00", **produced: 33**
2.2.4 :006 > h.update **produced:3000**
(0.1ms) SAVEPOINT active_record_1
SQL (1.4ms) UPDATE "products" SET "produced" = ?, "updated_at" = ? WHERE "products"."id" = ? [[**"produced", 3000**], ["updated_at", "2016-04-20 13:57:59.377954"], ["id", 1]]
(0.1ms) RELEASE SAVEPOINT active_record_1
=> true
2.2.4 :007 > h
=> Product id: 1, name: "MAC Air Filter", partnumber: 33440, description: "Air filter", **stock: 6440**, created_at: "2016-04-08 11:38:58", updated_at: "2016-04-20 13:57:59", **produced: 3000**>
You have save the product stock to update the product. Here in the update_stock callback you only set the value of product stock. self.stock = self.stock + self.produced only set the value of product stock.
To update the stock value you have to save in callback as:
class Product < ActiveRecord::Base
after_update :update_stock, :if => :produced_changed?
private
def update_stock
self.stock = self.stock + self.produced
self.save
end
end
But it runs the infinite loop and gives error. So you have to set the stock value before update using before_update callback.
class Product < ActiveRecord::Base
before_update :update_stock, :if => :produced_changed?
private
def update_stock
self.stock = self.stock + self.produced
end
end
And save the value of product in controller.
you should call save method in your callback or use callback before_update

quantmod R getsymbols.MySQL modification

I have already studied the case
Quantmod: Error loading symbols from MySQL DB
and already try to fix the getSymbols.MySQL function in R
However, I found that my database just contain
date, open, high, low, close, volume (without the close.adj column).
So, if I want to further modify the getSymbols.MySQL function, what can I do?
I have tried to use 'fix(getSymbols.MySQL)' to fix the function. However, it returns
Error in colnames<-(*tmp*, value = c("H0001.Open", "H0001.High", "H0001.Low", : length of 'dimnames' [2] not equal to array extent
when I connect to my database.
function (Symbols, env, return.class = "xts", db.fields = c("date",
"o", "h", "l", "c", "v", "a"), field.names = NULL, user = NULL,
password = NULL, dbname = NULL, host = "localhost", port = 3306,
...)
{
importDefaults("getSymbols.MySQL")
this.env <- environment()
for (var in names(list(...))) {
assign(var, list(...)[[var]], this.env)
}
if (!hasArg(verbose))
verbose <- FALSE
if (!hasArg(auto.assign))
auto.assign <- TRUE
if (!requireNamespace("DBI", quietly = TRUE))
stop("package:", dQuote("DBI"), "cannot be loaded.")
if (!requireNamespace("RMySQL", quietly = TRUE))
stop("package:", dQuote("RMySQL"), "cannot be loaded.")
if (is.null(user) || is.null(password) || is.null(dbname)) {
stop(paste("At least one connection argument (", sQuote("user"),
sQuote("password"), sQuote("dbname"), ") is not set"))
}
con <- DBI::dbConnect("MySQL", user = user, password = password,
dbname = dbname, host = host, port = port)
db.Symbols <- DBI::dbListTables(con)
if (length(Symbols) != sum(Symbols %in% db.Symbols)) {
missing.db.symbol <- Symbols[!Symbols %in% db.Symbols]
warning(paste("could not load symbol(s): ", paste(missing.db.symbol,
collapse = ", ")))
Symbols <- Symbols[Symbols %in% db.Symbols]
}
for (i in 1:length(Symbols)) {
if (verbose) {
cat(paste("Loading ", Symbols[[i]], paste(rep(".",
10 - nchar(Symbols[[i]])), collapse = ""), sep = ""))
}
query <- paste("SELECT ", paste(db.fields, collapse = ","),
" FROM ", Symbols[[i]], " ORDER BY date")
rs <- DBI::dbSendQuery(con, query)
fr <- DBI::fetch(rs, n = -1)
fr <- xts(as.matrix(fr[, -1]), order.by = as.Date(fr[,
1], origin = "1970-01-01"), src = dbname, updated = Sys.time())
colnames(fr) <- paste(Symbols[[i]], c("Open", "High",
"Low", "Close", "Volume", "Adjusted"), sep = ".")
fr <- convert.time.series(fr = fr, return.class = return.class)
if (auto.assign)
assign(Symbols[[i]], fr, env)
if (verbose)
cat("done\n")
}
DBI::dbDisconnect(con)
if (auto.assign)
return(Symbols)
return(fr)
}
I think the problem is the function was designed to read 7 column of data rather than 6 column of data. Hope someone can help.
Here's a patch that should allow you to do what you want. I'm unable to test because I don't have a MySQL installation to test against. Please let me know whether or not it works.
diff --git a/R/getSymbols.R b/R/getSymbols.R
index 0a2e814..7a9be66 100644
--- a/R/getSymbols.R
+++ b/R/getSymbols.R
## -634,9 +634,9 ## function(Symbols,env,return.class='xts',
fr <- xts(as.matrix(fr[,-1]),
order.by=as.Date(fr[,1],origin='1970-01-01'),
src=dbname,updated=Sys.time())
- colnames(fr) <- paste(Symbols[[i]],
- c('Open','High','Low','Close','Volume','Adjusted'),
- sep='.')
+ if(is.null(field.names))
+ field.names <- c('Open','High','Low','Close','Volume','Adjusted')
+ colnames(fr) <- paste(Symbols[[i]], field.names, sep='.')
fr <- convert.time.series(fr=fr,return.class=return.class)
if(auto.assign)
assign(Symbols[[i]],fr,env)
Then your function call should be:
getSymbols.MySQL("H0001", env, return.class = 'xts',
db.fields = c("date", "open", "high", "low", "close", "volume"),
field.names = c("date", "open", "high", "low", "close", "volume"),
user = 'xxxx', password = 'xxxx', host='xxxx', dbname = 'xxxx')

Parse JSON object in SAS macro

Here is the input JSON file. It have to parse in SAS dataset.
"results":
[
{
"acct_nbr": 1234,
"firstName": "John",
"lastName": "Smith",
"age": 25,
"address": {
"streetAddress": "21 2nd Street",
"city": "New York",
"state": "NY",
"postalCode": "10021"
}
}
,
{
"acct_nbr": 3456,
"firstName": "Sam",
"lastName": "Jones",
"age": 32,
"address": {
"streetAddress": "25 2nd Street",
"city": "New Jersy",
"state": "NJ",
"postalCode": "10081"
}
}
]
And I want the output for only Address field in SAS dataset like this :
ACCT_NBR FIELD_NAME FIELD_VALUE
1234 streetAddress 21 2nd Street
1234 city New York
1234 state NY
1234 postalCode 10021
3456 streetAddress 25 2nd Street
3456 city New Jersy
3456 state NJ
3456 postalCode 10081
I have tried separate way, but no similar output.
even tried scanover from PDF ... but cannot get desired output...
here is my code......and output....
LIBNAME src '/home/user/read_JSON';
filename data '/home/user/read_JSON/test2.json';
data src.testdata2;
infile data lrecl = 32000 truncover scanover;
input #'"streetAddress": "' streetAddress $255. #'"city": "' city $255. #'"state": "' state $2. #'"postalCode": "' postalCode $255.;
streetAddress = substr(streetAddress,1,index(streetAddress,'",')-2);
city = substr( city,1,index( city,'",')-2);
state = substr(state,1,index(state,'",')-2);
postalCode = substr(postalCode,1,index(postalCode,'",')-2);
run;
proc print data=src.testdata2;
RUN;
My OUTPUT in .lst file
The SAS System 09:44 Tuesday, January 14, 2014 1
street postal
Obs Address city state Code
1 21 2nd Stree New Yor NY 10021"
2 25 2nd Stree New Jers NJ 10081"
To answer your question with a SAS-only solution, your problems are twofold:
Use SCAN instead of substr to get the un-comma/quotationed portion
acct_nbr is a number, so you need to remove the final quotation mark from the input.
Here's the correct code (I changed directories, you'll need to change them back):
filename data 'c:\temp\json.txt';
data testdata2;
infile data lrecl = 32000 truncover scanover;
input
#'"acct_nbr": ' acct_nbr $255.
#'"streetAddress": "' streetAddress $255.
#'"city": "' city $255.
#'"state": "' state $2.
#'"postalCode": "' postalCode $255.;
acct_nbr=scan(acct_nbr,1,',"');
streetAddress = scan(streetAddress,1,',"');
city = scan(city,1,',"');
state = scan(state,1,',"');
postalCode = scan(postalCode,1,',"');
run;
proc print data=testdata2;
RUN;
You can use proc groovy to parse JSON pretty easily (assuming you know Groovy). This SAS blog on authenticating to Twitter shows a detailed example of how to do it; here is some of the highlights.
This assumes you have the Groovy JAR files (http://groovy.codehaus.org/Download) and a way to output the files (the example uses OpenCSV).
The below is my attempt at it; I don't think it quite works, but I don't know Groovy, either. The general concept should be correct. If you want to try this approach, but can't figure out the specifics of this, you might either retag your question groovy or ask a new question with that tag.
%let groovydir=C:\Program Files\SASHome_9.4\SASFoundation\9.4\groovy; *the location the groovy JARs are located at;
%let sourcefile=c:\temp\json.txt;
%let outfile=c:\temp\json.csv;
proc groovy classpath="&groovydir.\groovy-all-2.2.0.jar;&groovydir.\opencsv-2.3.jar";
submit "&sourcefile" "&outfile";
import groovy.json.*
import au.com.bytecode.opencsv.CSVWriter
def input = new File(args[0]).text
def output = new JsonSlurper().parseText(input)
def csvoutput = new FileWriter(args[1])
CSVWriter writer = new CSVWriter(csvoutput);
String[] header = new String[8];
header[0] = "results.acct_nbr";
header[1] = "results.firstName";
header[2] = "results.lastName";
header[3] = "results.age";
header[4] = "results.address.streetAddress";
header[5] = "results.address.city";
header[6] = "results.address.state";
header[7] = "results.address.postalCode";
writer.writeNext(header);
output.statuses.each {
String[] content = new String[8];
content[0] = it.results.acct_nbr.toString();
content[1] = it.results.firstName.toString();
content[2] = it.results.lastName.toString();
content[3] = it.results.age.toString();
content[4] = it.results.address.streetAddress.toString();
content[5] = it.results.address.city.toString();
content[6] = it.results.address.state.toString();
content[7] = it.results.address.postalCode.toString();
writer.writeNext(content)
}
writer.close();
endsubmit;
quit;
I used this json file and above code as an example in a thread on sas.com. One of the expert programmers on there was extremely generous and came up with a solution. Note the json file should be wrapped in "{}".
Link: https://communities.sas.com/thread/72163
Code:
filename cp temp;
proc groovy classpath=cp;
add classpath="C:\Program Files\Java\groovy-2.3.4\embeddable\groovy-all-2.3.4.jar";
/*or*/
/*
add classpath="C:\Program Files\Java\groovy-2.3.4\lib\groovy-2.3.4.jar";
add classpath="C:\Program Files\Java\groovy-2.3.4\lib\groovy-json-2.3.4.jar";
*/
submit parseonly;
import groovy.json.JsonSlurper
class MyJsonParser {
def parseFile(path) {
def jsonFile = new File(path)
def jsonText = jsonFile.getText()
def InputJSON = new JsonSlurper().parseText(jsonText)
def accounts = []
InputJSON.results.each{
accounts << [
acct_nbr : it.acct_nbr.toString(),
firstName : it.firstName,
lastName : it.lastName,
age : it.age.toString(),
streetAddress : it.address.streetAddress,
city : it.address.city,
state : it.address.state,
postalCode : it.address.postalCode
]
}
return accounts
}
}
endsubmit;
submit parseonly;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
public class MyJsonParser4Sas {
public String filename = "";
public void init() {
MyJsonParser myParser = new MyJsonParser();
accounts = myParser.parseFile(filename);
iter = accounts.iterator();
}
public boolean hasNext() {
return iter.hasNext();
}
public void getNext() {
account = ((LinkedHashMap) (iter.next()));
}
public String getString(String k) {
return account.get(k);
}
protected ArrayList accounts;
protected Iterator iter;
protected LinkedHashMap account;
}
endsubmit;
quit;
options set=classpath "%sysfunc(pathname(cp,f))";
data accounts;
attrib id label="Account Index" length= 8
acct_nbr label="Account Number" length=$ 10
firstName label="First Name" length=$ 20
lastName label="Last Name" length=$ 30
age label="Age" length=$ 3
streetAddress label="Street Address" length=$ 128
city label="City" length=$ 40
state label="State" length=$ 2
postalCode label="Postal Code" length=$ 5;
dcl javaobj accounts("MyJsonParser4Sas");
accounts.exceptiondescribe(1);
accounts.setStringField("filename", "C:\\foo.json");
accounts.callVoidMethod("init");
accounts.callBooleanMethod("hasNext",rc);
do id=1 by 1 while(rc);
accounts.callVoidMethod("getNext");
accounts.callStringMethod("getString", "acct_nbr", acct_nbr);
accounts.callStringMethod("getString", "firstName", firstName);
accounts.callStringMethod("getString", "lastName", lastName);
accounts.callStringMethod("getString", "age", age);
accounts.callStringMethod("getString", "streetAddress", streetAddress);
accounts.callStringMethod("getString", "city", city);
accounts.callStringMethod("getString", "state", state);
accounts.callStringMethod("getString", "postalCode", postalCode);
output;
accounts.callBooleanMethod("hasNext",rc);
end;
drop rc;
run;