Android Room how to query related table? - relational-database

First thing, my data is POKEMON!!! enjoy 😉
I need to do this on the database side, filtering and sorting the returned data isn't an option as using Paging...
I'm using Room I have my database working well but I now want to query the pokemonType list in my relation
Given this data class
data class PokemonWithTypesAndSpecies #JvmOverloads constructor(
#Ignore
var matches : Int = 0,
#Embedded
val pokemon: Pokemon,
#Relation(
parentColumn = Pokemon.POKEMON_ID,
entity = PokemonType::class,
entityColumn = PokemonType.TYPE_ID,
associateBy = Junction(
value = PokemonTypesJoin::class,
parentColumn = Pokemon.POKEMON_ID,
entityColumn = PokemonType.TYPE_ID
)
)
val types: List<PokemonType>,
#Relation(
parentColumn = Pokemon.POKEMON_ID,
entity = PokemonSpecies::class,
entityColumn = PokemonSpecies.SPECIES_ID,
associateBy = Junction(
value = PokemonSpeciesJoin::class,
parentColumn = Pokemon.POKEMON_ID,
entityColumn = PokemonSpecies.SPECIES_ID
)
)
val species: PokemonSpecies?
)
I can get my data with a simple query and even search it
#Query("SELECT * FROM Pokemon WHERE pokemon_name LIKE :search")
fun searchPokemonWithTypesAndSpecies(search: String): LiveData<List<PokemonWithTypesAndSpecies>>
But now what I want is to add filtering on pokemon types which as you can see is a list (which is probably a transaction under the hood) and is in a seperate table, so given a list of string called filters I want to:
only return pokemon that contain an item in filters
sort pokemon by both number of matching types and by ID
So I want my tests to look something like this
val bulbasaurSpeciesID = 1
val squirtleSpeciesID = 2
val charmanderSpeciesID = 3
val charizardSpeciesID = 4
val pidgeySpeciesID = 5
val moltresSpeciesID = 6
val bulbasaurID = 1
val squirtleID = 2
val charmanderID = 3
val charizardID = 4
val pidgeyID = 5
val moltresID = 6
val grassTypeID = 1
val poisonTypeID = 2
val fireTypeID = 3
val waterTypeID = 4
val flyingTypeID = 5
val emptySearch = "%%"
val allPokemonTypes = listOf(
"normal",
"water",
"fire",
"grass",
"electric",
"ice",
"fighting",
"poison",
"ground",
"flying",
"psychic",
"bug",
"rock",
"ghost",
"dark",
"dragon",
"steel",
"fairy",
"unknown",
)
#Before
fun createDb() {
val context = ApplicationProvider.getApplicationContext<Context>()
db = Room.inMemoryDatabaseBuilder(
context, PokemonRoomDatabase::class.java,
).setTransactionExecutor(Executors.newSingleThreadExecutor())
.allowMainThreadQueries()
.build()
pokemonDao = db.pokemonDao()
speciesDao = db.pokemonSpeciesDao()
speciesJoinDao = db.pokemonSpeciesJoinDao()
pokemonTypeDao = db.pokemonTypeDao()
pokemonTypeJoinDao = db.pokemonTypeJoinDao()
}
#After
#Throws(IOException::class)
fun closeDb() {
db.close()
}
#Test
#Throws(Exception::class)
fun testFiltering() {
insertPokemonForFilterTest()
val pokemon =
pokemonDao.searchAndFilterPokemon(search = emptySearch, filters = allPokemonTypes)
.getValueBlocking(scope)
assertThat(pokemon?.size, equalTo(6)) // This fails list size is 9 with the current query
val pokemonFiltered =
pokemonDao.searchAndFilterPokemon(search = emptySearch, filters = listOf("fire", "flying"))
.getValueBlocking(scope)
assertThat(pokemon?.size, equalTo(4))
assertThat(pokemonFiltered!![0].pokemon.name, equalTo("charizard")) // matches 2 filters and ID is 4
assertThat(pokemonFiltered!![1].pokemon.name, equalTo("moltres")) // matches 2 filters and ID is 6
assertThat(pokemonFiltered!![2].pokemon.name, equalTo("charmander")) // matches one filter and ID is 3
assertThat(pokemonFiltered!![3].pokemon.name, equalTo("pidgey")) // mayches one filter and ID is 5
}
private fun insertPokemonForFilterTest() = runBlocking {
insertBulbasaur()
insertSquirtle()
insertCharmander()
insertCharizard()
insertMoltres()
insertPidgey()
}
private fun insertBulbasaur() = runBlocking {
val bulbasaur = bulbasaur()
val grassJoin = PokemonTypesJoin(pokemon_id = bulbasaurID, type_id = grassTypeID)
val poisonJoin = PokemonTypesJoin(pokemon_id = bulbasaurID, type_id = poisonTypeID)
val bulbasaurSpeciesJoin =
PokemonSpeciesJoin(pokemon_id = bulbasaurID, species_id = bulbasaurSpeciesID)
pokemonDao.insertPokemon(bulbasaur.pokemon)
speciesDao.insertSpecies(bulbasaur.species!!)
speciesJoinDao.insertPokemonSpeciesJoin(bulbasaurSpeciesJoin)
pokemonTypeDao.insertPokemonType(pokemonType = bulbasaur.types[0])
pokemonTypeDao.insertPokemonType(pokemonType = bulbasaur.types[1])
pokemonTypeJoinDao.insertPokemonTypeJoin(grassJoin)
pokemonTypeJoinDao.insertPokemonTypeJoin(poisonJoin)
}
private fun insertSquirtle() = runBlocking {
val squirtle = squirtle()
val squirtleSpeciesJoin =
PokemonSpeciesJoin(pokemon_id = squirtleID, species_id = squirtleSpeciesID)
val waterJoin = PokemonTypesJoin(pokemon_id = squirtleID, type_id = waterTypeID)
pokemonDao.insertPokemon(squirtle.pokemon)
speciesDao.insertSpecies(squirtle.species!!)
speciesJoinDao.insertPokemonSpeciesJoin(squirtleSpeciesJoin)
pokemonTypeDao.insertPokemonType(pokemonType = squirtle.types[0])
pokemonTypeJoinDao.insertPokemonTypeJoin(waterJoin)
}
private fun insertCharmander() = runBlocking {
val charmander = charmander()
val fireJoin = PokemonTypesJoin(pokemon_id = charmanderID, type_id = fireTypeID)
val charmanderSpeciesJoin =
PokemonSpeciesJoin(pokemon_id = charmanderID, species_id = charmanderSpeciesID)
pokemonDao.insertPokemon(charmander.pokemon)
speciesDao.insertSpecies(charmander.species!!)
speciesJoinDao.insertPokemonSpeciesJoin(charmanderSpeciesJoin)
pokemonTypeDao.insertPokemonType(pokemonType = charmander.types[0])
pokemonTypeJoinDao.insertPokemonTypeJoin(fireJoin)
}
private fun insertCharizard() = runBlocking {
val charizard = charizard()
val charizardSpeciesJoin =
PokemonSpeciesJoin(pokemon_id = charizardID, species_id = charizardSpeciesID)
val fireJoin = PokemonTypesJoin(pokemon_id = charizardID, type_id = fireTypeID)
val flyingJoin = PokemonTypesJoin(pokemon_id = charizardID, type_id = flyingTypeID)
pokemonDao.insertPokemon(charizard.pokemon)
speciesDao.insertSpecies(charizard.species!!)
speciesJoinDao.insertPokemonSpeciesJoin(charizardSpeciesJoin)
pokemonTypeDao.insertPokemonType(pokemonType = charizard.types[0])
pokemonTypeDao.insertPokemonType(pokemonType = charizard.types[1])
pokemonTypeJoinDao.insertPokemonTypeJoin(fireJoin)
pokemonTypeJoinDao.insertPokemonTypeJoin(flyingJoin)
}
private fun insertPidgey() = runBlocking {
val pidgey = pidgey()
val pidgeySpeciesJoin =
PokemonSpeciesJoin(pokemon_id = pidgeyID, species_id = pidgeySpeciesID)
val flyingJoin = PokemonTypesJoin(pokemon_id = pidgeyID, type_id = flyingTypeID)
pokemonDao.insertPokemon(pidgey.pokemon)
speciesDao.insertSpecies(pidgey.species!!)
speciesJoinDao.insertPokemonSpeciesJoin(pidgeySpeciesJoin)
pokemonTypeDao.insertPokemonType(pokemonType = pidgey.types[0])
pokemonTypeJoinDao.insertPokemonTypeJoin(flyingJoin)
}
private fun insertMoltres() = runBlocking {
val moltres = moltres()
val moltresSpeciesJoin =
PokemonSpeciesJoin(pokemon_id = moltresID, species_id = moltresSpeciesID)
val fireJoin = PokemonTypesJoin(pokemon_id = moltresID, type_id = fireTypeID)
val flyingJoin = PokemonTypesJoin(pokemon_id = moltresID, type_id = flyingTypeID)
pokemonDao.insertPokemon(moltres.pokemon)
speciesDao.insertSpecies(moltres.species!!)
speciesJoinDao.insertPokemonSpeciesJoin(moltresSpeciesJoin)
pokemonTypeDao.insertPokemonType(pokemonType = moltres.types[0])
pokemonTypeDao.insertPokemonType(pokemonType = moltres.types[1])
pokemonTypeJoinDao.insertPokemonTypeJoin(fireJoin)
pokemonTypeJoinDao.insertPokemonTypeJoin(flyingJoin)
}
fun bulbasaur(): PokemonWithTypesAndSpecies = PokemonWithTypesAndSpecies(
pokemon = Pokemon(id = bulbasaurID, name = "bulbasaur"),
species = PokemonSpecies(
id = bulbasaurSpeciesID,
species = "Seed pokemon",
pokedexEntry = "There is a plant seed on its back right from the day this Pokémon is born. The seed slowly grows larger."
),
types = listOf(
PokemonType(id = poisonTypeID, name = "poison", slot = 1),
PokemonType(id = grassTypeID, name = "grass", slot = 2)
)
)
fun squirtle(): PokemonWithTypesAndSpecies = PokemonWithTypesAndSpecies(
pokemon = Pokemon(id = squirtleID, name = "squirtle"),
species = PokemonSpecies(
id = squirtleSpeciesID,
species = "Turtle pokemon",
pokedexEntry = "Small shell pokemon"
),
types = listOf(PokemonType(id = waterTypeID, name = "water", slot = 1))
)
fun charmander(): PokemonWithTypesAndSpecies = PokemonWithTypesAndSpecies(
pokemon = Pokemon(id = charmanderID, name = "charmander"),
species = PokemonSpecies(
id = charmanderSpeciesID,
species = "Fire lizard pokemon",
pokedexEntry = "If the flame on this pokemon's tail goes out it will die"
),
types = listOf(PokemonType(id = fireTypeID, name = "fire", slot = 1))
)
fun charizard(): PokemonWithTypesAndSpecies = PokemonWithTypesAndSpecies(
pokemon = Pokemon(id = charizardID, name = "charizard"),
species = PokemonSpecies(
id = charizardSpeciesID,
species = "Fire flying lizard pokemon",
pokedexEntry = "Spits fire that is hot enough to melt boulders. Known to cause forest fires unintentionally"
),
types = listOf(
PokemonType(id = fireTypeID, name = "fire", slot = 1),
PokemonType(id = flyingTypeID, name = "flying", slot = 2)
)
)
fun moltres(): PokemonWithTypesAndSpecies = PokemonWithTypesAndSpecies(
pokemon = Pokemon(id = moltresID, name = "moltres"),
species = PokemonSpecies(
id = moltresSpeciesID,
species = "Fire bird pokemon",
pokedexEntry = "Known as the legendary bird of fire. Every flap of its wings creates a dazzling flash of flames"
),
types = listOf(
PokemonType(id = fireTypeID, name = "fire", slot = 1),
PokemonType(id = flyingTypeID, name = "flying", slot = 2)
)
)
fun pidgey(): PokemonWithTypesAndSpecies = PokemonWithTypesAndSpecies(
pokemon = Pokemon(id = pidgeyID, name = "pidgey"),
species = PokemonSpecies(
id = pidgeySpeciesID,
species = "Bird pokemon",
pokedexEntry = "Pidgey is a Flying Pokémon. Among all the Flying Pokémon, it is the gentlest and easiest to capture. A perfect target for the beginning Pokémon Trainer to test his Pokémon's skills."
),
types = listOf(PokemonType(id = flyingTypeID, name = "flying", slot = 1))
)
And the query would be
#Query("SELECT * FROM Pokemon INNER JOIN PokemonType, PokemonTypesJoin ON Pokemon.pokemon_id = PokemonTypesJoin.pokemon_id AND PokemonType.type_id = PokemonTypesJoin.type_id WHERE pokemon_name LIKE :search AND type_name IN (:filters) ORDER BY pokemon_id ASC")
fun searchAndFilterPokemon(search: String, filters: List<String>): LiveData<List<PokemonWithTypesAndSpecies>>
I'm guessing this doesn't work because at this point Room hasn't collected the types from the other table and it's probably not even querying a list, I think this part
type_name IN (:filters)
is checking a column against a list when what I want is a List against a list 🤷‍♂️ but honestly I'm happy to just say I've fallen and can't get up 🤣 can anyone help? any help appreciated

Maybe I could misuse some columns' names, but try this query:
#Query("SELECT pok.id, pok.name FROM Pokemon AS pok
INNER JOIN PokemonTypesJoin AS p_join ON pok.id = p_join.pokemon_id
INNER JOIN PokemonType AS pok_type ON pok_type.id = p_join.type_id
WHERE pok.name LIKE :search AND pok_type.name IN (:filters)
GROUP BY pok.id, pok.name ORDER BY count(*) DESC, pok.id ASC")

Have you compared Room to Cmobilecom-JPA for android? JPA is very good at query relationships. The advantage of using JPA (standard) is obvious, making your code reusable on android, server side java, or swing project.

Thanks to #serglytikhonov my query works and now looks like this
#Query("""SELECT * FROM Pokemon
INNER JOIN PokemonTypesJoin
ON Pokemon.pokemon_id = PokemonTypesJoin.pokemon_id
INNER JOIN PokemonType
ON PokemonType.type_id = PokemonTypesJoin.type_id
WHERE pokemon_name LIKE :search AND type_name IN (:filters)
GROUP BY Pokemon.pokemon_id, Pokemon.pokemon_name
ORDER BY count(*) DESC, pokemon_id ASC""")
fun searchAndFilterPokemon(search: String, filters: List<String>): LiveData<List<PokemonWithTypesAndSpecies>>
the main piece being this count(*) and group by many thanks

Related

VIPER: some of genes in the pathway gene list are missing in the result

I have a problem when I use VIPER with aracne networks.
I used regulonbrca in aracne to calculate protein activity,
The number of genes in regulonbrca is 6054.
And the number of genes in regulonbrca also in my TPM file is 5918.
But I got only 4506 genes as a viper result.
Why some genes are missing when running viper? Is there a default setting about it?
Here is my code.
expdat= "TPM-brca(entrez_avg).csv"
clsdat= "immunesubtype-BRCA.csv"
regulon= regulonbrca
dat0 = data.frame(read.csv(expdat))
dat1 = dat0[,-c(1,2)]
rownames(dat1) = dat0[,1]
cls0 = data.frame(read.csv(clsdat))
colnames(cls0) = c("id", "description")
cls1 = cls0 %>% dplyr::filter(id %in% colnames(dat1))
dat = dat1[,cls1$id]
cls = data.frame(description=cls1[,2])
rownames(cls) = cls1[,1]
meta = data.frame(labelDescription=c("description"), row.names = colnames(cls))
pheno = new("AnnotatedDataFrame", data=cls, varMetadata=meta)
dset = ExpressionSet(assayData = as.matrix(dat), phenoData = pheno)
signature = rowTtest(dset, "description", "TRUE", "FALSE")
signature = (qnorm(signature$p.value/2, lower.tail = FALSE) * + sign(signature$statistic))[, 1]
nullmodel = ttestNull(dset, "description", "TRUE", "FALSE", per = 1000, repos = TRUE, verbose = FALSE)
vpres = viper(dset, regulon, verbose = FALSE) ## single sample viper
res_ss0 = data.frame(id=row.names(vpres#assayData$exprs), vpres#assayData$exprs)
geneid_ss0 = ldply(sapply(row.names(res_ss0), converter), data.frame) ## gene mapping
colnames(geneid_ss0) = c('id', 'gene')
res_ss = merge(geneid_ss0, res_ss0, key='id')[, -1]

how to parse multiple JSON structures in spark program

I am working on parsing logs(Json format) in Scala. I don't know how to proceed. I may get different kinds of logs to be processed.
how do i write/design my code to handle different types of Json structures?
can i give my Scala program a schema and let it parse?
I wrote some code using Object mapper and read through the nodes but i want a more structure agnostic approach.
I am not sure where to start. please point me to some reading or examples. i tried to google or search in Stackoverflow resulting in too many examples and it is confusing as i am learning Scala also.
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Calendar;
import org.apache.spark.sql.hive.HiveContext
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import org.apache.spark.rdd.RDD;
sc.setLogLevel("OFF");
val args = sc.getConf.get("spark.driver.args").split("\\s+")
args.foreach(println);
var envStr = "dev";
var srcStr = "appm"
val RootFolderStr = "/source_folder/";
val DestFolderStr = "/dest_folder/";
val dateformatter = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss.SSS'Z'");
val formatter = new SimpleDateFormat("yyyy-MM-dd");
val theMonthFormatter = new SimpleDateFormat("yyyy-MM");
var fromDay: Date = formatter.parse("2018-04-29");
var toDay: Date = formatter.parse("2018-05-01");
if (args.length < 2) {
printf("usage: need at least 2 parameters in spark.driver.args");
sys.exit(2);
}
envStr = args(0).toLowerCase();
srcStr = args(1).toLowerCase();
if (args.length == 4) {
fromDay = formatter.parse(args(2));
toDay = formatter.parse(args(3));
}
if (args.length == 2) {
// default to be yesterday to today
toDay = formatter.parse(formatter.format(Calendar.getInstance().getTime()));
val previousDay = Calendar.getInstance();
previousDay.add(Calendar.DATE, -1);
fromDay = formatter.parse(formatter.format(previousDay.getTime()));
}
// get the sub-folder for the monthly partition
val monthFolder = theMonthFormatter.format(fromDay);
var rootFolder = RootFolderStr.replaceAll("ENV", envStr) + monthFolder;
rootFolder = rootFolder.replaceAll("SRC", srcStr);
val destFolder = DestFolderStr.replaceAll("ENV", envStr);
var toCalendar = Calendar.getInstance();
toCalendar.setTime(toDay);
toCalendar.add(Calendar.DATE, 1);
// need to consider the case across the month boundary
val toDay2 = formatter.parse(formatter.format(toCalendar.getTime()));
// filter out .tmp files and 0-size files
// .tmp files are not safe to read from, it's possible that the files are under updating by Flume job and the message data is incomplete
// when the Spark job starts to read from it.
val pathInfos = FileSystem.get(sc.hadoopConfiguration).listStatus(new Path(rootFolder));
// filter out the 0-length files, .tmp files which is of today
val allfiles = pathInfos.filter(fileStatus => {
if (fileStatus.getLen == 0)
false
else {
val aPath = fileStatus.getPath().getName();
// use the modification time is more accurate.
val lastTime = fileStatus.getModificationTime();
val aDate = new Date(lastTime);
// all files between fromDay and toDay2
aDate.after(fromDay) && aDate.before(toDay2);
}
}
).map(_.getPath.toString);
case class event_log(
time_stp: Long,
msg_sze: Int,
msg_src: String,
action_path: String,
s_code: Int,
s_desc: String,
p_code: String,
c_id: String,
m_id: String,
c_ip: String,
c_gp: String,
gip: String,
ggip: String,
rbody: String
);
def readEvents(fileList: Array[String], msgSrc: String, fromTS: Long, toTS: Long): RDD[(event_log)] = {
val records =
sc.sequenceFile[Long, String](fileList.mkString(","))
.filter((message) => {
(message._1 >= fromTS && message._1 < toTS);
}
)
val eventLogs = records.map((message) => {
val time_stp = message._1;
var msg_sze = message._2.length();
var c_id = ""
var m_id = "";
var p_code = "";
var c_ip = "";
var c_gp = "";
var gip = "";
var ggip = "";
var rbody = "";
var action_path = "";
var s_code: Int = 200;
var s_desc = "";
try {
// parse the message
val mapper = new ObjectMapper();
val aBuff = message._2.getBytes();
val root = mapper.readTree(aBuff);
var aNode = root.path("rbody");
rbody = aNode.textValue();
if (rbody != null && rbody.length() > 0) {
val mapper_2 = new ObjectMapper();
val aBuff_2 = rbody.getBytes();
var root2 = mapper_2.readTree(aBuff_2);
aNode = root2.path("p_code");
if (aNode != null && aNode.isValueNode())
p_code = String.valueOf(aNode.intValue());
aNode = root2.path("mkay");
if (aNode != null && aNode.isObject()) {
root2 = aNode;
}
{
aNode = root2.get("c_id");
if (aNode != null && aNode.isValueNode())
c_id = aNode.textValue();
aNode = root2.get("m_id");
if (aNode != null && aNode.isValueNode()) {
m_id = String.valueOf(aNode.intValue());
}
}
}
aNode = root.path("c_ip");
c_ip = aNode.textValue();
aNode = root.path("c_gp");
c_gp = aNode.textValue();
aNode = root.path("gip");
gip = aNode.textValue();
aNode = root.path("ggip");
ggip = aNode.textValue();
aNode = root.path("action_path");
action_path = aNode.textValue();
aNode = root.path("s_code");
val statusNodeValue = aNode.textValue().trim();
s_code = Integer.valueOf(statusNodeValue.substring(0, 3));
s_desc = statusNodeValue.substring(3).trim();
}
catch {
// return empty string as indicator that it's not a well-formatted JSON message
case jex: JsonParseException => {
msg_sze = 0
};
case ioEx: java.io.IOException => {
msg_sze = 0
};
case rtEx: JsonMappingException => {
msg_sze = 0
};
}
event_log(time_stp, msg_sze, msgSrc, action_path, s_code, s_desc,
p_code, c_id, m_id,
c_ip, c_gp, gip, ggip,
rbody);
});
eventLogs;
}
val hiveContext = new HiveContext(sc)
if (allfiles.length == 0)
sys.exit(3);
val fromTime = fromDay.getTime();
val toTime = toDay.getTime();
val events = readEvents(allfiles, srcStr, fromTime, toTime);
val df = hiveContext.createDataFrame(events).coalesce(1);
df.write.parquet(destFolder);
sys.exit(0);

F# CsvTypeProvider extracting the same columns from slightly different csv-files

I am creating a program that reads football matches from different CSV files. The columns I am interested in are present in all the files, but the files have a varying number of columns.
This left me creating a separate mapping function for each variation of file, with a different sample for each type:
type GamesFile14 = CsvProvider<"./data/sample_14.csv">
type GamesFile15 = CsvProvider<"./data/sample_15.csv">
type GamesFile1617 = CsvProvider<"./data/sample_1617.csv">
let mapRows14 (rows:seq<GamesFile14.Row>) = rows |> Seq.map ( fun c -> { Division = c.Div; Date = DateTime.Parse c.Date;
HomeTeam = { Name = c.HomeTeam; Score = c.FTHG; Shots = c.HS; ShotsOnTarget = c.HST; Corners = c.HC; Fouls = c.HF };
AwayTeam = { Name = c.AwayTeam; Score = c.FTAG; Shots = c.AS; ShotsOnTarget = c.AST; Corners = c.AC; Fouls = c.AF };
Odds = { H = float c.B365H; U = float c.B365D; B = float c.B365A } } )
let mapRows15 (rows:seq<GamesFile15.Row>) = rows |> Seq.map ( fun c -> { Division = c.Div; Date = DateTime.Parse c.Date;
HomeTeam = { Name = c.HomeTeam; Score = c.FTHG; Shots = c.HS; ShotsOnTarget = c.HST; Corners = c.HC; Fouls = c.HF };
AwayTeam = { Name = c.AwayTeam; Score = c.FTAG; Shots = c.AS; ShotsOnTarget = c.AST; Corners = c.AC; Fouls = c.AF };
Odds = { H = float c.B365H; U = float c.B365D; B = float c.B365A } } )
let mapRows1617 (rows:seq<GamesFile1617.Row>) = rows |> Seq.map ( fun c -> { Division = c.Div; Date = DateTime.Parse c.Date;
HomeTeam = { Name = c.HomeTeam; Score = c.FTHG; Shots = c.HS; ShotsOnTarget = c.HST; Corners = c.HC; Fouls = c.HF };
AwayTeam = { Name = c.AwayTeam; Score = c.FTAG; Shots = c.AS; ShotsOnTarget = c.AST; Corners = c.AC; Fouls = c.AF };
Odds = { H = float c.B365H; U = float c.B365D; B = float c.B365A } } )
These are again consumed by the loadGames function:
let loadGames season resource =
if season.Year = 14 then GamesFile14.Load(resource).Rows |> mapRows14
else if season.Year = 15 then GamesFile15.Load(resource).Rows |> mapRows15
else GamesFile1617.Load(resource).Rows |> mapRows1617
It seems to me that there must be better ways to get around this problem.
Is there any way I could make my mapping function more generic so that I don't have to repeat the same function over and over?
Is it possible to create the CsvProvider on the fly based on the resource, or do I need to explicitly declare a sample for each variation of my csv-files like in the code above?
Other suggestions?
In your scenario, you might get better results from FSharp.Data's CsvFile type. It uses a more dynamic approach to CSV parsing, using the dynamic ? operator for data access: you lose some of the type-safety guarantees that the type provider gives you, since each separate CSV file will be loaded into the save CsvRow type -- which means that you can't guarantee at compile time that any given column will be in a file, and you have to be prepared for runtime errors. But in your case, that's just what you want, because it would allow your three functions to be rewritten like this:
let mapRows14 rows = rows |> Seq.map ( fun c -> { Division = c?Div; Date = DateTime.Parse c?Date;
HomeTeam = { Name = c?HomeTeam; Score = c?FTHG; Shots = c?HS; ShotsOnTarget = c?HST; Corners = c?HC; Fouls = c?HF };
AwayTeam = { Name = c?AwayTeam; Score = c?FTAG; Shots = c?AS; ShotsOnTarget = c?AST; Corners = c?AC; Fouls = c?AF };
Odds = { H = float c?B365H; U = float c?B365D; B = float c?B365A } } )
Give CsvFile a try and see if it solves your problem.

Linq To SQL Group By and Sum

Her is an image showing a the table i have, b the grid i need to display.
opps cant post image. ill try to explain. My table have four colums.
Project Number(String)
ItemNumber(String)
Location(String)
Qty.(Real).
My grid need to look like this.
ProjectNumber
ItemNumber
QtyMain.
QtyOther.
I Need to write a linq query Grouping evry line so i will se 1 Line pr Project/ItemNumber combination summing qty into 2 differetn colums 1 showing to qty where location is main and 1 showing the qty where location is not (!=) main. Can linq do this for me, or how can thsi be done?
public class Foo
{
public Int32 ProjectNumber;
public String ItemNumber;
public String InventLocation;
public Int32 Qty;
}
void Main()
{
List<Foo> foos = new List<Foo>(new[]{
new Foo { ProjectNumber = 1, ItemNumber = "a", InventLocation = "Main", Qty = 3 },
new Foo { ProjectNumber = 1, ItemNumber = "a", InventLocation = "Main", Qty = 3 },
new Foo { ProjectNumber = 1, ItemNumber = "a", InventLocation = "Sub", Qty = 2 },
new Foo { ProjectNumber = 1, ItemNumber = "a", InventLocation = "Sub", Qty = 1 },
new Foo { ProjectNumber = 1, ItemNumber = "a", InventLocation = "Sub2", Qty = 5 }
});
var foo = from f in foos
group f by new { f.ProjectNumber, f.ItemNumber } into fGroup
select new {
ProjectNumber = fGroup.Key.ProjectNumber,
ItemNumber = fGroup.Key.ItemNumber,
QtyMain = fGroup.Where (g => g.InventLocation == "Main").Sum (g => g.Qty),
Rest = fGroup.Where (g => g.InventLocation != "Main").Sum (g => g.Qty)
};
foo.Dump();
}
Resulting in:
IEnumerable<> (1 item)
ProjectNumber ItemNumber QtyMain Rest
1 a 6 8

Groovy - Cleaning up functions

I have a gross looking function and am wanting to clean it up. All it contains is some maps, a for loop, and if statements.
Basically I have a map that I would like to grab only certain information from but one of the keys that I need from it changes by one number in each map.
I was thinking maybe a simple switch statement or something should fix it, but I often get confused with simplifying such things.
Here is what the function looks like:
public void separateBooksFromList() {
book1 = [:] //map for book1
book2 = [:] //map for book2
book3 = [:] //map for book3
book4 = [:] //map for book4
book5 = [:] //map for book5
book6 = [:] //map for book6
book7 = [:] //map for book7
book8 = [:] //map for book8
book9 = [:] //map for book9
book10 = [:] //map for book10
lastModified = new Date(dataFile.lastModified()) //last time the file was scanned
readDate = new Date() //current date the text file was read
for(int i = 0; i < bookList.size(); i++) {
if(i==0) {
book1['lastScan'] = lastModified
book1['readDate'] = readDate
book1['bookNumber'] = bookList['Book Number 0'][i] // <- only part of the map that changes
book1['bookTitle'] = bookList['Book Title'][i]
}
if(i==1) {
book2['lastScan'] = lastModified
book2['readDate'] = readDate
book2['bookNumber'] = bookList['Book Number 1'][i] // <- only part of the map that changes
book2['bookTitle'] = bookList['Book Title'][i]
}
if(i==2) {
book3['lastScan'] = lastModified
book3['readDate'] = readDate
book3['bookNumber'] = bookList['Book Number 2'][i] // <- only part of the map that changes
book3['bookTitle'] = bookList['Book Title'][i]
}
if(i==3) {
book4['lastScan'] = lastModified
book4['readDate'] = readDate
book4['bookNumber'] = bookList['Book Number 3'][i] // <- only part of the map that changes
book4['bookTitle'] = bookList['Book Title'][i]
}
if(i==4) {
book5['lastScan'] = lastModified
book5['readDate'] = readDate
book5['bookNumber'] = bookList['Book Number 4'][i] // <- only part of the map that changes
book5['bookTitle'] = bookList['Book Title'][i]
}
if(i==5) {
book6['lastScan'] = lastModified
book6['readDate'] = readDate
book6['bookNumber'] = bookList['Book Number 5'][i] // <- only part of the map that changes
book6['bookTitle'] = bookList['Book Title'][i]
}
if(i==6) {
book7['lastScan'] = lastModified
book7['readDate'] = readDate
book7['bookNumber'] = bookList['Book Number 6'][i] // <- only part of the map that changes
book7['bookTitle'] = bookList['Book Title'][i]
}
if(i==7) {
book8['lastScan'] = lastModified
book8['readDate'] = readDate
book8['bookNumber'] = bookList['Book Number 7'][i] // <- only part of the map that changes
book8['bookTitle'] = bookList['Book Title'][i]
}
if(i==8) {
book9['lastScan'] = lastModified
book9['readDate'] = readDate
book9['bookNumber'] = bookList['Book Number 8'][i] // <- only part of the map that changes
book9['bookTitle'] = bookList['Book Title'][i]
}
if(i==9) {
book10['lastScan'] = lastModified
book10['readDate'] = readDate
book10['bookNumber'] = bookList['Book Number 9'][i] // <- only part of the map that changes
book10['bookTitle'] = bookList['Book Title'][i]
}
}
}
As you can see, it's quite an ugly function.
Am I able to do a simple switch statement or something to cut down the code and make it look more professional?
There are a few things you can do here. You can use a list of books rather than book1, book2, etc... This will save repeating yourself so much.
This is almost the same but with that change. It will create a list of size 10 (assuming 10 books) with a map entry like what you had before.
public void separateBooksFromList() {
lastModified = new Date(dataFile.lastModified()) //last time the file was scanned
readDate = new Date() //current date the text file was read
// Use a list of Maps instead of separate variables
int numberOfBooks = bookList.size()
def books = []
numberOfBooks.times {
books[it] = [
lastScan: lastModified,
readDate: readDate,
bookNumber: booklist["Book Number $it"][it],
bookTitle: booklist["BookTitle"][it]
]
}
}