Swift Large Data File Into Struct (JSON or CSV) - json

Question

I've got a huge amount of data (80 columns over 60k rows). It's created in Excel and exported to CSV. The CSV file is 9.4mb. I then converted the CSV to JSON and it became 94.5mb. Obviously I can't have a 100mb data file included with the app so I tried thinking of different solutions.
I used the CSV itself and parsed it out into a struct. However, because the CSV has commas within some of the data (surrounded by double quotes of course), it isn't parsing correctly.
I could reformat the JSON to have the headers in one array and the data in another which would solve my comma issue, but I haven't found an up-to-date real-world solution to doing this.
This is medical drug data so some of the names can be quite long/complex, and a lot of rows have blank values for various fields simply because there is no corresponding data for that entry.
Here's a small example of the CSV formatted data:
medID,type,nameLong,isCurrent,nameShort,isCompound,strength,mUnit,strengthPer,mUnitPer,strengthIn,mUnitIn,ing1,ing1Strength,ing1MUnit,ing2,ing2Strength,ing2MUnit,ing3,ing3Strength,ing3MUnit,ing4,ing4Strength,ing4MUnit,ing5,ing5Strength,ing5MUnit,ing6,ing6Strength,ing6MUnit,ing7,ing7Strength,ing7MUnit,ing8,ing8Strength,ing8MUnit,ing9,ing9Strength,ing9MUnit,ing10,ing10Strength,ing10MUnit,ing11,ing11Strength,ing11MUnit,ing12,ing12Strength,ing12MUnit,ing13,ing13Strength,ing13MUnit,ing14,ing14Strength,ing14MUnit,ing15,ing15Strength,ing15MUnit,ing16,ing16Strength,ing16MUnit,ing17,ing17Strength,ing17MUnit,ing18,ing18Strength,ing18MUnit,ing19,ing19Strength,ing19MUnit,ing20,ing20Strength,ing20MUnit,ing21,ing21Strength,ing21MUnit,ing22,ing22Strength,ing22MUnit,ing23,ing23Strength,ing23MUnit
123456,Inhalant,"Fluticasone Propionate/Salmeterol 500/50 mcg/INHAL Dry Powder Inhaler, 14 Blisters, Inhalant",N,,TRUE,,,,,,,Fluticasone Propionate,500,mcg/INHAL,Salmeterol,50,mcg/INHAL,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
234567,Inhalant,"Advair Diskus 500/50 mcg/INHAL Dry Powder Inhaler, 14 Blisters, Inhalant",N,Advair Diskus,TRUE,,,,,,,Fluticasone Propionate,500,mcg/INHAL,Salmeterol,50,mcg/INHAL,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
345678,Powder,"Fluticasone Propionate 500 mcg / Salmeterol 50 mcg (Salmeterol Xinafoate 72.5 mcg) Per Actuat Dry Powder Inhaler, 28 Actuat",O,,TRUE,,,,,,,Fluticasone Propionate,500,mcg,Salmeterol,50,mcg,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
456789,Powder,"Advair Diskus 500/50 (Fluticasone Propionate 0.5 mg/ACTUAT / Salmeterol 0.05 mg/ACTUAT) Dry Powder Inhaler, 28 Actuat",O,,TRUE,,,,,,,Fluticasone Propionate,0.5,mg/ACTUAT,Salmeterol,0.05,mg/ACTUAT,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
567890,Inhalant,"Fluticasone Propionate/Salmeterol 500/50 mcg/INHAL Dry Powder Inhaler, 60 Blisters, Inhalant",N,,TRUE,,,,,,,Fluticasone Propionate,500,mcg/INHAL,Salmeterol,50,mcg/INHAL,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
678901,Inhalant,"Advair Diskus 500/50 mcg/INHAL Dry Powder Inhaler, 60 Blisters, Inhalant",N,Advair Diskus,TRUE,,,,,,,Fluticasone Propionate,500,mcg/INHAL,Salmeterol,50,mcg/INHAL,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
What I need is a solution to getting all of this data into the app so it can be accessed in the fastest manner. As it will only be used in one section of the app, I wasn't sure if putting it all in CoreData was a good idea (plus, it will be constantly updated), and instead thought it best to load it where it's needed and then unload it once the user leaves that section of the app, thus freeing up memory. However, I'm open to the best solution.
This is where I'm at right now with parsing the CSV file directly (and it works, sort of).
struct DrugsList: Codable, Hashable {
let medID: String
let type: String
let nameLong: String
let isCurrent: String
let nameShort: String?
let isCompound: String
let strength: String?
let mUnit: String?
let strengthPer: String?
let mUnitPer: String?
let strengthIn: String?
let mUnitIn: String?
let ing1: String?
let ing1Strength: String?
let ing1MUnit: String?
let ing2: String?
let ing2Strength: String?
let ing2MUnit: String?
let ing3: String?
let ing3Strength: String?
let ing3MUnit: String?
let ing4: String?
let ing4Strength: String?
let ing4MUnit: String?
let ing5: String?
let ing5Strength: String?
let ing5MUnit: String?
let ing6: String?
let ing6Strength: String?
let ing6MUnit: String?
let ing7: String?
let ing7Strength: String?
let ing7MUnit: String?
let ing8: String?
let ing8Strength: String?
let ing8MUnit: String?
let ing9: String?
let ing9Strength: String?
let ing9MUnit: String?
let ing10: String?
let ing10Strength: String?
let ing10MUnit: String?
let ing11: String?
let ing11Strength: String?
let ing11MUnit: String?
let ing12: String?
let ing12Strength: String?
let ing12MUnit: String?
let ing13: String?
let ing13Strength: String?
let ing13MUnit: String?
let ing14: String?
let ing14Strength: String?
let ing14MUnit: String?
let ing15: String?
let ing15Strength: String?
let ing15MUnit: String?
let ing16: String?
let ing16Strength: String?
let ing16MUnit: String?
let ing17: String?
let ing17Strength: String?
let ing17MUnit: String?
let ing18: String?
let ing18Strength: String?
let ing18MUnit: String?
let ing19: String?
let ing19Strength: String?
let ing19MUnit: String?
let ing20: String?
let ing20Strength: String?
let ing20MUnit: String?
let ing21: String?
let ing21Strength: String?
let ing21MUnit: String?
let ing22: String?
let ing22Strength: String?
let ing22MUnit: String?
let ing23: String?
let ing23Strength: String?
let ing23MUnit: String?
init(raw: [String]) {
medID = raw[0]
type = raw[1]
nameLong = raw[2]
isCurrent = raw[3]
nameShort = raw[4]
isCompound = raw[5]
strength = raw[6]
mUnit = raw[7]
strengthPer = raw[8]
mUnitPer = raw[9]
strengthIn = raw[10]
mUnitIn = raw[11]
ing1 = raw[12]
ing1Strength = raw[13]
ing1MUnit = raw[14]
ing2 = raw[15]
ing2Strength = raw[16]
ing2MUnit = raw[17]
ing3 = raw[18]
ing3Strength = raw[19]
ing3MUnit = raw[20]
ing4 = raw[21]
ing4Strength = raw[22]
ing4MUnit = raw[23]
ing5 = raw[24]
ing5Strength = raw[25]
ing5MUnit = raw[26]
ing6 = raw[27]
ing6Strength = raw[28]
ing6MUnit = raw[29]
ing7 = raw[30]
ing7Strength = raw[31]
ing7MUnit = raw[32]
ing8 = raw[33]
ing8Strength = raw[34]
ing8MUnit = raw[35]
ing9 = raw[36]
ing9Strength = raw[37]
ing9MUnit = raw[38]
ing10 = raw[39]
ing10Strength = raw[40]
ing10MUnit = raw[41]
ing11 = raw[42]
ing11Strength = raw[43]
ing11MUnit = raw[44]
ing12 = raw[45]
ing12Strength = raw[46]
ing12MUnit = raw[47]
ing13 = raw[48]
ing13Strength = raw[49]
ing13MUnit = raw[50]
ing14 = raw[51]
ing14Strength = raw[52]
ing14MUnit = raw[53]
ing15 = raw[54]
ing15Strength = raw[55]
ing15MUnit = raw[56]
ing16 = raw[57]
ing16Strength = raw[58]
ing16MUnit = raw[59]
ing17 = raw[60]
ing17Strength = raw[61]
ing17MUnit = raw[62]
ing18 = raw[63]
ing18Strength = raw[64]
ing18MUnit = raw[65]
ing19 = raw[66]
ing19Strength = raw[67]
ing19MUnit = raw[68]
ing20 = raw[69]
ing20Strength = raw[70]
ing20MUnit = raw[71]
ing21 = raw[72]
ing21Strength = raw[73]
ing21MUnit = raw[74]
ing22 = raw[75]
ing22Strength = raw[76]
ing22MUnit = raw[77]
ing23 = raw[78]
ing23Strength = raw[79]
ing23MUnit = raw[80]
}
}
func loadDrugsCSV(from fileName: String) -> [DrugsList] {
var csvToStruct = [DrugsList]()
// Locate the CSV File
guard let filePath = Bundle.main.path(forResource: fileName, ofType: "csv") else {
return []
}
// Convert the contents of the CSV file into one long string
var data = ""
do {
data = try String(contentsOfFile: filePath)
} catch {
print(error)
return []
}
// Split the long string into an array of rows
// Detect carriage returns "\n", then split
var rows = data.components(separatedBy: "\n")
// Remove header rows
rows.removeFirst()
// Loop through each row and split
for row in rows {
let csvColumns = row.components(separatedBy: ",")
let drugStruct = DrugsList.init(raw: csvColumns)
csvToStruct.append(drugStruct)
}
return csvToStruct
}