I am pulling data from a .csv file generated after a URL request using UrlFetchApp.fetch and storing the data using the logger.
Is there a way to write each row of fetched data from the logger into separate rows in a spreadsheet?
I did this quite a while ago. It's how I import data from website visitor logs. To review them. I don't use commas because you find them in the data too often. I use 3 tildes '~~~' instead and the lines are separated with a line feed '\n'.
But I basically split the lines and then fields on each line into array and feed them into the sheets one line at a time and usually I deal with 20 or 30 files about 100K or less. And I find that it loads the files very quickly.
function importData1(myFolderID,myFolderName,myFileName) {
var myFolderID = typeof(myFolderID) !== 'undefined' ? myFolderID : 'FolderID';
var myFileName = typeof(myFileName) !== 'undefined' ? myFileName : '';
if(myFileName && myFolderID)
{
var fi = DriveApp.getFolderById(myFolderID).getFilesByName(myFileName); // Selected IPLogYYMMDD.txt file
var ssid = SpreadsheetApp.getActive().getId();
var ss = SpreadsheetApp.openById(ssid);
if (fi.hasNext() ) // proceed if file exists in the IPlogs folder
{
var file = fi.next();
var data = file.getBlob().getDataAsString();
var lines = data.split('\n');
var newsheet = ss.insertSheet(myFolderName + '/' + myFileName);
var j=0;
for (var i=0; i<lines.length; i++ )
{
var fields = lines[i].split('~~~');
if(fields.length>=8)//There's supposed to be 8 or 9 fields
{
Logger.log('i=' + i + 'fields.length=' + fields.length);
newsheet.getRange(j+1, 1, 1,fields.length).setValues([fields]);
j=j+1;
}
}
}
}
else
{
displayStatus('Error Importing Data','Either Folder or File not found in importData1');
}
Most of the variables are easy to figure out. You'll probably be able to adapt it to what you need. And probably there will be several optional answers for you to chose from.
Related
I would like to download a 10000 row long singular sheet from Google Sheets split into 20 separate csv files with batch size of 500 rows each. Could this be possibly done using Google Sheets?
I wasn't able to find a native support for sheets to download and separate the files into multiple parts/fragments. But I was able to find a Google Apps Script code that saves the sheet into a csv.
So what I did was modify the code and then make it turn a sheet into multiple csv's.
The catch is, it will upload all those files into a folder in the logged in google drive and then you can download them in a zip file. (I'll show this later below)
Here is the source code.
And here is the modified code I did:
function saveAsCSV() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getActiveSheet();
// Update numFiles into how many files you want is outputted
// The script will calculate how many rows it can place in a file equally
// Excess will be placed into an additional file
var numFiles = 5;
var rowsPerFile = Math.floor(sheet.getDataRange().getLastRow() / numFiles);
// create a folder from the name of the spreadsheet
var folder = DriveApp.createFolder(ss.getName().toLowerCase().replace(/ /g,'_') + '_csv_' + new Date().getTime());
// append ".csv" extension to the sheet name
fileName = sheet.getName();
// convert all available sheet data to csv format
var csvFiles = convertRangeToCsvFile_(sheet, rowsPerFile);
// create a file in the Docs List with the given name and the csv data
csvFiles.forEach(function(csvFile, i){
var csvName = fileName + '_' + (i + 1) + '.csv';
folder.createFile(csvName, csvFile);
});
}
function convertRangeToCsvFile_(sheet, rowsPerFile) {
// get available data range in the spreadsheet
var activeRange = sheet.getDataRange();
try {
var data = activeRange.getValues();
var csvFiles = [];
// loop through the data in the range and build a string with the csv data
if (data.length > 1) {
var csv = "";
for (var row = 0; row < data.length; row++) {
for (var col = 0; col < data[row].length; col++) {
if (data[row][col].toString().indexOf(",") != -1) {
data[row][col] = "\"" + data[row][col] + "\"";
}
}
// join each row's columns
// add a carriage return to end of each row, except for the last one
if (row < data.length-1) {
csv += data[row].join(",") + "\r\n";
}
else {
csv += data[row];
}
if ((row + 1) % rowsPerFile === 0) {
csvFiles.push(csv);
csv = "";
}
}
}
// If there are remainders that doesnt reach the number of files needed.
// It will add another file, containing all remainder rows there.
// 10010 rows / 20 files = 10 rows excess
// 21st file will contain those 10 rows excess
// No excess means no 21st file
if(csv) {
csvFiles.push(csv);
}
return csvFiles;
}
catch(err) {
Logger.log(err);
}
}
Sample Data:
Sample Output:
Upon right clicking the folder created in Google Drive, you will have the option to download the whole folder as zip file containing all the csv files.
Here is the downloaded zip file containing all csv files:
Since there are 22 rows and the sample code wants 5 files, a 6th file was created containing the excess 2 rows
I am trying to get the file name for a file I only have the Google Drive URL for. I am currently using a Google Sheets regexextract function to extract the file ID from the URL, and then using the script to find the file based on the ID, but would prefer to do the regex within the script.
I've looked through various posts on here to try to figure it out with no luck. Hopefully someone can spell out what I have done wrong in trying to incorporate the regex.
var sheet = SpreadsheetApp.getActive().getSheetByName("Test1");
var link1 = sheet.getRange("N2:N").getValues();
var regex_ids = new RegExp("/file/d/[a-zA-Z0-9]g");
var links = regex_ids.exec(link1);
var filenames = [];
for (var i = 0; i < links.length; i++) {
var url = links[i][0];
if (url != "") {
var filename = DriveApp.getFileById(links[i][0]).getName();
filenames.push([filename]);
}
}
var startRow = 2; // print in row 2 since row 1 is the header row
var fileNameColumn = 18; // Column B = column 2
var destination = sheet.getRange(startRow, fileNameColumn, filenames.length, filenames[0].length);
destination.setValues(filenames);
}
Currently I am stuck with error "TypeError: Cannot read property "length" from null. (line 7, file "Code")" because the regex is not configured correctly.
Issues/Solution:
Invalid Syntax:
Regexp() accepts a regex string and a flag string as arguments, while the code provides a concatenated regex flag string.
exec() accepts a string argument, while the code provides a 2D array.
Insufficient regex:
Filename IDs also contain underscores _.
Regex should capture() only the ID. The regex provided also captures /file/d
ID contains more than 1 character. Use +
Snippet:
var link1 = sheet.getRange("N2:N" + sheet.getLastRow()).getValues();//modified
var regex_ids = /\/file\/d\/([^\/]+)/;//or new RegExp("/file/d/([a-zA-Z0-9_]+)","g");() =capture ids
//var links = regex_ids.exec(link1);
var filenames = [];
for (var i = 0; i < link1.length; i++) {//modified;loop through values2D array
var url = link1[i][0];//modified;
var preId = regex_ids.exec(url);//added;
var id;
if (preId && (id=preId[1])) {//modified; [1] = first capture group
var filename = DriveApp.getFileById(id).getName();//modified
filenames.push([filename]);
} else {
filenames.push([""]);
}
}
References:
Regex guide
Regexp#exec
I receive a new CSV file every hour in my Google Drive.
I need my spreadsheet updated with the data in the latest CSV file after it has been received in the Google Drive folder.
The files coming into the folder has a unique name for each new one according to date and time.
For example: FileName_date_24hourtime.csv
FileName_20190524_1800.csv then FileName_20190524_1900.csv etc.
Firstly I'm not sure what the best approach is:
simply with a formula (probably not possible with not knowing the exact filename?) like =IMPORTDATA
a google script to find latest .csv file and automatically import as soon as file was added to Google Drive folder
Any assistance will be great!
The .csv file:
.csv file contains 28 rows and data should be split by ;
.csv file looks like this:
NAME;-63.06;-58.08;50.62;-66.67;-80.00
NAME;-61.82;-56.83;-50.55;-77.78;-70.00
NAME;-57.77;-50.21;52.88;-77.78;-70.00
NAME1;-57.69;-61.48;-55.59;-55.56;-60.00
NAME2;-61.62;-53.79;50.34;-66.67;-70.00
NAME3;-54.62;-54.57;-52.22;55.56;-60.00
... with total of 28 rows
Data should go to "Import_Stats" sheet.
The best approach here would be a script with a trigger that runs a function that performs data import to a spreadsheet.
Create a time-based trigger with 1-hour offset:
function trigger() {
var trg = ScriptApp.newTrigger('checkFiles');
trg.timeBased().everyHours(1).create();
}
Create function that checks files in a folder (e.g. "checkFiles").
function checkFiles(alreadyWaited) {
//get spreadsheet and sheet;
var id = 'yourSpreadsheetId';
var ss = SpreadsheetApp.openById(id);
var sh = ss.getSheetByName('Import_Stats');
var folderId = 'yourIdHere'; //folder by id is the simplest way;
//get folder and files in it;
var folder = DriveApp.getFolderById(folderId);
var files = folder.getFilesByType('text/csv');
var filesImport = folder.getFilesByType('text/csv'); //fetch files again;
//try to fetch number of files;
var scriptProps = PropertiesService.getScriptProperties();
var numFiles = scriptProps.getProperty('numFiles');
//count current number of files;
var numCurr = 0;
while(files.hasNext()) {
var f = files.next();
numCurr++;
}
//if this is the first time, save current number;
if(numFiles===null) {
scriptProps.setProperty('numFiles',numCurr);
}else {
numFiles = parseInt(numFiles);
}
if(numFiles===null||numFiles===(numCurr-1)) {
//get today and reset everything except hours;
var today = new Date();
today.setMinutes(0);
today.setSeconds(0);
today.setMilliseconds(0);
//iterate over files;
while(files.hasNext()) {
var file = files.next();
//get file creation date and reset;
var created = file.getDateCreated();
created.setMinutes(0);
created.setSeconds(0);
created.setMilliseconds(0);
//calculate offset, equals 0 for each file created this hour;
var offset = today.valueOf()-created.valueOf();
if(offset===0) {
//perform data import here;
var data = file.getBlob().getDataAsString();
//ignore empty files;
if(data!=='') {
//split data in rows;
var arr = data.split('\r\n');
//resplit array if only one row;
if(arr.length===1) {
arr = data.split('\n');
}
//append rows with data to sheet;
arr.forEach(function(el){
el = el.split(';');
sh.appendRow(el);
});
}
}
}
}else {
//if never waited, set minute to wait, else add minute;
if(!alreadyWaited) {
alreadyWaited = 60000;
}else {
alreadyWaited += alreadyWaited;
}
//if waited for 10 minutes -> end recursion;
if(alreadyWaited===600000) {
Logger.log('Waited 10 minutes but recieved no files!');
return;
}
//wait a minute and recheck;
Utilities.sleep(60000);
return checkFiles(alreadyWaited);
}
}
And this is what should happen:
I am trying to replicate some functionality from a Excel document with some macros in a Google Spreadsheet. The end goal is to export a fixed-width txt file. Unfortunately, the vendor cannot use a CSV file. So, does anybody know of a way to generate a fixed-width TXT file using Google Scripts?
Just in case anybody else comes looking. I used a few sources and came up with this:
function saveAsFixedWidthTxt() {
// get Spreadsheet Name
var fileName = SpreadsheetApp.getActiveSpreadsheet().getName();
//var fileName = SpreadsheetApp.getActiveSheet().getSheetName();
// get Directory the Spreadsheet is stored in.
var dirName = DocsList.getFileById(SpreadsheetApp.getActive().getId()).getParents()[0].getName();
// Add the ".txt" extension to the file name
fileName = fileName + ".txt";
// Convert the range data to fixed width format
var txtFile = convertRangeToTxtFile_(fileName);
// Delete existing file
deleteDocByName(fileName);
// Create a file in the Docs List with the given name and the data
DocsList.getFolder(dirName).createFile(fileName, txtFile);
}
function convertRangeToTxtFile_(txtFileName) {
try {
var txtFile = undefined;
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheets()[0];
var rows = sheet.getDataRange();
var data = rows.getValues();
// Loop through the data in the range and build a string with the data
if (data.length > 1) {
var txt = "";
for (var row = 2; row < data.length; row++) {
for (var j=6; j<=10; j++){
if(data[row][j] != ''){
// Employee ID
var empID = "" + data[row][3];
txt += empID;
//Fill to 6 characters
for (i=empID.length; i<6; i++){
txt += " ";
}
// Name
var fullName = data[row][5] + " " + data[row][4]
txt += fullName;
//Fill to 43 characters
for (i=fullName.length; i<44; i++){
txt += " ";
}
// etc, etc to build out your line
txt += "\r\n";
}
}
}
txtFile = txt;
}
return txtFile;
}
catch(err) {
Logger.log(err);
Browser.msgBox(err);
}
}
function deleteDocByName(fileName){
var docs=DocsList.find(fileName)
for(n=0;n<docs.length;++n){
if(docs[n].getName() == fileName){
var ID = docs[n].getId()
DocsList.getFileById(ID).setTrashed(true)
}
}
}
This will work:
export the sheet as a xlsx, then open that file in Excel.
Export that file as Space Delimited Text (.prn, for some reason).
Change the file extension on the resultant file to .txt.
This will result in a file like this:
Column Another Column 3
Val 1 2 $ 5.00
Or do you need to get a .txt file directly out of Google Apps Script?
Edit since a direct .txt download is necessary.
Here's how you could design a script to do that:
Find a way to convert sheet.getColumnWidth(n) to a number of spaces. You may find that there are 5 pixels to 1 character, or whatever the ratio
Run getColumnWidth() for each column to find the width you need for each column. Alternatively, find the length longest string in each cell.
Go through each cell and add it to a large string you begin building. As you add it, add the number of spaces necessary to match the value converted from getColumnWidth(). At the end of each row, append \n to represent a new line.
Once the output string is complete, you could then use the Content Service to download the text like so:
ContentService.createTextOutput(youBigString).downloadAsFile('filename.txt')
This would involve deploying your script as a web app, which could work well - you'd go to a URL and the page would trigger a download of the fixed-width file.
In my case, DocList use returns ReferenceError.#MSCF your lines 7 & 27 :
var dirName = DocsList.getFileById(SpreadsheetApp.getActive().getId()).getParents()[0].getName();
DocsList.getFolder(dirName).createFile(fileName, txtFile);
become
var dirName = DriveApp.getFileById(SpreadsheetApp.getActiveSpreadsheet().getId()).getParents().next().getId();
DriveApp.getFileById(SpreadsheetApp.getActiveSpreadsheet().getId()).getParents().next().createFile(fileName, txtFile);
Problem:-
I need to get Name and ID of the latest created document from some Google Drive folders.
Solution:-
I wasn’t able to find an easy way to get it, hence, created a loop for files within the folder to get max date - then loop again in folder to match the date and Log the name and date – the below code (when commented as mentioned) works properly for the solution.
Instead of rewriting the whole code with just a different folder name, I tried creating a loop for folder name as well.
However app script goes into infinite loop giving out the Maximum execution time limit message.
Any help appreciated.
function Get_lastestdate() {
//Define Spreadsheet and sheet
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName("PEOPLE");
//comment from here to work properly
for (var i =1; i < 4; i++) {
if (i = 1) { var foldername = "Folder1"}
if (i = 2) { var foldername = "Folder2"}
if (i = 3) { var foldername = "Folder3"}
// till here
//Define folder and files
var folder = DocsList.getFolder(foldername); //set folder name in case commenting.
var lastfile = folder.getFiles();
//Make a blank array
var noofdaysarray = []
var yeararray = []
//Loop to get all No of days in a year & the year
for (var i in lastfile) {
noofdaysarray.push(Utilities.formatDate(lastfile[i].getDateCreated(),"GMT","D"));
yeararray.push(Utilities.formatDate(lastfile[i].getDateCreated(),"GMT","y"));
}
//Get the max date from date and year
var largestdate = Math.max.apply(Math, noofdaysarray);
var largestyear = Math.max.apply(Math, yeararray);
//Get maximum available date
var matchcriteria = largestdate + largestyear
//Again loop for matching criteria with the actual date uploaded
for (var i in lastfile) {
var lastdate = Utilities.formatDate(lastfile[i].getDateCreated(),"GMT","D");
var lastyear = Utilities.formatDate(lastfile[i].getDateCreated(),"GMT","y");
var wholedate = parseInt(lastdate) + parseInt(lastyear); //parseInt is for converting text to number
//Get doc name if both dates matches
if (wholedate == matchcriteria) {
Logger.log(lastfile[i].getId());
Logger.log(lastfile[i].getName());
}
}
} //comment this as a part of loop
}
Between:- If there's an easier way to do it, please let me know.
You can use DocsList to retrieve the last modified files. you told that you want the last created but maybe the last modified can be usefull. Check the code bellow:
function findLastModified() {
//Retrieve folder
var folder = DocsList.getFolderById("0B0kQD4hSd4KASUJKb2cya0NET1U");
//Ask for 5 files
var numberOfFiles = 5;
//The first parameter is the search query, in this case we want all files
var filesResult = folder.findForPaging("", 5);
//By Default they will be sorted by last modified date
var files = filesResult.getFiles();
Logger.log("Found "+files.length+" files");
//Iterate
for(var x in files) {
var file = files[x];
Logger.log(file.getLastUpdated());
}
}
Live version here.
Dont use "i" in both inner and outer loops. that will be a problem.
wouldn't it be easier to do something like this so you only do one loop?
var mostRecentDate = new Date(2000,01,01);
var fileName="";
var fileId="";
for(var i = 1; i<5; i++){
var folder = DocsList.getFolder('Folder'+i);
var files = folder.getFiles();
for(var j in files) {
if(files[j].getDateCreated()>mostRecentDate){ //or getLastUpdated
mostRecentDate=files[j].getDateCreated();
fileName=files[j].getName();
fileId=files[j].getId();
}
}
}
Logger.log("File: " + filename + " Id: " + fileId + " Created: " + mostRecentDate);
You may need to do paging if you have a huge number of files & folders to iterate through.