I'm trying to get permanent download URLs when I upload files to Firebase Storage (Google Cloud Storage) from Firebase Cloud Functions (Google Cloud Functions).
I tried setting predefinedAcl to authenticatedRead and to publicRead. Both resulted in 403 (Forbidden) errors when my app tried to download the files. This is from the documentation for CreateWriteStreamOptions. Here's the code:
const {Storage} = require('#google-cloud/storage');
const storage = new Storage({ projectId: 'languagetwo-cd94d' });
const myBucket = storage.bucket('languagetwo-cd94d.appspot.com');
var mp3Promise = new Promise(function(resolve, reject) {
let options = {
metadata: {
contentType: 'audio/mp3',
public: true
}
};
synthesizeParams.accept = 'audio/mp3';
var file = myBucket.file('Audio/' + longLanguage + '/' + pronunciation + '/' + word + '.mp3');
textToSpeech.synthesize(synthesizeParams)
.then(function(audio) {
audio.pipe(file.createWriteStream(options));
})
.then(function() {
resolve('http://storage.googleapis.com/languagetwo-cd94d.appspot.com/Audio/' + longLanguage + '/' + pronunciation + '/' + word + '.mp3');
})
.catch(error => console.error(error));
});
That code executes without an error, writes the file to Storage, and passes the download URL to the next function. When I try to download the file with this URL:
http://storage.googleapis.com/languagetwo-cd94d.appspot.com/Audio/English/United_States-Allison-Female-IBM/catbirds.mp3
I get this error:
<Error>
<Code>AccessDenied</Code>
<Message>Access denied.</Message>
<Details>
Anonymous caller does not have storage.objects.get access to languagetwo-cd94d.appspot.com/Audio/English/United_States-Allison-Female-IBM/catbirds.mp3.
</Details>
</Error>
Downloading the file from my app (as an authorized user) I get the 403 (Forbidden) error message.
I've also tried this property, with the same result:
let options = {
metadata: {
contentType: 'audio/webm',
predefinedAcl: 'publicRead'
}
};
Moving on, I tried file.makePublic():
const {Storage} = require('#google-cloud/storage');
const storage = new Storage({ projectId: 'languagetwo-cd94d' });
const myBucket = storage.bucket('languagetwo-cd94d.appspot.com');
var mp3Promise = new Promise(function(resolve, reject) {
let options = {
metadata: {
contentType: 'audio/mp3'
}
};
synthesizeParams.accept = 'audio/mp3';
var file = myBucket.file('Audio/' + longLanguage + '/' + pronunciation + '/' + word + '.mp3');
textToSpeech.synthesize(synthesizeParams)
.then(function(audio) {
audio.pipe(file.createWriteStream(options));
})
.then(function() {
file.makePublic()
.then(function(data) {
console.log(data)
resolve('http://storage.googleapis.com/languagetwo-cd94d.appspot.com/Audio/' + longLanguage + '/' + pronunciation + '/' + word + '.mp3');
})
.catch(error => console.error(error));
})
.catch(error => console.error(error));
});
This code didn't even execute. It crashed at file.makePublic() and the error message was
{ Error: No such object: languagetwo-cd94d.appspot.com/Audio/English/United_States-Allison-Female-IBM/warblers.mp3
I'm not sure what this error means. My guess is that file.createWriteStream() wrote to a file location, and then file.makePublic() couldn't find that file location.
I found the permissions for warblers.mp3 on the Cloud Storage Browser:
the first error is due the access permissions to the bucket, in the link you can
find Identity and Access Management (IAM) instructions, there you can check the roles
and permission to manage the bucket access.
The second error could be a consequence of the first one.
Please let me know if the information works for you.
You need to change options to this:
let options = {
public: true,
metadata: {
contentType: 'audio/mp3'
}
};
https://googleapis.dev/nodejs/storage/latest/global.html#CreateWriteStreamOptions
I've been struggling with the same issue for a while and in the end here is what solved it for me:
const stream = file.createWriteStream(options)
audio.pipe(stream)
stream.on('finish', () => {
// file APIs will now work
})
The problem is that audio.pipe(file.createWriteStream(options)) creates a stream that writes to the file asyncronously. At the moment when you are calling file.makePublic() there is no guarantee that that write stream has completed.
Related
This question already has answers here:
Get Download URL from file uploaded with Cloud Functions for Firebase
(25 answers)
Closed 6 days ago.
I'm trying to get a Firebase Storage download URL (with long-lived token) from Cloud Functions for Firebase. I'm guessing two possible issues:
I'm using Google Cloud Storage, not Firebase Cloud Storage.
getDownloadURL() doesn't work from Cloud Functions.
I'm looking at the documentation for Storage and I see sections for
iOS+
Android
Web
Flutter
Admin
C++
Unity
Security & Rules
Why do I not see section for Cloud Functions for Firebase? Is Admin the same as Cloud Functions for Firebase?
There's a section Extend Cloud Storage with Cloud Functions but it's about triggering Cloud Functions from Storage, not using Cloud Functions to do stuff in Storage.
Let's try to get that download URL. The documentation says to do this:
var storage = firebase.storage();
var gsReference = storage.refFromURL('gs://bucket/images/stars.jpg');
storageRef.child('images/stars.jpg').getDownloadURL()
.then((url) => {
console.log(url);
});
But I create a Storage client with Google Cloud Storage:
import * as functions from "firebase-functions";
const admin = require('firebase-admin');
admin.initializeApp({
apiKey: 'abc123',
authDomain: 'my-app.firebaseapp.com',
credential: admin.credential.cert({serviceAccount}),
databaseURL: 'https://my-project.firebaseio.com',
storageBucket: 'gs://my-project.appspot.com',
});
const { Storage } = require('#google-cloud/storage');
const storage = new Storage();
const bucketName = 'gs://my-project.appspot.com';
const bucket = storage.bucket('my-project.appspot.com');
I can write a file to Firebase Storage:
await storage.bucket(bucketName).file(destFileName).save(file['rawBody']);
But this doesn't work:
await storage.bucket(bucketName).file(destFileName).getDownloadURL()
.then((url) => {
console.log("Download URL: " + url);
return url;
});
That throws an error:
TypeError: storage.bucket(...).file(...).getDownloadURL is not a function
That's making me suspect that getDownloadURL isn't available for Cloud Functions.
Let's try this:
var storageRef = admin.firebase.storage().ref(destFileName);
await storageRef.getDownloadURL()
.then(function (url) {
console.log(url);
});
That throws this error:
TypeError: Cannot read properties of undefined (reading 'storage')
That's worse, it's crashing on storage in the first line and not even getting to getDownloadURL.
Let's try this:
var gsReference = storage.refFromURL(bucketName + '/' + destFileName);
await gsReference.getDownloadURL()
.then(function (url) {
console.log("Download URL: " + url);
return url;
});
That throws the same error:
TypeError: Cannot read properties of undefined (reading 'storage')
Same problem, it can't get past storage in the first line.
I have been looking into the same problem last time, and you are right getDownloadURL() does not work in cloud function.
This is the other way round to get the downloadURL of a file in Cloud Storage that I am using currently, it basically generates a signed download url with expiration duration.
exports.signedUrl = functions.region("asia-southeast1").https.onRequest((request, response) => {
const { Storage } = require("#google-cloud/storage");
const storage = new Storage({ projectId: "YOUR_PROJECT_ID", keyFilename: "./serviceAccountKey.json" });
const bucketName = "YOUR_BUCKET_NAME";
generateV4ReadSignedUrl("YOUR_FILE_PATH").catch(console.error);
async function generateV4ReadSignedUrl(filename) {
const options = {
version: "v4",
action: "read",
expires: Date.now() + 2 * 60 * 1000, // 2 minutes
};
const [url] = await storage.bucket(bucketName).file(filename).getSignedUrl(options);
console.log(`Generated GET signed URL: ${url}`);
return cors()(request, response, () => {
response.send(url);
});
}
});
This question already has an answer here:
How can I resize all existing images in firebase storage?
(1 answer)
Closed 9 months ago.
I have requirement to resize new and existing images stored in firebase store. For new image, I enabled firebase's resize image extension. For existing image, how can I resized the image and get the newly resized image url to update back to database via api.
Here is my firebase function to get existing image urls from database. My question is how to resize the image and get the new image url?
const functions = require("firebase-functions");
const axios =require("axios");
async function getAlbums() {
const endpoint = "https://api.mydomain.com/graphql";
const headers = {
"content-type": "application/json",
};
const graphqlQuery = {
"query": `query Albums {
albums {
id
album_cover
}
}`
};
functions.logger.info("Call API");
const response = await axios({
url: endpoint,
method: 'post',
headers: headers,
data: graphqlQuery
});
if(response.errors) {
functions.logger.info("API ERROR : ", response.errors) // errors if any
} else {
return response.data.data.albums;
}
}
exports.manualGenerateResizedImage = functions.https.onRequest(async () => {
const albums = await getAlbums();
functions.logger.info("No. of Album : ", albums.length);
});
I think the below answer from Renaud Tarnec will definitely help you.
If you look at the code of the "Resize Images" extension, you will see that the Cloud Function that underlies the extension is triggered by a onFinalize event, which means:
When a new object (or a new generation of an existing object) is
successfully created in the bucket. This includes copying or rewriting
an existing object.
So, without rewriting/regenerating the existing images the Extension will not be triggered.
However, you could easily write your own Cloud Function that does the same thing but is triggered, for example, by a call to a specific URL (HTTPS cloud Function) or by creating a new document in a temporary Firestore Collection (background triggered CF).
This Cloud Function would execute the following steps:
Get all the files of your bucket, see the getFiles() method of the
Google Cloud Storage Node.js Client API. This method returns a
GetFilesResponse object which is an Array of File instances.
By looping over the array, for each file, check if the file has a
corresponding resized image in the bucket (depending on the way you
configured the Extension, the resized images may be in a specific
folder)
If a file does not have a corresponding resized image, execute the
same business logic of the Extension Cloud Function for this File.
There is an official Cloud Function sample which shows how to create a Cloud Storage triggered Firebase Function that will create resized thumbnails from uploaded images and upload them to the database URL, (see the last lines of index.js file)
Note : If you have a lot of files to treat, you should most probably work by batch, since there is a limit of 9 minutes for Cloud Function execution. Also, depending on the number of images to treat, you may need to increase the timeout value and/or the allocated memory of your Cloud Function, see https://firebase.google.com/docs/functions/manage-functions#set_timeout_and_memory_allocation
In case someone need it. This is how I resized existing image.
const functions = require("firebase-functions");
const axios = require("axios");
const { Storage } = require("#google-cloud/storage");
const storage = new Storage();
// Don't forget to replace with your bucket name
const bucket = storage.bucket("projectid.appspot.com");
async function getAlbums() {
const endpoint = "https://api.mydomain.com/graphql";
const headers = {
"content-type": "application/json",
};
const graphqlQuery = {
query: `query Albums {
albums {
id
album_cover
}
}`,
};
const response = await axios({
url: endpoint,
method: "post",
headers: headers,
data: graphqlQuery,
});
if (response.errors) {
functions.logger.error("API ERROR : ", response.errors); // errors
if any
} else {
return response.data.data.albums;
}
}
function getFileName(url) {
var decodeURI = decodeURIComponent(url);
var index = decodeURI.lastIndexOf("/") + 1;
var filenameWithParam = decodeURI.substr(index);
index = filenameWithParam.lastIndexOf("?");
var filename = filenameWithParam.substr(0, index);
return filename;
}
function getFileNameFromFirestore(url) {
var index = url.lastIndexOf("/") + 1;
var filename = url.substr(index);
return filename;
}
const triggerBucketEvent = async () => {
bucket.getFiles(
{
prefix: "images/albums", // you can add a path prefix
autoPaginate: false,
},
async (err, files) => {
if (err) {
functions.logger.error(err);
return;
}
const albums = await getAlbums();
await Promise.all(
files.map((file) => {
var fileName = getFileNameFromFirestore(file.name);
var result = albums.find((obj) => {
return getFileName(obj.album_cover) === fileName;
});
if (result) {
var file_ext = fileName.substr(
(Math.max(0, fileName.lastIndexOf(".")) || Infinity) + 1
);
var newFileName = result.id + "." + file_ext;
// Copy each file on thumbs directory with the different name
file.copy("images/albums/" + newFileName);
} else {
functions.logger.info(file.name, " not found in album list!");
}
})
);
}
);
};
exports.manualGenerateResizedImage = functions.https.onRequest(async () => {
await triggerBucketEvent();
});
I am using this approach to upload images to aws s3 bucket:
https://grokonez.com/aws/angular-4-amazon-s3-example-how-to-upload-file-to-s3-bucket
This works fine as an individual task but as far as I rely on the result which is coming a bit later due to async behavior may be. I would like the next task to be executed just after the confirmation.
upload() {
let file: any;
// let urltype = '';
let filename = '';
// let b: boolean;
for (let i = 0 ; i < this.urls.length ; i++) {
file = this.selectedFiles[i];
// urltype = this.urltype[i];
filename = file.name;
const k = uuid() + '.' + filename.substr((filename.lastIndexOf('.') + 1));
this.uploadservice.uploadfile(file, k);
console.log('done');
// console.log('file: ' + file + ' : ' + filename);
// let x = this.userservice.PostImage('test', file);
// console.log('value of ' + x);
}
// return b;
}
fileupload service:
bucket.upload(params, function (err, data) {
if (err) {
console.log('There was an error uploading your file: ', err);
return false;
}
console.log('Successfully uploaded file.', data);
return true;
}).promise();
}
Here, done is getting executed before the file upload is done.
I think you should check out a tutorial for asynchronous programming and try to play around with couple of examples using simple timeouts to get the hang of it and then proceed with more complex things like s3 and aws.
Here is how I suggest you start your journey:
1) Learn the basic concepts of asynchronous programming using pure JS
https://eloquentjavascript.net/11_async.html
2) Play around with your own examples using callbacks and timeouts
3) Replace the callbacks with Promises
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Using_promises
4) Do it the "angular" way with rxjs Observables (similar to JS Observable)
http://reactivex.io/rxjs/class/es6/Observable.js~Observable.html
PS: To be more concrete:
Your code fails because the following line is executed in an asynchronous manner. Thus the code will call your uploadfile function and will immedietly continue executing without waiting.
this.uploadservice.uploadfile(file, k);
Once you follow all the points I described above you will be able to do something like this (using a Promise):
this.uploadservice.uploadfile(file, k)
.then( result => {
console.log('Upload finished');
})
.catch(error => {
console.log('Something went wrong');
});
I tried to get a random Wikipedia page over their API via Google Cloud Functions. The Wikipedia API works fine. This is my request:
https://de.wikipedia.org/w/api.php?action=query&format=json&generator=random
For testing you can change the format to jsonfm in see the result in the browser. Click here 👍.
But it seems that my functions get destroyed even before the request was completely successfully. If I want to parse the data (or even if I want to log that data) I got a
SyntaxError: Unexpected end of json
The log look like (for example) that (no I haven't cut it by myself):
DATA: ue||"},"query":{"pages":{"2855038":{"pageid":2855038,"ns":0,"title":"Thomas Fischer
Of course, that is not a valid json and can't be parsed. Whatever this is my function:
exports.randomWikiPage = function getRandomWikiPage (req, res) {
const httpsOptions = {
host: "de.wikipedia.org",
path: "/w/api.php?action=query&format=json&generator=random"
};
const https = require('https');
https.request(httpsOptions, function(httpsRes) {
console.log('STATUS: ' + httpsRes.statusCode)
console.log('HEADERS: ' + JSON.stringify(httpsRes.headers))
httpsRes.setEncoding('utf8')
httpsRes.on('data', function (data) {
console.log("DATA: " + data)
const wikiResponse = JSON.parse(data);
const title = wikiResponse.query.title
res.status(200).json({"title": title})
});
}).end();
};
I've already tried to return something here. Like that video explained. But as I look into the node docs https.request don't return a Promise. So return that is wrong. I've also tried to extract the on('data', callback) into it's own function so that I can return the callback. But I haven't a success with that either.
How have to look my function that it return my expected:
{"title": "A random Wikipedia Page title"}
?
I believe your json comes through as a stream in chunks. You're attempting to parse the first data chunk that comes back. Try something like:
https.request(httpsOptions, function(httpsRes) {
console.log('STATUS: ' + httpsRes.statusCode)
console.log('HEADERS: ' + JSON.stringify(httpsRes.headers))
httpsRes.setEncoding('utf8')
let wikiResponseData = '';
httpsRes.on('data', function (data) {
wikiResponseData += data;
});
httpRes.on('end', function() {
const wikiResponse = JSON.parse(wikiResponseData)
const title = wikiResponse.query.title
res.status(200).json({"title": title})
})
}).end();
};
I am trying to create a skill for Amazon Echo that will call a JSON file from AWS S3. When I call the code from s3 basic get function it works. And the Amazon Alexa code works on its own.
But when I call them together the function gets skipped. So for the following code the console gets called before and after s3.getObject(). But the middle one gets skipped. I do not understand why.
I also checked whether s3 was being called, and it is.
let aws = require('aws-sdk');
let s3 = new aws.S3({ apiVersion: '2006-03-01'});
function callS3() {
console.log('loading S3 function');
var myData = [];
const params = {
Bucket: 'cvo-echo',
Key: 'data.json'
};
console.log("trying to get s3");
s3.getObject(params, (err, data) => {
if (err) {
console.log('error in s3 get: \n' + err);
//const message = `Error getting object ${key} from bucket ${bucket}.
// Make sure they exist and your bucket is in same region as this function.
//console.log(message);
} else {
console.log('CONTENT TYPE: ', data.ContentType);
console.log('Data body: \n' + data.Body.toString());
myData = JSON.parse(data.Body.toString());
console.log('myData.length = ' + myData.length);
}
console.log('myData >> ' + myData);
});
console.log('finished callS3() func');
return myData;
}
This might be a control flow issue, I've worked with amazons sdk before and was running into similar issues. Try implementing async within your code to have a better control of what happens when. This way methods won't skip.
UPDATE: adding some code examples of what you could do.
function callS3(callback) {
console.log('loading S3 function');
var myData = [];
const params = {
Bucket: 'cvo-echo',
Key: 'data.json'
};
console.log("trying to get s3");
s3.getObject(params, (err, data) => {
if (err) {
console.log('error in s3 get: \n' + err);
//const message = `Error getting object ${key} from bucket ${bucket}.
// Make sure they exist and your bucket is in same region as this function.
//console.log(message);
callback(err,null);//callback the error.
} else {
console.log('CONTENT TYPE: ', data.ContentType);
console.log('Data body: \n' + data.Body.toString());
myData = JSON.parse(data.Body.toString());
console.log('myData.length = ' + myData.length);
console.log('myData >> ' + myData);
console.log('finished callS3() func');
//Include the callback inside of the S3 call to make sure this function returns until the S3 call completes.
callback(null,myData); // first element is an error and second is your data, first element is null if no error ocurred.
}
});
}
/*
This MIGHT work without async but just in case you can read more about
async.waterfall where functions pass down values to the next function.
*/
async.waterfall([
callS3()//you can include more functions here, the callback from the last function will be available for the next.
//myNextFunction()
],function(err,myData){
//you can use myData here.
})
It's a timing issue. Here is an example of loading a JSON file from an S3 share when a session is started.
function onLaunch(launchRequest, session, callback) {
var sBucket = "your-bucket-name";
var sFile = "data.json";
var params = {Bucket: sBucket, Key: sFile};
var s3 = new AWS.S3();
var s3file = s3.getObject(params)
new AWS.S3().getObject(params, function(err, data) {
if (!err) {
var json = JSON.parse(new Buffer(data.Body).toString("utf8"));
for(var i = 0; i < json.length; i++) {
console.log("name:" + json[i].name + ", age:" + json[i].age);
}
getWelcomeResponse(callback);
} else {
console.log(err.toString());
}
});
}