Stripe API: how to export a csv report of Payments with python? - csv

I currently export data as CSV from the Payments-> Export page on the stripe website. I use this data to create some invoices. Is there a way to do this with the Stripe API? I'm interested in the fields: converted_amount, customer_email, card_name that I can find in the exported CSV, but i'm not able to find in the API. I tried the Charge API, but the amount is not converted in EUR. The best thing for me would be have an API the behaves like the export of the CSV as i do now, without entering the stripe website. Is it possible?

Asking the Stripe support they confirmed there is no report to obtain the data i was looking for.
I ended up using the Charge API and the BalanceTransaction API to get the converted amount in Euro. I share the code if anyone has the same needs
import stripe
import pandas as pd
from datetime import datetime
import numpy as np
stripe.api_key= "rk_test_pippo"
from decimal import Decimal
def stripe_get_data(resource, start_date=None, end_date=None, **kwargs):
if start_date:
# convert to unix timestamp
start_date = int(start_date.timestamp())
if end_date:
# convert to unix timestamp
end_date = int(end_date.timestamp())
resource_list = getattr(stripe, resource).list(limit=5, created={"gte": start_date,"lte": end_date}, **kwargs)
lst = []
for i in resource_list.auto_paging_iter():
lst.extend([i])
df = pd.DataFrame(lst)
if len(df) > 0:
df['created'] = pd.to_datetime(df['created'], unit='s')
return df
#extract Charge data
df= stripe_get_data('Charge',pd.to_datetime("2022-08-08 08:32:14"),pd.to_datetime("2022-09-20 08:32:14"))
#check if amount is EUR, if not, take the converted amount from the BalanceTransaction object
result=[]
for i, row in df.iterrows():
if df['currency'][i]=='eur':
result.append('{:.2f}'.format(df['amount'][i]/100))
else:
eur_amount_cents= getattr(stripe.BalanceTransaction.retrieve(df['balance_transaction'][i]), 'amount')
result.append('{:.2f}'.format(eur_amount_cents/100))
print('amount is ' +df['currency'][i]+' converted amount in '+ str(result[i]) +' eur')
df['converted_amount']=result
#convert amount to string because importing to google sheet does conversion
df['converted_amount'] = df['converted_amount'].apply(lambda x: str(x).replace('.',','))
df['customer_email']= df['receipt_email']
df['card_name']= df['billing_details'].apply(lambda x: x.get('name'))
df.to_csv('unified_payments.csv',encoding="utf8")

Yes Stripe has an equivalent Report API. See their Doc and API Reference. Those report results are pretty much the same csv you can download from Dashboard.

Related

unable to load csv from GCS bucket to BigQuery table accurately

I am trying to load the airbnb_nyc data set from GCS bucket to BigqueryTable. Link to the dataset.
I am using the following Code:
def parse_file(element):
for line in csv.reader([element],delimiter=','):
return line
class DataIngestion2:
def parse_method2(self, values):
row1 = dict(
zip(('id', 'name', 'host_id', 'host_name', 'neighbourhood_group', 'neighbourhood', 'latitude', 'longitude',
'room_type', 'price', 'minimum_nights', 'number_of_reviews', 'last_review', 'reviews_per_month',
'calculated_host_listings_count', 'availability_365'),
values))
return row1
with beam.Pipeline(options=pipeline_options) as p:
lines= p | 'Read' >> ReadFromText(known_args.input,skip_header_lines=1)\
| 'parse' >> beam.Map(parse_file)
pipeline2 = lines | 'Format to Dict _ original CSV' >> beam.Map(lambda x: data_ingestion2.parse_method2(x))
pipeline2 | 'Load2' >> beam.io.WriteToBigQuery(table_spec, schema=table_schema,
write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE,
create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED
)
`
But my output on BigQuery Table is wrong.
I am only getting values for the first two columns and the rest of the 14 columns are showing NULL. I am not able to figure out what I am doing wrong. Can Someone Help me find the error in my logic. I basically want to know how to transfer a csv from GCS bucket to BigQuery through DataFlow pipeline.
Thank you,
You can use the ReadFromText method and then create your own transform by extending beam.DoFn. Attached the code below for reference.
https://beam.apache.org/releases/pydoc/2.32.0/apache_beam.io.textio.html#apache_beam.io.textio.ReadFromText
Note that you can use gs:// for GCS in file_pattern.
More details about Pardo and DoFn
https://beam.apache.org/documentation/programming-guide/#pardo
import apache_beam as beam
from apache_beam.io.textio import ReadAllFromText,ReadFromText
from apache_beam.io.gcp.bigquery import WriteToBigQuery
from apache_beam.io.gcp.gcsio import GcsIO
import csv
COLUMN_NAMES = ['id','name','host_id','host_name','neighbourhood_group','neighbourhood','latitude','longitude','room_type','price','minimum_nights','number_of_reviews','last_review','reviews_per_month','calculated_host_listings_count','availability_365']
def files(path='gs:/some/path'):
return list(GcsIO(storage_client='<ur storage client>').list_prefix(path=path).keys())
def transform_csv(element):
rows = []
with open(element,newline='\r\n') as f:
itr = csv.reader(f, delimiter = ',',quotechar= '"')
skip_head = next(itr)
for row in itr:
rows.append(row)
return rows
def to_dict(element):
rows = []
for item in element:
row_dict = {}
zipped = zip(COLUMN_NAMES,item)
for key,val in zipped:
row_dict[key] =val
rows.append(row_dict)
yield rows
with beam.Pipeline() as p:
read =(
p
|'read-file'>> beam.Create(files())
|'transform-dict'>>beam.Map(transform_csv)
|'list-to-dict'>>beam.FlatMap(to_dict )
|'print'>>beam.Map(print)
#|'write-to-bq'>>WriteToBigQuery(schema=COLUMN_NAMES,table='ur table',project='',dataset='')
)
EDITED1 The ReadFromText supports \r\n as newline char.But,this fails to consider the condition where column data itself has \r\n. Updating the code below.
EDITED 2 GcsIo error fixed.
Note - I have used GCSIO for getting the list of files.
Details here
Please Up-vote and mark as answer if this helps.
Let me suggest another approch for this use case. BiqQuery offers special feature for uploading from Google Could Storage (GCS) to Bigquery. You can load data in several formats and CSV is among them.
There is nice tutorial on Google documentation explaining how to do it. You do not have to use Dataflow or apache_beam. Such process is available through BigQuery API itself.
This is working in many languages, but you do not have to use any language as such process can be done from console or via Cloud SDK using bq command. Everything can be found in mentioned tutorial.

JSONDecodeError: Expecting value: line 1 column 1 (char 0) while getting data from Pokemon API

I am trying to scrape the pokemon API and create a dataset for all pokemon. So I have written a function which looks like this:
import requests
import json
import pandas as pd
def poke_scrape(x, y):
'''
A function that takes in a range of pokemon (based on pokedex ID) and returns
a pandas dataframe with information related to the pokemon using the Poke API
'''
#GATERING THE DATA FROM API
url = 'https://pokeapi.co/api/v2/pokemon/'
ids = range(x, (y+1))
pkmn = []
for id_ in ids:
url = 'https://pokeapi.co/api/v2/pokemon/' + str(id_)
pages = requests.get(url).json()
# content = json.dumps(pages, indent = 4, sort_keys=True)
if 'error' not in pages:
pkmn.append([pages['id'], pages['name'], pages['abilities'], pages['stats'], pages['types']])
#MAKING A DATAFRAME FROM GATHERED API DATA
cols = ['id', 'name', 'abilities', 'stats', 'types']
df = pd.DataFrame(pkmn, columns=cols)
The code works fine for most pokemon. However, when I am trying to run poke_scrape(229, 229) (so trying to load ONLY the 229th pokemon), it gives me the JSONDecodeError. It looks like this:
So far I have tried using json.loads() instead but that has not solved the issue. What is even more perplexing is that specific pokemon has loaded before and the same issue was with another ID - otherwise I could just manually enter the stats for the specific pokemon that is unable to load into my dataframe. Any help is appreciated!
Because of the way the PokeAPI works, some links to the JSON data for each pokemon only load when the links end with a '/' (such as https://pokeapi.co/api/v2/pokemon/229/ vs https://pokeapi.co/api/v2/pokemon/229 - first link will work and the second will return not found). However, others will respond with a response error because of the added '/' so fixed the issue with a few if statements right after the for loop in the beginning of the function

Timestamp problem when fetching data from iforge and importing it to csv

Ok I am a python beginner who tries fetching data from iforge. However I get problem with timestamp when exporting to CSV. I think timestamp should look like this "2019-03-22 23:00:00" but instead I get 1553460483. Why is that and how to fix it so it becomes in correct format in the csv file?
# coding: utf-8
import json
import csv
import urllib.request
import datetime
data = json.load(request)
time = data[0]['timestamp']
price = data[0]['price']
data = json.load(request) contains this -
[{'symbol': 'EURUSD',
'bid': 1.2345,
'ask': 1.2399,
'price': 1.2343,
'timestamp': 1553460483}]
But since I was only interested in price and timestamp I did-
time = data[0]['timestamp']
price = data[0]['price']
myprice = {'Date':time,'price':price}
And then made csv from myprice....it works but I dont know if correct =)
Now to problem -
How to fix timestamp to show up correctly in CSV?
You would have to figure out what unit 'timestamp' is in. My guess would be seconds since a certain start date, so go for:
import pandas as pd
pd.to_datetime(1553460483, unit='s')
Out: Timestamp('2019-03-24 20:48:03')

How to use PySpark to load a rolling window from daily files?

I have a large number of fairly large daily files stored in a blog storage engine(S3, Azure datalake exc.. exc..) data1900-01-01.csv, data1900-01-02.csv,....,data2017-04-27.csv. My goal is to preform a rolling N-day linear regression but I am having trouble with the data loading aspect. I am not sure how to do this without nested RDD's.
The schema for every .csv file is the same.
In other words for every date d_t, I need data x_t and to join data (x_t-1, x_t-2,... x_t-N).
How can I use PySpark to load an N-day Window of these daily files? All of the PySpark examples I can find seem to load from one very large file or data set.
Here's an example of my current code:
dates = [('1995-01-03', '1995-01-04', '1995-01-05'), ('1995-01-04', '1995-01-05', '1995-01-06')]
p = sc.parallelize(dates)
def test_run(date_range):
dt0 = date_range[-1] #get the latest date
s = '/daily/data{}.csv'
df0 = spark.read.csv(s.format(dt0), header=True, mode='DROPMALFORM')
file_list = [s.format(dt) for dt in date_range[:-1]] # Get a window of trailing dates
df1 = spark.read.csv(file_list, header=True, mode='DROPMALFORM')
return 1
p.filter(test_run)
p.map(test_run) #fails with same error as p.filter
I'm on PySpark version '2.1.0'
I'm running this on an Azure HDInsight cluster jupyter notebook.
spark here is of type <class 'pyspark.sql.session.SparkSession'>
A smaller more reproducible example is as follows:
p = sc.parallelize([1, 2, 3])
def foo(date_range):
df = spark.createDataFrame([(1, 0, 3)], ["a", "b", "c"])
return 1
p.filter(foo).count()
You are better off with using Dataframes rather than RDD. Dataframe's read.csv api accepts list of paths like -
pathList = ['/path/to/data1900-01-01.csv','/path/to/data1900-01-02.csv']
df = spark.read.csv(pathList)
have a look at documentation for read.csv
You can form the list of paths to date files to your data files by doing some date operation over window of N days like "path/to/data"+datetime.today().strftime("%Y-%m-%d"))+.csv" (This will get you file name of today only but its not hard to figure out date calculation for N days)
However keep in mind that schema of all date csvs should be same for above to work.
edit : When you parallelize list of dates i.e. p, each date gets processed individually by different executors, so input to test_run2 wasnt really as list of dates, it was one individual string like 1995-01-01
Try this instead, see if this works.
# Get the list of dates
date_range = window(dates, N)
s = '/daily/data{}.csv'
dt0 = date_range[-1] # most recent file
df0 = spark.read.csv(s.format(dt0), header=True, mode='DROPMALFORM')
# read previous files
file_list = [s.format(dt) for dt in date_range[:-1]]
df1 = spark.read.csv(file_list, header=True, mode='DROPMALFORM')
r, resid = computeLinearRegression(df0,df1)
r.write.save('daily/r{}.csv'.format(dt0))
resid.write.save('/daily/resid{}.csv'.format(dt0))

What is "orcl" in the following pyalgotrade code? Further, i have a csv file and how can i upload data from it into my code?

I wanted to implement my own strategy for backtesting but am unable to modify the code according to my needs
from pyalgotrade.tools import yahoofinance
yahoofinance.download_daily_bars('orcl', 2000, 'orcl-2000.csv')
from pyalgotrade import strategy
from pyalgotrade.barfeed import yahoofeed
from pyalgotrade.technical import ma
#class to create objects
class MyStrategy(strategy.BacktestingStrategy):
def __init__(self, feed, instrument):
strategy.BacktestingStrategy.__init__(self, feed)
# We want a 15 period SMA over the closing prices.
self.__sma = ma.SMA(feed[instrument].getCloseDataSeries(), 15)
self.__instrument = instrument
def onBars(self, bars):
bar = bars[self.__instrument]
self.info("%s %s" % (bar.getClose(), self.__sma[-1]))
# Load the yahoo feed from the CSV file
feed = yahoofeed.Feed()
feed.addBarsFromCSV("orcl", "orcl-2000.csv")
# Evaluate the strategy with the feed's bars.
myStrategy = MyStrategy(feed, "orcl")
myStrategy.run()
Slightly modified from the documentation
documentation:
from pyalgotrade.tools import yahoofinance;
for instrument in ["AA","ACN"]:
for year in [2015, 2016]:
yahoofinance.download_daily_bars(instrument, year, r'D:\tmp\Trading\%s-%s.csv' % (instrument,year))
"orcl" is the name of the stock Oracle. If you want to use a different stock ticker, place it there.
You need to go to yahoo finance here: http://finance.yahoo.com/q/hp?s=ORCL&a=02&b=12&c=2000&d=05&e=26&f=2015&g=d
then save the file as orcl-2000.csv
This program reads in the orcl-2000.csv file from the directory and prints the prices.
If you want to download the data through python, then use a command like
instrument = "orcl"
feed = yahoofinance.build_feed([instrument], 2011, 2014, ".")
This will make files that say orcl-2011-yahoofinance.csv and so on from 2011 through 2014.