pandas parallel urllib request for Loop - json

Question

i have a code to parse JSON data through API using urllib as the following:
import pandas as pd
import json
import urllib.request
import os
import time
import csv
import datetime
# Send URL Request & Get JSON Data
with urllib.request.urlopen("https://bittrex.com/api/v1.1/public/getmarketsummaries") as url:
data = json.loads(url.read().decode())
# Select Data from result section
df = pd.DataFrame(data=data['result'])
# tickers = df['MarketName']
tickers= ["BTC-1ST", "BTC-2GIVE", "BTC-ABY", "BTC-ARDR", "BTC-WAVE"]
print(tickers)
for ticker in tickers:
with urllib.request.urlopen("https://bittrex.com/Api/v2.0/pub/market/GetTicks?marketName=" + ticker + "&tickInterval=thirtyMin") as URL:
data = json.loads(URL.read().decode())
df2 = pd.DataFrame(data=data['result'])
Market01 = "Market"
df2[Market01] = ticker
df2.to_csv('all.csv', encoding="utf-8-sig", index=False, mode='a', header=False)
print("done " + ticker)
actually it's not request for five currency only .. they are 295 request for 295 currency .. which take 5 minutes to complete all required data to csv file (very long time)
i wonder if there are a method to send all requests in parallel to decrease time with the same option to save data to csv file as dataframe
i searched many times and found multiprocessor module but couldn't found a sample similar to my case
any one can help me please!!!!!!

Answer 1

What about something like this?
import pandas as pd
import json
import urllib.request
import os
from urllib import parse
import csv
import datetime
from multiprocessing import Process, Pool
import time
# Send URL Request & Get JSON Data
with urllib.request.urlopen("https://bittrex.com/api/v1.1/public/getmarketsummaries") as url:
data = json.loads(url.read().decode())
# Select Data from result section
df = pd.DataFrame(data=data['result'])
# tickers = df['MarketName']
tickers= ["BTC-1ST", "BTC-2GIVE", "BTC-ABY", "BTC-ARDR", "BTC-WAVE"]
print(tickers)
def http_get(url):
result = {"url": url, "data": urllib.request.urlopen(url, timeout=5).read()}
return result
urls = [ "https://bittrex.com/Api/v2.0/pub/market/GetTicks?marketName=" + ticker + "&tickInterval=thirtyMin" for ticker in tickers ]
pool = Pool(processes=5)
results = pool.map(http_get, urls)
for result in results:
j = json.loads(result['data'].decode())
df2 = pd.DataFrame(data=j)
Market01 = "Market"
marketName = parse.parse_qs(parse.urlparse(result['url']).query)['marketName'][0]
df2[Market01] = marketName
df2.to_csv('all.csv', encoding="utf-8-sig", index=False, mode='a', header=False)
print("done " + marketName)