Extracting data out from javascript code from an webpage - json
I am trying to extract the data which will have the structure of the code i have pasted below and from this data i would like to extract the X and y for each 10v, 20V... and keep them separated. I am ultimately planning to generate a plot of each of them after the parsing for the list of the data i will have.
The Soup:
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-72511212-1', 'auto');
ga('send', 'pageview');
</script>
<!DOCTYPE html>
<html>
<head>
<meta content="text/html; charset=utf-8" http-equiv="Content-Type"/>
<title>Metabolite Chart</title>
<!-- 1. Add these JavaScript inclusions in the head of your page -->
<script src="lib/js/jquery-1.6.x/jquery-1.6.1.min.js" type="text/javascript"></script>
<script src="lib/js/misc/highcharts.src.delta.js" type="text/javascript"></script>
<!--<script type="text/javascript" src="/lib/highcharts.js"></script>-->
<script src="lib/js/misc/excanvas.compiled.js" type="text/javascript"></script>
<!--
<script src="https://code.highcharts.com/highcharts.js"></script>
<script src="https://code.highcharts.com/modules/exporting.js"></script>
-->
<!-- 2. Add the JavaScript to initialize the chart on document ready -->
<script type="text/javascript">
//alert("molI: " + "203");
//alert("molN: " + "Chenodeoxycholic acid glycine conjugate");
$(document).ready(function() {
//alert("molN: " + "Chenodeoxycholic acid glycine conjugate");
// function resetchart() {
// fireEvent(chart, 'selection', { resetSelection: true }, zoom);
// }
var count = 0;
//alert("molI: " + "203");
//var mid = 203; // Pass MID here!
var mid = "203"; // Pass
var mole = "Chenodeoxycholic acid glycine conjugate"; // Pass molecule name here!
var chart = new Highcharts.Chart({
chart: {
renderTo: 'container',
defaultSeriesType: 'column',
zoomType: 'xy',
margin: [50, 50, 200, 80]
},
title: {
text: '' + mole
},
subtitle: {
text: "MID: 203 <font color='blue'><b>Insilico predicted spectra<\/b><\/font>" },
credits: {
enabled: false
},
xAxis: {
min: 0,
// max: 200,
title: {
enabled: true,
text: 'Mass (m/z)'
},
maxZoom: 0.1,
tickPixelInterval: 100
},
yAxis: {
min: 0,
max: 100,
title: {
text: 'Intensity (%)'
}
},
legend: {
enabled: true,
showFragments: true,
showNeutrals: false,
showPeaks: false,
exclusiveSelect: true, // Turns on exclusive radio style buttons
dblClick: false,
startNumber: 0, // The default legend item when page loads
borderWidth: 1,
layout: 'vertical',
backgroundColor: '#FFFFFF',
style: {
left: '50px',
top: '300px',
bottom: 'auto'
}
},
// Tooltip HTML
tooltip: {
second: true,
neutral: false,
borderRadius: 0,
formatter: function() {
var namestr;
if (this.series.name.match(/\+/g) && !this.series.name.match("Cl"))
namestr = "Mode: <b><font size=\"4\">(+)</font></b> Collision Energy: ";
else if (this.series.name.match('-'))
namestr = "Mode: <b><font size=\"4\">(-)</font></b> Collision Energy: ";
if (!(this.series.name.match("10 V")||this.series.name.match("20 V")||this.series.name.match("40 V")))
namestr += "<b><font size=\"3\">0 V</font></b>";
else if (this.series.name.match("10 V"))
namestr += "<b><font size=\"3\">10 V</font></b>";
else if (this.series.name.match("20 V"))
namestr += "<b><font size=\"3\">20 V</font></b>";
else if (this.series.name.match("40 V"))
namestr += "<b><font size=\"3\">40 V</font></b>";
return '<center><br/> '+ namestr +'<br/>' + ' m/z: <b><font size="3">' + this.x.toFixed(4) + '</font></b> Intensity: <b><font size="3">' + parseInt(Math.abs(this.y)) + ' % </font></b></center><br/>';
},
formatter2: function() {
var namestr;
if (this.series.name.match(/\+/g))
namestr = "Mode: (+), Collision Energy: ";
else if (this.series.name.match('-'))
namestr = "Mode: (-), Collision Energy: ";
if (!(this.series.name.match("10 V")||this.series.name.match("20 V")||this.series.name.match("40 V")))
namestr += "0 V, Adduct: ";
else if (this.series.name.match("10 V"))
namestr += "10 V, Adduct: ";
else if(this.series.name.match("20 V"))
namestr += "20 V, Adduct: ";
else if (this.series.name.match("40 V"))
namestr += "40 V, Adduct: ";
return false;
}
},
plotOptions: {
column: {
pointPadding: 0.53,
// pointPadding: 0.99,
borderWidth: 0,
shadow: false
// borderColor: '#000000'
}
},
series: [{name: ' (+) 10 V [M+H]+ ',data:[{x:450.321,y:84,fragment: false},{x:434.29,y:0,fragment: false},{x:432.311,y:8,fragment: false},{x:432.311,y:100,fragment: false},{x:416.28,y:0,fragment: false},{x:414.3,y:24,fragment: false},{x:406.332,y:0,fragment: false},{x:404.316,y:16,fragment: false},{x:390.3,y:0,fragment: false},{x:390.264,y:0,fragment: false},{x:388.321,y:0,fragment: false},{x:386.305,y:8,fragment: false},{x:375.289,y:24,fragment: false},{x:372.29,y:0,fragment: false},{x:357.279,y:12,fragment: false},{x:347.294,y:4,fragment: false},{x:331.3,y:0,fragment: false},{x:329.284,y:4,fragment: false},{x:319.263,y:0,fragment: false},{x:301.253,y:0,fragment: false},{x:291.232,y:0,fragment: false},{x:273.221,y:0,fragment: false},{x:240.159,y:0,fragment: false},{x:158.081,y:0,fragment: false},{x:130.05,y:0,fragment: false},{x:76.0393,y:44,fragment: false},{x:74.0237,y:0,fragment: false},{x:59.0128,y:8,fragment: false},{x:58.0287,y:0,fragment: false},{x:30.0338,y:0,fragment: false},{x:28.0182,y:0,fragment: false} ]},{name: ' (+) 20 V [M+H]+ ',data:[{x:450.321,y:11.764705882353,fragment: false},{x:434.29,y:0,fragment: false},{x:432.311,y:35.294117647059,fragment: false},{x:432.311,y:5.8823529411765,fragment: false},{x:416.28,y:5.8823529411765,fragment: false},{x:414.3,y:58.823529411765,fragment: false},{x:404.316,y:11.764705882353,fragment: false},{x:388.321,y:5.8823529411765,fragment: false},{x:386.305,y:35.294117647059,fragment: false},{x:375.289,y:29.411764705882,fragment: false},{x:372.29,y:5.8823529411765,fragment: false},{x:357.279,y:35.294117647059,fragment: false},{x:347.294,y:17.647058823529,fragment: false},{x:333.279,y:5.8823529411765,fragment: false},{x:331.3,y:5.8823529411765,fragment: false},{x:329.284,y:17.647058823529,fragment: false},{x:327.268,y:5.8823529411765,fragment: false},{x:319.263,y:0,fragment: false},{x:315.268,y:5.8823529411765,fragment: false},{x:301.253,y:5.8823529411765,fragment: false},{x:273.221,y:5.8823529411765,fragment: false},{x:158.081,y:5.8823529411765,fragment: false},{x:130.05,y:5.8823529411765,fragment: false},{x:111.08,y:5.8823529411765,fragment: false},{x:102.019,y:0,fragment: false},{x:76.0393,y:100,fragment: false},{x:74.0237,y:5.8823529411765,fragment: false},{x:59.0128,y:23.529411764706,fragment: false},{x:58.0287,y:17.647058823529,fragment: false},{x:30.0338,y:11.764705882353,fragment: false},{x:28.0182,y:11.764705882353,fragment: false} ]},{name: ' (+) 40 V [M+H]+ ',data:[{x:416.28,y:18.181818181818,fragment: false},{x:414.3,y:100,fragment: false},{x:388.321,y:18.181818181818,fragment: false},{x:386.305,y:54.545454545455,fragment: false},{x:372.29,y:9.0909090909091,fragment: false},{x:359.294,y:9.0909090909091,fragment: false},{x:357.279,y:63.636363636364,fragment: false},{x:355.263,y:9.0909090909091,fragment: false},{x:333.279,y:0,fragment: false},{x:331.3,y:18.181818181818,fragment: false},{x:331.263,y:9.0909090909091,fragment: false},{x:329.284,y:36.363636363636,fragment: false},{x:327.268,y:9.0909090909091,fragment: false},{x:317.284,y:9.0909090909091,fragment: false},{x:315.268,y:18.181818181818,fragment: false},{x:303.268,y:9.0909090909091,fragment: false},{x:301.253,y:18.181818181818,fragment: false},{x:275.237,y:9.0909090909091,fragment: false},{x:273.221,y:18.181818181818,fragment: false},{x:261.221,y:0,fragment: false},{x:111.08,y:9.0909090909091,fragment: false},{x:97.0648,y:9.0909090909091,fragment: false},{x:76.0393,y:45.454545454545,fragment: false},{x:59.0128,y:63.636363636364,fragment: false},{x:58.0287,y:81.818181818182,fragment: false},{x:55.0542,y:9.0909090909091,fragment: false},{x:44.9971,y:9.0909090909091,fragment: false},{x:41.0022,y:18.181818181818,fragment: false},{x:32.0495,y:9.0909090909091,fragment: false},{x:30.0338,y:36.363636363636,fragment: false},{x:28.0182,y:54.545454545455,fragment: false} ]}]
});
});
</script>
</head>
<body style="border:0;overflow:visible">
<!-- 3. Add the container -->
<div id="container" style="width: 720px; height: 460px; margin: 0 auto">
</div>
<!-- <table align = "center" style="border-width:0; cellpadding:5; table-layout:fixed; bordercolor:'#00FF00'"> -->
<table align="center" style="border-width:0; cellpadding:5; table-layout:fixed; bordercolor:'#00FF00'">
<tr>
<td style="border-style: solid; border-color:#FFF8C6"><img align="top" alt="attention" src="img/attn.png" title="how to use spectrum"/>
<font color="red" face="helvetica,arial" size="2">
<b>Please mouse over the spectrum to view the detail information of each peak<br/>
Use left mouse button to zoom in (click and drag) and zoom out (double-click)</b></font>
</td>
</tr>
</table>
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-1907670-5']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();
</script>
</body>
</html>
The data i am trying to extract out:
series: [{name: ' (+) 10 V [M+H]+ ',data:[{x:450.321,y:84,fragment: false},{x:434.29,y:0,fragment: false},{x:432.311,y:8,fragment: false},{x:432.311,y:100,fragment: false},{x:416.28,y:0,fragment: false},{x:414.3,y:24,fragment: false},{x:406.332,y:0,fragment: false},{x:404.316,y:16,fragment: false},{x:390.3,y:0,fragment: false},{x:390.264,y:0,fragment: false},{x:388.321,y:0,fragment: false},{x:386.305,y:8,fragment: false},{x:375.289,y:24,fragment: false},{x:372.29,y:0,fragment: false},{x:357.279,y:12,fragment: false},{x:347.294,y:4,fragment: false},{x:331.3,y:0,fragment: false},{x:329.284,y:4,fragment: false},{x:319.263,y:0,fragment: false},{x:301.253,y:0,fragment: false},{x:291.232,y:0,fragment: false},{x:273.221,y:0,fragment: false},{x:240.159,y:0,fragment: false},{x:158.081,y:0,fragment: false},{x:130.05,y:0,fragment: false},{x:76.0393,y:44,fragment: false},{x:74.0237,y:0,fragment: false},{x:59.0128,y:8,fragment: false},{x:58.0287,y:0,fragment: false},{x:30.0338,y:0,fragment: false},{x:28.0182,y:0,fragment: false} ]},{name: ' (+) 20 V [M+H]+ ',data:[{x:450.321,y:11.764705882353,fragment: false},{x:434.29,y:0,fragment: false},{x:432.311,y:35.294117647059,fragment: false},{x:432.311,y:5.8823529411765,fragment: false},{x:416.28,y:5.8823529411765,fragment: false},{x:414.3,y:58.823529411765,fragment: false},{x:404.316,y:11.764705882353,fragment: false},{x:388.321,y:5.8823529411765,fragment: false},{x:386.305,y:35.294117647059,fragment: false},{x:375.289,y:29.411764705882,fragment: false},{x:372.29,y:5.8823529411765,fragment: false},{x:357.279,y:35.294117647059,fragment: false},{x:347.294,y:17.647058823529,fragment: false},{x:333.279,y:5.8823529411765,fragment: false},{x:331.3,y:5.8823529411765,fragment: false},{x:329.284,y:17.647058823529,fragment: false},{x:327.268,y:5.8823529411765,fragment: false},{x:319.263,y:0,fragment: false},{x:315.268,y:5.8823529411765,fragment: false},{x:301.253,y:5.8823529411765,fragment: false},{x:273.221,y:5.8823529411765,fragment: false},{x:158.081,y:5.8823529411765,fragment: false},{x:130.05,y:5.8823529411765,fragment: false},{x:111.08,y:5.8823529411765,fragment: false},{x:102.019,y:0,fragment: false},{x:76.0393,y:100,fragment: false},{x:74.0237,y:5.8823529411765,fragment: false},{x:59.0128,y:23.529411764706,fragment: false},{x:58.0287,y:17.647058823529,fragment: false},{x:30.0338,y:11.764705882353,fragment: false},{x:28.0182,y:11.764705882353,fragment: false} ]},{name: ' (+) 40 V [M+H]+ ',data:[{x:416.28,y:18.181818181818,fragment: false},{x:414.3,y:100,fragment: false},{x:388.321,y:18.181818181818,fragment: false},{x:386.305,y:54.545454545455,fragment: false},{x:372.29,y:9.0909090909091,fragment: false},{x:359.294,y:9.0909090909091,fragment: false},{x:357.279,y:63.636363636364,fragment: false},{x:355.263,y:9.0909090909091,fragment: false},{x:333.279,y:0,fragment: false},{x:331.3,y:18.181818181818,fragment: false},{x:331.263,y:9.0909090909091,fragment: false},{x:329.284,y:36.363636363636,fragment: false},{x:327.268,y:9.0909090909091,fragment: false},{x:317.284,y:9.0909090909091,fragment: false},{x:315.268,y:18.181818181818,fragment: false},{x:303.268,y:9.0909090909091,fragment: false},{x:301.253,y:18.181818181818,fragment: false},{x:275.237,y:9.0909090909091,fragment: false},{x:273.221,y:18.181818181818,fragment: false},{x:261.221,y:0,fragment: false},{x:111.08,y:9.0909090909091,fragment: false},{x:97.0648,y:9.0909090909091,fragment: false},{x:76.0393,y:45.454545454545,fragment: false},{x:59.0128,y:63.636363636364,fragment: false},{x:58.0287,y:81.818181818182,fragment: false},{x:55.0542,y:9.0909090909091,fragment: false},{x:44.9971,y:9.0909090909091,fragment: false},{x:41.0022,y:18.181818181818,fragment: false},{x:32.0495,y:9.0909090909091,fragment: false},{x:30.0338,y:36.363636363636,fragment: false},{x:28.0182,y:54.545454545455,fragment: false} ]}]
i am having trouble getting to that segment of the data and extracting it in a way that is useful. I did notice it has a JSON format structure but i cant gain access to it from the soup to make use of that.
If i am not clear on what i am trying to do please let me know.
Below is my python script:
from bs4 import BeautifulSoup
import urllib
import urllib.request
import xlwt
import xlrd
import requests
import re
import json
CASNUMBERS =xlrd.open_workbook("./OUTPUTFILE.xls")
CASNUMBERS_sheet = CASNUMBERS.sheet_by_index(0)
# import sqlite3
# conn = sqlite3.connect('CurationParsedData.db')
# c = conn.cursor()
#
# ##Create Table
# c.execute('''CREATE TABLE CurationParsedData(Exp_website TEXT,InSilico_website TEXT)''')
#
#
# def add_website(exp,insil):
# c.execute("INSERT INTO CurationParsedData VALUES("+ exp ","+ insil")")
# MID = "154"
# NAMEID = "glucose"
# CASID = "492-62-6"
# KEGGID = "C00267"
def make_soup(url):
thepage = urllib.request.urlopen(url)
soupdata = BeautifulSoup(thepage, "html.parser")
return soupdata
def ESILINK(number):
ESIlink = "https://metlin.scripps.edu/showChart.php?molid=" + number + "&h=240&collE=&Imode=p&etype=experimental"
return ESIlink
def insilico(number):
insilicolink = "https://metlin.scripps.edu/showChart.php?molid=" + number + "&h=240&collE=&Imode=p&etype=insilico"
return insilicolink
def metlinsearch(NAME= "",CAS = "", KEGG=""):
metlin_search = "https://metlin.scripps.edu/advanced_search_result.php?molid=&mass_min=&mass_max=&name=" + NAME + "&formula=&cas=" + CAS + "&kegg=" + KEGG + "&smilefile=&msmspeaks_min=&AminoAcid=add&drug=add&toxinEPA=add&smilesExactMatchCheckBox=false&nameExactMatchCheckBox=false"
return metlin_search
def HMDBsearch(number):
hmdb_search = "http://www.hmdb.ca/unearth/q?utf8=%E2%9C%93&query=" + number + "&searcher=metabolites&button="
return hmdb_search
for items in CASNUMBERS_sheet.col_values(2, 1):
print(items)
hmdbsearch_link = HMDBsearch(items)
print(hmdbsearch_link)
metlinsearch_link = metlinsearch(CAS = items)
metlinesearch_soup = make_soup(metlinsearch_link)
firstMID = metlinesearch_soup.find("th", {"scope": "row"})
allMID = metlinesearch_soup.findAll("th", {"scope": "row"})
ESI = "NO LINK"
INSILICO = "NO LINK"
if firstMID != None:
firstMID = firstMID.text
ESI = ESILINK(firstMID)
INSILICO = insilico(firstMID)
if allMID != None:
MIDlist = []
for items in allMID:
MIDlist.append(items.text)
esi = ESILINK(ESI)
sil = insilico(INSILICO)
print(ESI)
print(INSILICO)
# sil_soup = make_soup(sil)
sil_link = requests.get(INSILICO)
sil_soup = BeautifulSoup(sil_link.text, "lxml")
# print(sil_soup)
series = sil_soup.findAll('script', {"type": "text/javascript"})
series = series[3]
info = []
for x in series:
info.append(str(x))
for text in info:
head, body, tail = text.partition('series:')
tail = tail.replace(' ', '').replace(';', '').replace(' ', '')
print(tail)
json_string = tail
parse_json= json.loads(json_string)
print(parse_json['data'])
I was able to get the output i wanted by a few steps, definitely not the best and optimal way but i first turned it in to a string -> partition to get to the part i needed -> removed characters that were not of interest from the string -> run it through a for loop to reorganize the data back into a list -> run through and partition for x and y values -> convert to float -> values can be used for graphing.
very round about way to get the task done but might help someone with similar issues when trying to parse JavaScript data.
below is the code:
from bs4 import BeautifulSoup
import urllib
import urllib.request
import xlwt
import xlrd
import requests
import re
import json
import matplotlib.pyplot as plt
CASNUMBERS =xlrd.open_workbook("./OUTPUTFILE.xls")
CASNUMBERS_sheet = CASNUMBERS.sheet_by_index(0)
# import sqlite3
# conn = sqlite3.connect('CurationParsedData.db')
# c = conn.cursor()
#
# ##Create Table
# c.execute('''CREATE TABLE CurationParsedData(Exp_website TEXT,InSilico_website TEXT)''')
#
#
# def add_website(exp,insil):
# c.execute("INSERT INTO CurationParsedData VALUES("+ exp ","+ insil")")
def make_soup(url):
thepage = urllib.request.urlopen(url)
soupdata = BeautifulSoup(thepage, "html.parser")
return soupdata
def ESILINK(number):
ESIlink = "https://metlin.scripps.edu/showChart.php?molid=" + number + "&h=240&collE=&Imode=p&etype=experimental"
return ESIlink
def insilico(number):
insilicolink = "https://metlin.scripps.edu/showChart.php?molid=" + number + "&h=240&collE=&Imode=p&etype=insilico"
return insilicolink
def metlinsearch(NAME= "",CAS = "", KEGG=""):
metlin_search = "https://metlin.scripps.edu/advanced_search_result.php?molid=&mass_min=&mass_max=&name=" + NAME + "&formula=&cas=" + CAS + "&kegg=" + KEGG + "&smilefile=&msmspeaks_min=&AminoAcid=add&drug=add&toxinEPA=add&smilesExactMatchCheckBox=false&nameExactMatchCheckBox=false"
return metlin_search
def HMDBsearch(number):
hmdb_search = "http://www.hmdb.ca/unearth/q?utf8=%E2%9C%93&query=" + number + "&searcher=metabolites&button="
return hmdb_search
for items in CASNUMBERS_sheet.col_values(2, 1):
print(items)
hmdbsearch_link = HMDBsearch(items)
print(hmdbsearch_link)
metlinsearch_link = metlinsearch(CAS = items)
metlinesearch_soup = make_soup(metlinsearch_link)
firstMID = metlinesearch_soup.find("th", {"scope": "row"})
allMID = metlinesearch_soup.findAll("th", {"scope": "row"})
ESI = "NO LINK"
INSILICO = "NO LINK"
if firstMID != None:
firstMID = firstMID.text
ESI = ESILINK(firstMID)
INSILICO = insilico(firstMID)
if allMID != None:
MIDlist = []
for items in allMID:
MIDlist.append(items.text)
esi = ESILINK(ESI)
sil = insilico(INSILICO)
print(ESI)
print(INSILICO)
if ESI != "NO LINK":
sil_link = requests.get(ESI)
sil_soup = BeautifulSoup(sil_link.text, "lxml")
# print(sil_soup)
series = sil_soup.findAll('script', {"type": "text/javascript"})
series = series[3]
info = []
for x in series:
info.append(str(x))
for text in info:
head, body, tail = text.partition('series:')
tail = tail.replace(' ', '').replace(';', '').replace(' ', '').replace('fragment:false', '').replace('fragment:true', '').replace('\n', '').replace('[', '').replace(']', '').replace('name:', '').replace('data:', '')
# print(tail)
identifyers = ['{', '}']
datalist = []
temp = ''
for data in tail:
if data != identifyers:
temp = temp + data
# print(temp)
# print('active1')
if data in identifyers:
datalist.append(temp)
temp = ''
# print("active2")
# print(datalist)
finallist = []
for items in datalist:
items = items.replace("}", '').replace('{', '').replace(',', '').replace(')', '').replace('(','')
if items != '':
finallist.append(items)
print(finallist)
for items in finallist:
if items[0] == "x":
head,body,tail = items.partition("x:")
head,body,tail = tail.partition("y:")
xvalue = round(float(head), 3)
yvalue = round(float(tail), 3)
print("x:",xvalue, "y:", yvalue)
else:
# items[1] == "-" or "+":
print("energy", items)
Related
Saving/exporting SunburstR (package) plot
I am looking for help with saving/exporting this HTML widget so that I share it with others. The following is my code:
sun <- sunburst(df,
percent=TRUE,count=TRUE, legend = list(w=150), withD3 = T, width = "100%", height = 400)
htmlwidgets::onRender(
sun,
"
function(el, x) {
d3.selectAll('.sunburst-legend text').attr('font-size', '10px');
d3.select(el).select('.sunburst-togglelegend').property('checked',true); // force show the legend, check legend
d3.select(el).select('.sunburst-togglelegend').on('click')(); // simulate click
d3.select(el).select('.sunburst-togglelegend').remove() // remove the legend toggle
}
")
Example dataset:
df <- read.csv(system.file("examples/visit-sequences.csv",package="sunburstR"),header = FALSE,stringsAsFactors = FALSE)[1:100,]
Thank you.
With the following code, you can save your graph to HTML and PDF :
library(rmarkdown)
library(pagedown)
vector_Text_RMD <- c('---',
'title: ""',
'output: html_document',
'---',
'```{r setup, include=FALSE}',
'knitr::opts_chunk$set(echo = TRUE)',
'```',
'```{r cars, echo=FALSE}',
'library(sunburstR)',
'df <- read.csv(system.file("examples/visit-sequences.csv",package="sunburstR"),header = FALSE,stringsAsFactors = FALSE)[1:100,]',
'sun <- sunburst(df, percent = TRUE, count = TRUE, legend = list(w = 150), withD3 = T, width = "100%", height = 400)',
'htmlwidgets::onRender(sun,',
" 'function(el, x) {",
' d3.selectAll(".sunburst-legend text").attr("font-size", "10px");',
' d3.select(el).select(".sunburst-togglelegend").property("checked",true); // force show the legend, check legend',
' d3.select(el).select(".sunburst-togglelegend").on("click")(); // simulate click',
' d3.select(el).select(".sunburst-togglelegend").remove() // remove the legend toggle',
' }',
" ')",
'```')
zzfil <- tempfile(fileext = ".Rmd")
writeLines(text = vector_Text_RMD, con = zzfil)
render(input = zzfil,
output_file = "C:/stackoverflow100.html")
chrome_print("C:/stackoverflow100.html",
output = "C:/testpdf100.pdf")
how to read the output of a function as an object
Here is the output of my function sam() in HTML: ["2020-02-14", 324.7300, 325.9800, 322.8500, 324.9500, 20028447], ["2020-02-13", 324.1900, 326.2200, 323.3500, 324.8700, 23686892], ["2020-02-12", 321.4700, 327.2200, 321.4700, 327.2000, 28432573], ["2020-02-11", 323.6000, 323.9000, 318.7100, 319.6100, 23580780], ["2020-02-10", 314.1800, 321.5500, 313.8500, 321.5500, 27337215], I am trying to get that very output of sam() to be read as an object in this line of code below. "data": [ sam() ], At the moment sam() in that location keeps outputting that above sequence of dates and price numbers. I need for sam() in that location to be read so that the above sequence would be integrated into the stock chart that I am working on. If I add all of that code as a non-string into the "data"[ ], it will print out the stock chart with the dates and prices, but when I simply use the function sam() in there, it doesn't print out the chart. It only does dates and numbers
Here is the entire code that uses the sam() function. When you execute it, it will output a sequence of dates and numbers. When you copy and paste that output into the line of code near the bottom labeled "data" : [ ], it will output a nice looking stock chart using the dates and numbers. I need a function that can do the same.
<html>
<head>
<script data-require="jquery#2.1.4" data-semver="2.1.4" src="http://code.jquery.com/jquery-2.1.4.min.js"></script>
<script data-require="highstock#0.0.1" data-semver="0.0.1" src="http://code.highcharts.com/stock/highstock.js"></script>
<script data-require="highstock#0.0.1" data-semver="0.0.1" src="http://code.highcharts.com/stock/modules/exporting.js"></script>
<title>
How to get the array of dates between
two dates in JavaScript ?
</title>
</head>
<body style = "text-align:center;">
<div id="myData"></div>
<div id="demo"></div>
<div id="demo1"></div>
<script>var tday=["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"];
var tmonth=["-01-","-02-","-03-","-04-","-05-","-06-","-07-","-08-","-09-","-10-","-11-","-12-"];
function GetClock(){
var d=new Date();
var nday=d.getDay(),nmonth=d.getMonth(),ndate=d.getDate(),nyear=d.getFullYear();
var nhour=d.getHours(),nmin=d.getMinutes(),nsec=d.getSeconds(),ap;
if(nhour==0){ap=" AM";nhour=12;}
else if(nhour<12){ap=" AM";}
else if(nhour==12){ap=" PM";}
else if(nhour>12){ap=" PM";nhour-=12;}
if(nmin<=9) nmin="0"+nmin;
if(nsec<=9) nsec="0"+nsec;
var clocktext=""+nyear +tmonth[nmonth] +ndate +"";
document.getElementById('clockbox').innerHTML= clocktext;
}
GetClock();
setInterval(GetClock,1000);</script>
<script>
function heavy(){
var today = new Date("2014-10-01"); //YYYY-MM-DD
var startDate = new Date("2014-10-01");
var endDate = new Date("2016-10-10" );
//YYYY-MM-DD
var getDateArray = function(start, end) {
var arr = new Array();
while (startDate < end) {
arr.push(new Date(startDate));
startDate.setDate(startDate.getDate() + 1);
}
return arr.reverse().toLocaleString().replace(new RegExp('/', 'g'), '-').replace(new RegExp("8:", 'g'), '').replace(new RegExp("00", 'g'), '').replace(new RegExp("PM", 'g'), '').replace(new RegExp("AM", 'g'), '').replace(new RegExp(",", 'g'), '').replace(new RegExp(":", 'g'), '');
}
var dateArr = getDateArray(startDate, endDate);
// Output
for (var i = 0; i < dateArr.length; i++) {
document.write( dateArr[i] );
}
}
</script>
<script>
function heavy1(){
var cars = [["2020-02-13"], ["2020-02-12"]];
var text = "";
var i;
for (i = 0; i < cars.length; i++) {
text += cars[i] + "<br>";
}
document.getElementById("demo").innerHTML = text;}
</script>
<script>
function sam(){
fetch('https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=MSFT&outputsize=full&apikey=demo')
.then(function (response) {
return response.json();
})
.then(function (data) {
appendData(data);
})
.catch(function (err) {
console.log('error: ' + err);
});
function appendData(data) {
var mainContainer = document.getElementById("myData");
var arrw = ["2020-02-14",
"2020-02-13",
"2020-02-12",
"2020-02-11",
"2020-02-10",
"2020-02-07",
"2020-02-06",
"2020-02-05",
"2020-02-04",
"2020-02-03",
"2020-01-31",
"2020-01-30",
"2020-01-29",
"2020-01-28",
"2020-01-27",
"2020-01-24",
"2020-01-23",
"2020-01-22",
"2020-01-21",];
for (var i = 0; i < arrw.length; i++) {
var joe = '[' +"\""+ arrw[i]+ "\"" + "," + " " + data['Time Series (Daily)'][arrw[i]]['1. open']+"," + " " +data['Time Series (Daily)'][arrw[i]]['2. high']+"," + " " +data['Time Series (Daily)'][arrw[i]]['3. low']+","+ " " +data['Time Series (Daily)'][arrw[i]]['4. close']+","+ " " +data['Time Series (Daily)'][arrw[i]]['5. volume' ] + ']' + "," ;
document.write(joe);
}}}
</script>
<div id="container"></div>
<script>
var json = {
"dataset": {
"data": [
sam()
],
}
};
var hiJson = json.dataset.data.map(function(d) {
return [new Date(d[0]).getTime(), d[4]]
});
// Create the chart
$('#container').highcharts('StockChart', {
rangeSelector: {
selected: 1
},
title: {
text: 'MS Stock Price'
},
series: [{
name: 'MS',
data: hiJson,
tooltip: {
valueDecimals: 2
}
}]
});
</script>
</body>
</html>
find all label and product in json python
I'm trying to find all label and product in text/javascript, but I don't have any idea to how do it. I'm trying to parse label, id and products.
var spConfigDisabledProducts = [-1
, '290058', '290060', '290061', '290062', '290063', '290065', '290071', '290073', '290075', '290076', '290077', '290078' ];
var spConfig = new Product.Config({"attributes":{"959":{"id":"959","code":"aw_taglia","label":"Taglia","options":[{"id":"730","label":"36 ½","price":"0","oldPrice":"0","products":["290058"]},{"id":"731","label":"37 ½","price":"0","oldPrice":"0","products":["290060"]},{"id":"732","label":"38","price":"0","oldPrice":"0","products":["290061"]},{"id":"733","label":"38 ½","price":"0","oldPrice":"0","products":["290062"]},{"id":"734","label":"39","price":"0","oldPrice":"0","products":["290063"]},{"id":"735","label":"40","price":"0","oldPrice":"0","products":["290064"]},{"id":"736","label":"40 ½","price":"0","oldPrice":"0","products":["290065"]},{"id":"737","label":"41","price":"0","oldPrice":"0","products":["290066"]},{"id":"738","label":"42","price":"0","oldPrice":"0","products":["290067"]},{"id":"739","label":"42 ½","price":"0","oldPrice":"0","products":["290068"]},{"id":"740","label":"43","price":"0","oldPrice":"0","products":["290069"]},{"id":"741","label":"44","price":"0","oldPrice":"0","products":["290070"]},{"id":"742","label":"44 ½","price":"0","oldPrice":"0","products":["290071"]},{"id":"743","label":"45","price":"0","oldPrice":"0","products":["290072"]},{"id":"744","label":"45 ½","price":"0","oldPrice":"0","products":["290073"]},{"id":"745","label":"46","price":"0","oldPrice":"0","products":["290074"]},{"id":"746","label":"47","price":"0","oldPrice":"0","products":["290075"]},{"id":"747","label":"47 ½","price":"0","oldPrice":"0","products":["290076"]},{"id":"748","label":"13.5","price":"0","oldPrice":"0","products":["290077"]},{"id":"749","label":"48 ½","price":"0","oldPrice":"0","products":["290078"]}]}},"template":"#{price}\u00a0\u20ac","basePrice":"130","oldPrice":"130","productId":"290059","chooseText":"Seleziona","taxConfig":{"includeTax":true,"showIncludeTax":true,"showBothPrices":false,"defaultTax":0,"currentTax":0,"inclTaxTitle":"Incl. Tasse"}});
jQuery("#attribute959 option").each(function () {
var option = jQuery(this);
var id = option.attr('value');
jQuery.each(spConfig.config.attributes, function () {
jQuery.each(this.options, function () {
if (this.id == id) {
if (spConfigDisabledProducts.indexOf(this.products[0]) >= 0) {
option.data('disabled', true);
}
}
});
});
});
It's possible to do that less elegantly but without regex - just a series of splits and clean ups:
import json
code = [your script above]
code2 = html.replace('½','').split('":"Taglia","options":[{')[1].split('}]}},"template"')[0].split('},{')
for i in range(len(code2)):
data = json.loads('{'+code2[i]+'}')
print(data['id'],data['label'])
Output:
730 36
731 37
732 38
733 38
etc.
You can regex out javascript object and pass to json then parse out info
import re
import json
#html = response.content from requests
html = '''
var spConfigDisabledProducts = [-1
, '290058', '290060', '290061', '290062', '290063', '290065', '290071', '290073', '290075', '290076', '290077', '290078' ];
var spConfig = new Product.Config({"attributes":{"959":{"id":"959","code":"aw_taglia","label":"Taglia","options":[{"id":"730","label":"36 ½","price":"0","oldPrice":"0","products":["290058"]},{"id":"731","label":"37 ½","price":"0","oldPrice":"0","products":["290060"]},{"id":"732","label":"38","price":"0","oldPrice":"0","products":["290061"]},{"id":"733","label":"38 ½","price":"0","oldPrice":"0","products":["290062"]},{"id":"734","label":"39","price":"0","oldPrice":"0","products":["290063"]},{"id":"735","label":"40","price":"0","oldPrice":"0","products":["290064"]},{"id":"736","label":"40 ½","price":"0","oldPrice":"0","products":["290065"]},{"id":"737","label":"41","price":"0","oldPrice":"0","products":["290066"]},{"id":"738","label":"42","price":"0","oldPrice":"0","products":["290067"]},{"id":"739","label":"42 ½","price":"0","oldPrice":"0","products":["290068"]},{"id":"740","label":"43","price":"0","oldPrice":"0","products":["290069"]},{"id":"741","label":"44","price":"0","oldPrice":"0","products":["290070"]},{"id":"742","label":"44 ½","price":"0","oldPrice":"0","products":["290071"]},{"id":"743","label":"45","price":"0","oldPrice":"0","products":["290072"]},{"id":"744","label":"45 ½","price":"0","oldPrice":"0","products":["290073"]},{"id":"745","label":"46","price":"0","oldPrice":"0","products":["290074"]},{"id":"746","label":"47","price":"0","oldPrice":"0","products":["290075"]},{"id":"747","label":"47 ½","price":"0","oldPrice":"0","products":["290076"]},{"id":"748","label":"13.5","price":"0","oldPrice":"0","products":["290077"]},{"id":"749","label":"48 ½","price":"0","oldPrice":"0","products":["290078"]}]}},"template":"#{price}\u00a0\u20ac","basePrice":"130","oldPrice":"130","productId":"290059","chooseText":"Seleziona","taxConfig":{"includeTax":true,"showIncludeTax":true,"showBothPrices":false,"defaultTax":0,"currentTax":0,"inclTaxTitle":"Incl. Tasse"}});
jQuery("#attribute959 option").each(function () {
var option = jQuery(this);
var id = option.attr('value');
jQuery.each(spConfig.config.attributes, function () {
jQuery.each(this.options, function () {
if (this.id == id) {
if (spConfigDisabledProducts.indexOf(this.products[0]) >= 0) {
option.data('disabled', true);
}
}
});
});
});'''
p = re.compile(r'Product\.Config\((.*?)\)', re.DOTALL)
data = json.loads(p.findall(html)[0])
for attribute in data['attributes']:
print('-----------------attribute--------------')
print('label = ' + data['attributes'][attribute]['label'], 'id = ' + data['attributes'][attribute]['id'])
print('-----------------options----------------')
for product in data['attributes'][attribute]['options']:
print('label = ' + product['label'], 'id = ' + product['id'], 'product = ' + product['products'][0])
simplest way to update data in VegaEmbed
I made a small graph to show some data from a bluetooth device.
I used a sample I found for VegaEmbed, it was all very easy.
But the sample uses a timer to get data, so even if there is no data the dataset will be changed. What is the simples way to update data inside VegaEmbed from another part of the website ?
I cannot call res.view.change('table', changeSet).run(); from outside VegaEmbded..
Here is snappshot of the code :
(the function handleDataChanged is called when there is bluetooth data.)
function handleDataChanged(event) {
var value = event.target.value;
value = value.buffer ? value : new DataView(value);
let result = {};
let index = 1;
datapointx = value.getInt16(index, /*littleEndian=*/false);
console.log('X: ' + value.getInt16(index, /*littleEndian=*/false));
index += 2;
datapointy = value.getInt16(index, /*littleEndian=*/true);
console.log('Y: ' + value.getInt16(index, /*littleEndian=*/false));
index += 2;
datapointz = value.getInt16(index, /*littleEndian=*/true);
console.log('Z: ' + value.getInt16(index, /*littleEndian=*/false));
index += 2;
}
</script>
<script>
document.querySelector('button').addEventListener('click', function() {
onButtonClick();
});
</script>
<script type="text/javascript">
var vlSpec = {
$schema: 'https://vega.github.io/schema/vega-lite/v3.json',
data: {name: 'table'},
width: 400,
mark: 'line',
encoding: {
x: {field: 'x', type: 'quantitative', scale: {zero: false}},
y: {field: 'y', type: 'quantitative'},
color: {field: 'category', type: 'nominal'}
}
};
vegaEmbed('#chart', vlSpec).then(function(res) {
/**
* Generates a new tuple with random walk.
*/
function newGenerator() {
var counter = -1;
var previousY = [5, 5, 5];
return function() {
counter++;
var newVals = previousY.map(function(v, c)
{
console.log('c = ' + c);
var yval = 0;
if (c == 0)
yval = datapointx;
if (c == 1)
yval = datapointy;
if (c == 2)
yval = datapointz;
return {
x: counter,
// y: v + Math.round(Math.random() * 10 - c * 3),
y: yval,
category: c
};
});
previousY = newVals.map(function(v) {
return v.y;
});
return newVals;
};
}
var valueGenerator = newGenerator();
var minimumX = -100;
window.setInterval(function() {
minimumX++;
var changeSet = vega
.changeset()
.insert(valueGenerator())
.remove(function(t) {
return t.x < minimumX;
});
res.view.change('table', changeSet).run();
}, 100);
});
</script>
The simplest way to update data in an existing vega-lite chart is to use a streaming data model. There is an example in the Vega-Lite documentation here: https://vega.github.io/vega-lite/tutorials/streaming.html
Download/Export json file on mac OS with Angular 5
I tried to download a json object on my device "mac os" using the code below. The result was show the json body on the browser, although it works with other os "android, windows".
Is there any alternative solution?
function downloadJsonFile(fileName, jsonObject) {
let fileContents = JSON.stringify(jsonObject);
let data = "text/json;charset=utf-8," + encodeURIComponent(fileContents);
let a = document.createElement('a');
a.href = 'data:' + data;
a.download = fileName + '.json';
a.click();
}
Try something like this:
DEMO ----> Download Json
download(strData, strFileName, strMimeType) {
strData = JSON.stringify(strData);
var D = document,
A = arguments,
a = D.createElement("a"),
n = A[1];
a.href = "data:" + strMimeType + "charset=utf-8," + (strData);
if ('download' in a) {
a.setAttribute("download", n);
a.innerHTML = "downloading...";
D.body.appendChild(a);
setTimeout(function () {
var e = D.createEvent("MouseEvents");
e.initMouseEvent("click", true, false, window, 0, 0, 0, 0, 0, false, false, false, false, 0, null);
a.dispatchEvent(e);
D.body.removeChild(a);
}, 66);
return true;
};
var f = D.createElement("iframe");
D.body.appendChild(f);
f.src = "data:" + (A[2] ? A[2] : "application/octet-stream") + (window.btoa ? ";base64" : "") + "," + (strData);
setTimeout(function () {
D.body.removeChild(f);
}, 333);
return true;
}