Perl script to parse JSON data - json

I am trying to create a script to parse JSON content but it is not working:
#!/usr/local/bin/perl-w
use DBI;
use Parallel::ForkManager;
use LWP::Simple;
use XML::Simple;
use JSON qw( decode_json );
use Data::Dumper;
use DateTime;
use WWW::Mechanize;
use strict;
use warnings;
use JSON;
my $ua = LWP::UserAgent->new();
my $req = new HTTP::Request GET => 'https://google.com/pub/';
my $res = $ua->request($req);
my $contents = $res->content;
##json response:
#{"success":true,"data":"{\"campaign\":\"21490|\",\"format\":\"md5 \",\"delta_timestamp\":\"1528992718\",\"result\":\"success\",\"download_link\":\"https:\\\/\\\/gmail.net\\\/accesskey\\\/getfile\\\/m-spqn-e61-2aef2575a0b5250354f2b0fda033e703?token=HUSYjdC5jyJskXUHiKn13l1A1BaAjH2R&dcma=e8fae90c472ae146\"}","message":null}
print $contents;
#Check the outcome of the Response
if ( $res->is_success ) {
print $res->content;
}
# Decode the main json object
my $jsn = decode_json($res);
# Since 'data' is another serialized object, you need to decode that as well:
my $data = decode_json($jsn);
# Now you can access the contents of 'data'
#want to extract download_link object
print $data->{'download_url'};
I am looking at the content of download_link.

use JSON qw(decode_json);
# from $res->content
my $content = '{"success":true,"data":"{\"campaign\":\"21490|\",\"format\":\"md5 \",\"delta_timestamp\":\"1528992718\",\"result\":\"success\",\"download_link\":\"https:\\\/\\\/gmail.net\\\/accesskey\\\/getfile\\\/m-spqn-e61-2aef2575a0b5250354f2b0fda033e703?token=HUSYjdC5jyJskXUHiKn13l1A1BaAjH2R&dcma=e8fae90c472ae146\"}","message":null}';
print decode_json(decode_json($content)->{data})->{download_link};

Related

Escape special characters in JSON string

I have Perl script which contains variable $env->{'arguments'}, this variable should contain a JSON object and I want to pass that JSON object as argument to my other external script and run it using backticks.
Value of $env->{'arguments'} before escaping:
$VAR1 = '{"text":"This is from module and backslash \\ should work too"}';
Value of $env->{'arguments'} after escaping:
$VAR1 = '"{\\"text\\":\\"This is from module and backslash \\ should work too\\"}"';
Code:
print Dumper($env->{'arguments'});
escapeCharacters(\$env->{'arguments'});
print Dumper($env->{'arguments'});
my $command = './script.pl '.$env->{'arguments'}.'';
my $output = `$command`;
Escape characters function:
sub escapeCharacters
{
#$env->{'arguments'} =~ s/\\/\\\\"/g;
$env->{'arguments'} =~ s/"/\\"/g;
$env->{'arguments'} = '"'.$env->{'arguments'}.'"';
}
I would like to ask you what is correct way and how to parse that JSON string into valid JSON string which I can use as argument for my script.
You're reinventing a wheel.
use String::ShellQuote qw( shell_quote );
my $cmd = shell_quote('./script.pl', $env->{arguments});
my $output = `$cmd`;
Alternatively, there's a number of IPC:: modules you could use instead of qx. For example,
use IPC::System::Simple qw( capturex );
my $output = capturex('./script.pl', $env->{arguments});
Because you have at least one argument, you could also use the following:
my $output = '';
open(my $pipe, '-|', './script.pl', $env->{arguments});
while (<$pipe>) {
$output .= $_;
}
close($pipe);
Note that current directory isn't necessarily the directory that contains the script that executing. If you want to executing script.pl that's in the same directory as the currently executing script, you want the following changes:
Add
use FindBin qw( $RealBin );
and replace
'./script.pl'
with
"$RealBin/script.pl"
Piping it to your second program rather than passing it as an argument seems like it would make more sense (and be a lot safer).
test1.pl
#!/usr/bin/perl
use strict;
use JSON;
use Data::Dumper;
undef $/;
my $data = decode_json(<>);
print Dumper($data);
test2.pl
#!/usr/bin/perl
use strict;
use IPC::Open2;
use JSON;
my %data = ('text' => "this has a \\backslash", 'nums' => [0,1,2]);
my $json = JSON->new->encode(\%data);
my ($chld_out, $chld_in);
print("Executing script\n");
my $pid = open2($chld_out, $chld_in, "./test1.pl");
print $chld_in "$json\n";
close($chld_in);
my $out = do {local $/; <$chld_out>};
waitpid $pid, 0;
print(qq~test1.pl output =($out)~);

Working with JSON in perl

I need to print all "time" values from below JSON in Perl. How do it?
#!/usr/bin/perl -w
use strict;
use JSON;
use Data::Dumper;
$a = q({"Response":"Success","Type":100,"Aggregated":true,"Data":[{"time":1521064980,"close":8130.26,"high":8153.57,"low":8121.42,"open":8152.29,"volumefrom":711.3,"volumeto":5815443.51},{"time":1521065160,"close":8161.97,"high":8167.7,"low":8130.51,"open":8130.51,"volumefrom":527.3699999999999,"volumeto":4348419.180000001}{"time":1521075780,"close":8129.27,"high":8129.71,"low":8126.7,"open":8127.4,"volumefrom":50.35,"volumeto":412670.56000000006}],"TimeTo":1521075900,"TimeFrom":1521064980,"FirstValueInArray":true,"ConversionType":{"type":"direct","conversionSymbol":""}});
$b = decode_json $a;
You can try below code to print values of time in your json:
#!/usr/bin/perl -w
use strict;
use JSON;
use Data::Dumper;
my $json = q({"Response":"Success","Type":100,"Aggregated":true,"Data":[{"time":1521064980,"close":8130.26,"high":8153.57,"low":8121.42,"open":8152.29,"volumefrom":711.3,"volumeto":5815443.51},{"time":1521065160,"close":8161.97,"high":8167.7,"low":8130.51,"open":8130.51,"volumefrom":527.3699999999999,"volumeto":4348419.180000001},{"time":1521075780,"close":8129.27,"high":8129.71,"low":8126.7,"open":8127.4,"volumefrom":50.35,"volumeto":412670.56000000006}],"TimeTo":1521075900,"TimeFrom":1521064980,"FirstValueInArray":true,"ConversionType":{"type":"direct","conversionSymbol":""}});
my $ref = decode_json $json;
# access 'Data' in hashref $ref
my $data = $ref->{'Data'};
# loop through item in array ref $data
for my $item (#{$data}) {
# print value of "time" with a newline
print $item->{'time'} . "\n";
}
# print 'TimeTo' value
print "TimeTo: " . $ref->{'TimeTo'};

JSON encode/decode utf8 strings in Perl

I pass a utf8 encoded string from my command line into a Perl program:
> ./test.pl --string='ḷet ūs try ṭhiñgs'
which seems to recognize the string correctly:
use utf8;
GetOptions(
'string=s' => \$string,
) or die;
print Dumper($string);
print Dumper(utf8::is_utf8($string));
print Dumper(utf8::valid($string));
prints
$VAR1 = 'ḷet ūs try ṭhiñgs';
$VAR1 = '';
$VAR1 = 1;
When I store this string into a hash and call encode_json on it, the string seems to be again encoded whereas to_json seems to work (if I read the output correctly):
my %a = ( 'nāme' => $string ); # Note the Unicode character
print Dumper(\%a);
print Dumper(encode_json(\%a));
print Dumper(to_json(\%a));
prints
$VAR1 = {
"n\x{101}me" => 'ḷet ūs try ṭhiñgs'
};
$VAR1 = '{"nāme":"ḷet Å«s try á¹­hiñgs"}';
$VAR1 = "{\"n\x{101}me\":\"\x{e1}\x{b8}\x{b7}et \x{c5}\x{ab}s try \x{e1}\x{b9}\x{ad}hi\x{c3}\x{b1}gs\"}";
Turning this back into the original hash, however, doesn't seem to work with either methods and in both cases hash and string and broken:
print Dumper(decode_json(encode_json(\%a)));
print Dumper(from_json(to_json(\%a)));
prints
$VAR1 = {
"n\x{101}me" => "\x{e1}\x{b8}\x{b7}et \x{c5}\x{ab}s try \x{e1}\x{b9}\x{ad}hi\x{c3}\x{b1}gs"
};
$VAR1 = {
"n\x{101}me" => "\x{e1}\x{b8}\x{b7}et \x{c5}\x{ab}s try \x{e1}\x{b9}\x{ad}hi\x{c3}\x{b1}gs"
};
A hash lookup $a{'nāme'} now fails.
Question: How do I handle utf8 encoding and strings and JSON encode/decode correctly in Perl?
You need to decode your input:
use Encode;
my $string;
GetOptions('string=s' => \$string) or die;
$string = decode('UTF-8', $string);
Putting it all together, we get:
use strict;
use warnings;
use 5.012;
use utf8;
use Encode;
use Getopt::Long;
use JSON;
my $string;
GetOptions('string=s' => \$string) or die;
$string = decode('UTF-8', $string);
my %hash = ('nāme' => $string);
my $json = encode_json(\%hash);
my $href = decode_json($json);
binmode(STDOUT, ':encoding(utf8)');
say $href->{nāme};
Example:
$ perl test.pl --string='ḷet ūs try ṭhiñgs'
ḷet ūs try ṭhiñgs
Make sure your source file is actually encoded as UTF-8!

Parsing JSON Data::Dumper output array in Perl

I'm trying to edit an old perl script and I'm a complete beginner. The request from the server returns as:
$VAR1 = [
{
'keywords' => [
'bare knuckle boxing',
'support group',
'dual identity',
'nihilism',
'support',
'rage and hate',
'insomnia',
'boxing',
'underground fighting'
],
}
];
How can I parse this JSON string to grab:
$keywords = "bare knuckle boxing,support group,dual identity,nihilism,support,rage and hate,insomnia,boxing,underground fighting"
Full perl code
#!/usr/bin/perl
use LWP::Simple; # From CPAN
use JSON qw( decode_json ); # From CPAN
use Data::Dumper; # Perl core module
use strict; # Good practice
use warnings; # Good practice
use WWW::TheMovieDB::Search;
use utf8::all;
use Encode;
use JSON::Parse 'json_to_perl';
use JSON::Any;
use JSON;
my $api = new WWW::TheMovieDB::Search('APIKEY');
my $img = $api->type('json');
$img = $api->Movie_imdbLookup('tt0137523');
my $decoded_json = decode_json( encode("utf8", $img) );
print Dumper $decoded_json;
Thanks.
Based on comments and on your recent edit, I would say that what you are asking is how to navigate a perl data structure, contained in the variable $decoded_json.
my $keywords = join ",", #{ $decoded_json->[0]{'keywords'} };
say qq{ #{ $arrayref->[0]->{'keywords'} } };
As TLP pointed out, all you've shown is a combination of perl arrays/hashes. But you should look at the JSON.pm documentation, if you have a JSON string.
The result you present is similar to json, but the Perl-variant of it. (ie => instead of : etc). I don't think you need to look into the json part of it, As you already got the data. You just need to use Perl to join the data into a text string.
Just to eleborate on the solution to vol7ron :
#get a reference to the list of keywords
my $keywords_list = $decoded_json->[0]{'keywords'};
#merge this list with commas
my $keywords = join(',', #$keywords_list );
print $keywords;

Perl HTML Cleaning

How can i make the HTML::Clean Module work in this perl script.
#!/usr/bin/env perl
use strict;
use warnings;
require LWP::UserAgent;
open FH, "<", "text.txt";
my $ua = LWP::UserAgent->new;
$ua->timeout(10);
foreach my $line (<FH>) {
my $response = $ua->get($line);
my $h = new HTML::Clean(\$response);
if ($response->is_success) {
print $response->decoded_content;
}
else {
die $response->status_line;
}
}
close FH;
The HTML::Clean constructor method takes an argument which is either a filename or a reference to a string containing HTML. You seem to be passing it an HTTP::Response object. Try passing it the HTML instead.
my $response = $ua->get($line);
my $html = $response->decoded_content;
my $h = HTML::Clean->new(\$html);