Web scraper using dart - html

Question

I am trying to create a web scraper for a single website picking out just a title, Image and link to the website.
The title comes out fine but the image and link are not properly working can anyone please help me out in this.Here is my code and dependencies I used in .yaml
import 'package:flutter/material.dart';
import 'package:flutter_staggered_animations/flutter_staggered_animations.dart';
import 'package:html/dom.dart' as dom;
import 'package:html/parser.dart' as parser;
import 'package:http/http.dart' as http;
import 'package:url_launcher/url_launcher.dart';
void main() {
runApp(MyApp());
}
class MyApp extends StatelessWidget {
#override
Widget build(BuildContext context) {
return MaterialApp(
home: MyHomePage(),
);
}
}
class MyHomePage extends StatefulWidget {
#override
_MyHomePageState createState() => _MyHomePageState();
}
class _MyHomePageState extends State<MyHomePage> {
List<String> title;
List<String> image;
List<String> link;
#override
void initState() {
_getdataFromWeb();
}
void _getdataFromWeb() async {
final response = await http.get('https://www.bewakoof.com/men-shirts');
dom.Document document = parser.parse(response.body);
final pictures = document.getElementsByClassName('productGrid');
final description = document.getElementsByClassName('productCardDetail');
final nextPage =
//document.getElementsByClassName('entry-title');
document.getElementsByClassName('col-sm-4 col-xs-6');
image = pictures
.map((element) =>
element.getElementsByTagName("img")[0].attributes['src'])
.toList();
title = description
.map((element) => element.getElementsByTagName("h3")[0].innerHtml)
.toList();
link = nextPage
.map((element) => element.getElementsByTagName("a")[0].attributes['href'])
.toList();
print(link);
}
#override
Widget build(BuildContext context) {
print("hello");
if (image == null)
print("null");
else
print(image);
return SafeArea(
child: Scaffold(
backgroundColor: Colors.black87,
body: title == null || title.length == 0
? Text(
"No data",
style: TextStyle(
color: Colors.white,
),
)
: ListView.builder(
itemCount: title.length,
itemBuilder: (context, index) {
return AnimationConfiguration.staggeredList(
position: index,
duration: const Duration(milliseconds: 375),
child: SlideAnimation(
child: FadeInAnimation(
child: GestureDetector(
onTap: () async {
dynamic url = link[index];
if (await canLaunch(url))
launch(url);
else {
print('error');
}
},
child: Padding(
padding: const EdgeInsets.all(10),
child: Card(
child: Container(
color: Colors.black87,
child: Column(
children: <Widget>[
Align(
alignment: Alignment.centerLeft,
child: Text(
title[index],
style: TextStyle(
fontWeight: FontWeight.bold,
color: Colors.cyan,
fontSize: 20,
),
),
),
SizedBox(height: 15),
Image.network(image[0]),
],
),
),
),
),
),
),
),
);
},
),
),
);
}
}
cupertino_icons: ^1.0.1
http: ^0.12.0+4
html: ^0.14.0+3
flutter_staggered_animations: "^0.1.2"
url_launcher: ^5.4.0
I may need it tomorrow if it can be possible

Answer 1

enter image description here
Here I added the html pic for the elements rest part I debugged only the link is not working.
Here is the debugged code:
import 'package:flutter/material.dart';
import 'package:flutter_staggered_animations/flutter_staggered_animations.dart';
import 'package:html/dom.dart' as dom;
import 'package:html/parser.dart' as parser;
import 'package:http/http.dart' as http;
import 'package:url_launcher/url_launcher.dart';
void main() {
runApp(MyApp());
}
class MyApp extends StatelessWidget {
#override
Widget build(BuildContext context) {
return MaterialApp(
home: MyHomePage(),
);
}
}
class MyHomePage extends StatefulWidget {
#override
_MyHomePageState createState() => _MyHomePageState();
}
class _MyHomePageState extends State<MyHomePage> {
List<String> title;
List<String> image;
List<String> link;
#override
void initState() {
_getdataFromWeb();
}
void _getdataFromWeb() async {
final response = await http.get(
'https://www.bewakoof.com/');
dom.Document document = parser.parse(response.body);
final pictures =
document.getElementsByClassName('chWrprInner');
//document.getElementsByClassName('entry-header blog-entry-header');
final description =
//document.getElementsByClassName('entry-content');
document.getElementsByClassName('chWrprInner');
final nextPage =
//document.getElementsByClassName('entry-title');
document.getElementsByClassName('chWrprInner');
image = pictures
.map((element) =>
element.getElementsByTagName("img")[0]
.attributes['src'])
.toList();
title = description
.map((element) => element.getElementsByTagName("p")[0]
.innerHtml)
.toList();
link = nextPage
.map((element) =>
element.getElementsByTagName("a")[0]
.attributes['href'])
.toList();
print(link);
}
#override
Widget build(BuildContext context) {
print("hello");
if (image == null)
print("null");
else
print(image);
return SafeArea(
child: Scaffold(
backgroundColor: Colors.black87,
body: title == null || title.length == 0
? Text(
"No data",
style: TextStyle(
color: Colors.white,
),
)
: ListView.builder(
itemCount: title.length,
itemBuilder: (context, index) {
return AnimationConfiguration.staggeredList(
position: index,
duration: const Duration(milliseconds: 375),
child: SlideAnimation(
child: FadeInAnimation(
child: GestureDetector(
onTap: () async {
dynamic url = link[index];
if (await canLaunch(url))
launch(url);
else {
print('error');
}
},
child: Padding(
padding: const EdgeInsets.all(10),
child: Card(
child: Container(
color: Colors.black87,
child: Column(
children: <Widget>[
Align(
alignment: Alignment.centerLeft,
child: Text(
title[index],
style: TextStyle(
fontWeight: FontWeight.bold,
color: Colors.cyan,
fontSize: 20,
),
),
),
SizedBox(height: 15),
Text(
title[index],
style: TextStyle(
color: Colors.white,
),
),
Image.network(image[index]),
],
),
),
),
),
))),
);
},
),
),
);
}
}

Answer 2

you can use this package for web scraping:
web_scraper: ^0.0.9
if need see sample , you can visit this repository:
https://github.com/esmaeil-ahmadipour/Flutter_Web_Scraper