so basically i wrote a code to extract the names of the degrees Harvard is offering using scrapy however my code does not extract anything
import scrapy
from ..items import HarvardItem
class AmazonrevSpider(scrapy.Spider):
name = 'harvard'
pgNum=1
start_urls = [
"https://www.harvard.edu/programs/?degree_levels=undergraduate&page={pgNum}"
]
degree_urls=[]
def parse(self, response):
items = HarvardItem()
name = response.css('.c-programs-item__title::text').extract()
items['Name'] = name
yield items
Related
I have posted a question with the same title as this one, everyone can see it from this link
Unfortunately, because that was my first time making a question, some format mistakes were made and that question is closed for the moment. I have edited it and resubmitted for opening again but I don't know how long will that take so I might as well make a new one.
I'm working on a web application for reading novels and currently I'm stuck at a certain part in directing views using urls. I use Django as the back end with PostgreSQL as the database and HTML with bootsrap as front end. I will post my code below:
This is my urls.py (from the project folder):
from django.contrib import admin
from django.urls import path, include
from django.conf import settings
from django.conf.urls.static import static
urlpatterns = [
path('admin/', admin.site.urls),
path('', include('app.urls')),
]
urlpatterns += static(settings.MEDIA_URL, document_root = settings.MEDIA_ROOT)
This is my urls.py (from the app folder):
from django.contrib import admin
from django.urls import path, include
from django.contrib.auth import views as auth_views
from . import views
urlpatterns = [
path('', views.home),
path('book/<slug:book_slug>/<slug:chapter_slug>/', views.detail, name='detail'),
path('genre/<slug:category_slug>', views.category_detail, name='category_detail'),
path('book/<slug:book_slug>', views.book_detail, name='book_detail'),
path('register/', views.register, name="register"),
path('login/',auth_views.LoginView.as_view(template_name="app/login.html"), name="login"),
path('logout/',auth_views.LogoutView.as_view(next_page='/'),name='logout'),
]
This is my models.py:
from django.db import models
# Create your models here.
class Category(models.Model):
name = models.CharField(max_length=50)
slug = models.SlugField(max_length=50, unique=True, blank=True, editable=True)
def __str__(self):
return self.name
class Chapter(models.Model):
chapter = models.Count
chapter_name = models.CharField(max_length=100)
book = models.ForeignKey('Book', on_delete=models.CASCADE, null=True, related_name = 'Books')
detail = models.TextField()
slug = models.SlugField(max_length=100, unique=True, blank=True, editable=True)
def __str__(self):
return self.chapter_name
class Book(models.Model):
title = models.CharField(max_length=100)
slug = models.SlugField(max_length=100, unique=True, blank=True, editable=True)
cover_image = models.ImageField(upload_to= 'images/', blank= True, null = True)
author = models.CharField(max_length=100)
summary = models.TextField()
category = models.ForeignKey('Category', on_delete=models.CASCADE, null = True, related_name = 'books')
#date = models.DateField(auto_now_add=True)
recommended_book = models.BooleanField(default=False)
recommended_book_hot = models.BooleanField(default=False)
recommended_book_new = models.BooleanField(default=False)
recommended_book_finish = models.BooleanField(default=False)
def __str__(self):
return self.title
My views.py:
from django.shortcuts import render
from django.http import HttpResponse
from django.http import HttpResponseRedirect
from .forms import RegistrationForm
from .models import Book, Category, Chapter
from gtts import gTTS
from django.urls import reverse
from django.shortcuts import get_object_or_404
# Create your views here.
def home(request):
recommended_book = Book.objects.filter(recommended_book = True)
recommended_book_hot = Book.objects.filter(recommended_book_hot = True)
recommended_book_new = Book.objects.filter(recommended_book_new = True)
recommended_book_finish = Book.objects.filter(recommended_book_finish = True)
return render(request, 'app/home.html', {'recommended_book': recommended_book, 'recommended_book_hot': recommended_book_hot, 'recommended_book_new': recommended_book_new, 'recommended_book_finish': recommended_book_finish})
def detail(request, book_slug, chapter_slug):
book = get_object_or_404(Chapter, book__slug = book_slug,slug = chapter_slug)
title = book.slug
return render(request, 'app/detail.html', {'detail': book, 'title':title})
def register(request):
form = RegistrationForm()
if request.method == 'POST':
form = RegistrationForm(request.POST)
if form.is_valid():
form.save()
return HttpResponseRedirect('/')
return render(request, 'app/register.html', {'form': form})
def category_detail(request, category_slug):
category = Category.objects.get(slug = category_slug)
books = Category.objects.prefetch_related('books')
return render(request, 'app/genre_detail.html', {'category':category})
def book_detail(request, book_slug):
book = Book.objects.get(slug = book_slug)
book_category = book.category.first()
similar_books = Book.objects.filter(category__name__startswith = book_category)
return render(request, 'app/book_detail.html', {'book': book, 'similar_books': similar_books, 'book_category': book_category})
With the configurations in urls.py, I have managed to go from homepage to a specific novel index page with the url: localhost:8000/book/book.slug then read a chapter using urL:localhost:8000/book/book.slug/chapter.slug, I then place an HTML link to go back to the index page in case I want to stop reading the chapter that goes like this:
<div class="container p-2">
Home /
{{detail.book}}/
{{detail.chapter_name}}
</div>
This is where I'm met with issues as what I have in mind was that when clicked, the link will lead to the index page again which is localhost:8000/book/book.slug. But what actually happened was that it showed a NoReveseMatch error upon the chapter page reload again.
From what I can inferred after seeing the logs, there should have been a slug as an argument for the view Book_detail() to work, such as 'test' (slug for a novel name "Test"). But instead, the argument show a blank '('',)' which made the link failed to direct to Book_detail().
Does anyone know what I did wrong ? Any helps or suggestions are appreciated
Edit 1: The NoReverseMatching error has been fixed but now clicking on the url will return a DoesNotExist at book/chapter.slug error instead.
Edit 2: This question has been answered, you can see the answers below.
Change this to:
{{detail.book}}/
Try this
{{detail`.book}}
The problem about DoesNotExist has been fixed !
I found out that by adding another definition in views.py at detail() that goes like this:
def detail(request, book_slug, chapter_slug):
book = get_object_or_404(Chapter, book__slug = book_slug,slug = chapter_slug)
index = Book.objects.get(slug = book_slug)
title = index.slug
return render(request, 'app/detail.html', {'detail': book, 'title':title})
then add it to the HTML like this:
<div class="container p-2">
Home /
{{detail.book}}/
{{detail.chapter_name}}
</div>
I can now return to the Index page !
Thank you, everyone, for your helps and suggestions. I have also realized that a lot of clumsy mistakes were made and they weren't really that hard to solve, actually. It's just that I overlooked them.
im trying to post the recruitment date of an employee , but i keep having the 400 bad request error and recruitment_date: ["This field is required."] ,after searching i added the Date Formats on the settings but i still have the same error
views.py :
class CreateEmployee(generics.CreateAPIView):
lookup_field= 'pk'
serializer_class = EmployeeSerializer
def get_queryset(self):
return Employee.objects.all()
serializers.py :
class EmployeeSerializer(serializers.ModelSerializer):
url = serializers.SerializerMethodField(read_only=True)
recruitment_date =fields.DateField(format="%Y-%m-%d",input_formats=['%Y-%m-%d'])
class Meta:
model = Employee
fields =['url','id','f_name','l_name','job','telephone','recruitment_date','salary']
def get_url(self,obj):
request = self.context.get("request")
return obj.get_api_url(request=request)
settings.py :
"DATE_INPUT_FORMATS": ["iso-8601","%Y-%m-%d"],
I am working on a project from school and one of the questions is List all leagues with a (current) player named "Sophia". I have been able to get all the leagues period, and all the players name Sophia. How ever I cannot seem to figure out how to start in the last table where the first name is reference the first table where the leagues are to get the answer I need. I am hoping someone can see what I am having issues seeing. I have linked the models and views below.
Manage.py
from django.db import models
class League(models.Model):
name = models.CharField(max_length=50)
sport = models.CharField(max_length=15)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
class Team(models.Model):
location = models.CharField(max_length=50)
team_name = models.CharField(max_length=50)
league = models.ForeignKey(League, related_name="teams")
class Player(models.Model):
first_name = models.CharField(max_length=15)
last_name = models.CharField(max_length=15)
curr_team = models.ForeignKey(Team, related_name="curr_players")
all_teams = models.ManyToManyField(Team, related_name="all_players")
views.py
from django.shortcuts import render, redirect
from .models import League, Team, Player
from . import team_maker
def index(request):
context = {
"leagues": League.objects.all(),
"teams": Team.objects.all(),
"players": Player.objects.all(),
"baseball" : League.objects.filter(sport__contains="Baseball"),
"women" : League.objects.filter(name__contains="Womens'"),
"ice" : League.objects.filter(sport__contains="ice"),
"nofootball" : League.objects.exclude(sport__contains="football"),
"conferences" : League.objects.filter(name__contains="conference"),
"atlantic": League.objects.filter(name__contains="Atlantic"),
"teamdallas": Team.objects.filter(location__contains="Dallas"),
"raptor": Team.objects.filter(team_name__contains="Raptor"),
"cityloc": Team.objects.filter(location__contains="city"),
"startT": Team.objects.filter(team_name__startswith="T"),
"abc": Team.objects.order_by('location'),
"cba": Team.objects.order_by('location').reverse(),
"cooper": Player.objects.filter(last_name="Cooper"),
"joshua": Player.objects.filter(first_name="Joshua"),
"nocooper": Player.objects.filter(last_name="Cooper").exclude(first_name="Joshua"),
"wyatt": Player.objects.filter(first_name="Alexander")|Player.objects.filter(first_name="Wyatt"),
"atlanticsoccer" : Team.objects.filter(league__name__contains='atlantic', league__sport__contains="soccer"),
"bostonp" : Player.objects.filter(curr_team__team_name__contains='Penguins'),
"icbc" : Player.objects.filter(curr_team__league__name__contains='International Collegiate Baseball '),
"footballlopez" : Player.objects.filter(curr_team__league__name__contains="American Conference of Amateur Football", last_name__contains="lopez"),
"footballplayer" : Player.objects.filter(curr_team__league__sport__contains="football"),
"sophia" : Team.objects.filter(curr_players__first_name__contains='sophia'),
"sophiale" : Team.objects.filter(league__contains='sophia'),
}
print Team.objects.filter(league__contains='sophia')
return render(request, "leagues/index.html", context)
def make_data(request):
team_maker.gen_leagues(10)
team_maker.gen_teams(50)
team_maker.gen_players(200)
return redirect("index")
index.html just the part you need
<h5>Question 8</h5>
{% for player in sophiale %}
<li>{{player.name}}</li>
{% endfor %}
</ol>
Try this:
League.objects.filter(teams__ all_players__first_name__contains='sophia')
In django I have create query like wise if someone want information:
In district parameter select Pune, then output gives data for pune district only.
for example :
http:127.0.0.1/api/?district=Pune
htt:127.0.0.1:8000/?crop = groundnut
and so on.
Next,I want to create a Html page for my starting django page(index.html)
if I runserver http:127.0.0.1:8000/
display my html file , where Our models fields(paramter) is id and then user submit the question "if" condition will be trigger and searching information for that parameters.
like wise:
District = __________ submit
gives data only for selected district name only
also
Crop = ______________ submit
gives data only for selected crop name only
likewise run this query
http:/api/?crop=groundnut
if user choose crop name is groundnut,
if use choose crop name is guava, then http:/api/?crop=guava query will be run.
So,now I want to create index.html file multiple parameters works.(AND query will apply)
http:/api/district=Pune&crop=groundnut
So, I want to create html page which call to this query and its show me this type
Distrcit : ________ Crop : __________ submit
here is my models.py
from django.db import models
from mongoengine import Document, fields
class Tool(Document):
crop = fields.StringField(required=True)
district = fields.StringField(required=True)
def __str__(self):
return self.crop
def save(self,*args, **kwargs):
super().save(*args, **kwargs)
class Meta:
verbose_name_plural ='tool'
project/urls.py
from django.conf import settings
from django.conf.urls.static import static
from django.conf.urls import include, url
from django.contrib import admin
from app.views import *
from routers import HybridRouter
router = HybridRouter()
router.register(r'tool', ToolViewSet, r"tool")
urlpatterns = [
url(r'^admin/', include(admin.site.urls)),
url(r'^api/', include(router.urls, namespace='api')),
url(r'^$', index_view, {}, name='index'),
]
app/views.py
from django.template.response import TemplateResponse
from rest_framework_mongoengine.viewsets import ModelViewSet as MongoModelViewSet
from app.serializers import *
class ToolViewSet(MongoModelViewSet):
serializer_class = ToolSerializer
my_filter_fields = ('crop', 'district',) # specify the fields on which you want to filter
def get_kwargs_for_filtering(self):
filtering_kwargs = {}
for field in self.my_filter_fields: # iterate over the filter fields
field_value = self.request.query_params.get(field) # get the value of a field from request query parameter
if field_value:
filtering_kwargs[field] = field_value
return filtering_kwargs
def get_queryset(self):
queryset = Tool.objects.all()
filtering_kwargs = self.get_kwargs_for_filtering() # get the fields with values for filtering
if filtering_kwargs:
queryset = Tool.objects.filter(**filtering_kwargs) # filter the queryset based on 'filtering_kwargs'
return queryset
from .models import Tool
from django.shortcuts import render
def index_view(request):
questions=None
if request.GET.get('crop'):
crop = request.GET.get('crop')
questions = Tool.objects.filter(crop__icontains=crop)
district = request.GET.get('district')
query = Tool.object.create(crop=crop, district=district)
query.save()
return render(request, 'index.html',{
'questions': questions,
})
what I have to change in views.py file for index.html to get parameter and gives exact information after user click on submit button ?
index.html
<form method="GET">
Crop: <input type="text" name="crop"><br/>
Taluka: <input type="text" name="taluka"><br/>
<input type="submit" value="Submit" />
</form><br/><br/>
{% for question in questions %}
<p>{{question}}</p>
{% endfor %}
can you help me how to get data and run the query ?
where I need to changes ?
Its a TYPO error in this line :
query = Tool.object.create(crop=crop, district=district)
object should be objects ,like this :
query = Tool.objects.create(crop=crop, district=district)
Update your code like this :
from .models import Tool
from django.shortcuts import render
def index_view(request):
questions=None
if request.GET.get('crop'):
crop = request.GET.get('crop')
questions = Tool.objects.filter(crop__icontains=crop)
district = request.GET.get('district')
query = Tool.objects.create(crop=crop, district=district)
query.save()
return render(request, 'index.html',{
'questions': questions,
})
And, Update the save method of your Tool model like this :
class Tool(Document):
crop = fields.StringField(required=True)
district = fields.StringField(required=True)
def __str__(self):
return self.crop
def save(self,*args, **kwargs):
super(Tool, self).save(*args, **kwargs)
In your index.html file, change {{question}} to {{question.crop}}. You have to refer to the field (in your case crop) on the model you want to render on your page.
Also, the default manager is located at Tool.objects not Tool.object.
In your models.py, super should be super(Tool, self). Also model Tool should inherit from django.db.models.Model. Your fields must be from django.db.models
class Tool(Model):
crop = models.CharField()
I have the following model:
class Messages(models.Model):
userid = models.ForeignKey(User)
time = models.TimeField(default=datetime.datetime.now)
content = models.CharField(max_length=255)
id = models.AutoField(primary_key=True)
I want to json this model with custom representation to pass it in javascript. I used the follow code for this by iterating through QuerySet
messages = Messages.objects.all()
passed_messages = []
for singleMess in messages:
passed_messages.append(get_message(singleMess))
response = json.dumps(passed_messages)
def get_message(message):
return {
'user': User.objects.get_by_natural_key(message.userid).username,
'content': message.content,
'hour': message.time.hour,
'minute': message.time.minute,
'id': message.id
}
But this part looks ugly. Is there a way to replace it with something nicer. I could use the followings
__repr__ in models.py
django.core.serializers
django.forms.models.model_to_dict
Messages.objects.all().values()
I just don't know how to come across.
For example
json.dumps needs a pure Python list of dictionaries, but __repr__ returns QuerySet which I don't know how to conrent.
If I use model_to_dict or serializers or list(messages.values()) I can't join the User's field username instead of userid.
I added property to my model class
class Messages(models.Model):
userid = models.ForeignKey(User)
time = models.TimeField(default=datetime.datetime.now)
content = models.CharField(max_length=255)
id = models.AutoField(primary_key=True)
#property
def json(self):
return {
'user': User.objects.get_by_natural_key(self.userid).username,
'content': self.content,
'time': self.time.strftime("%H:%M:%S"),
'id': self.id
}
And used comprehensions:
import json
messages = Messages.objects.all()
json.dumps([message.json for message in messages])