I am currently making a course regarding to time series and sunspot data set is used in this course.
I wonder what is the role of the next() function in the below code when I ignore it code line 'sunspots.append(float(row[2]))' gives error;
'''
import csv
time_step = []
sunspots = []
with open('./Sunspots.csv') as csvfile:
reader = csv.reader(csvfile, delimiter=',')
print(reader)
next(reader)
for row in reader:
print(row)
sunspots.append(float(row[2]))
time_step.append(int(row[0]))
series = np.array(sunspots)
time = np.array(time_step)
plt.figure(figsize=(10, 6))
plot_series(time, series) '''
Thank you in advance
Related
I am trying to convert an image uploaded by user into a PDF , and then store it into an ImageField in a mysql database ,using a form, but am facing an error when trying to store the PDF into the database
My views.py is:
from django.core.files.storage import FileSystemStorage
from PIL import Image
import io
from io import BytesIO
from django.core.files.uploadedfile import InMemoryUploadedFile
from django.core.files.base import ContentFile
def formsubmit(request): #submits the form
docs = request.FILES.getlist('photos')
print(docs)
section = request.POST['section']
for x in docs:
fs = FileSystemStorage()
print(type(x.size))
img = Image.open(io.BytesIO(x.read()))
imgc = img.convert('RGB')
pdfdata = io.BytesIO()
imgc.save(pdfdata,format='PDF')
thumb_file = ContentFile(pdfdata.getvalue())
filename = fs.save('photo.pdf', thumb_file)
linkobj = Link(link = filename.file, person = Section.objects.get(section_name = section), date = str(datetime.date.today()), time = datetime.datetime.now().strftime('%H:%M:%S'))
linkobj.save()
count += 1
size += x.size
return redirect('index')
My models.py:
class Link(models.Model):
id = models.BigAutoField(primary_key=True)
person = models.ForeignKey(Section, on_delete=models.CASCADE)
link = models.ImageField(upload_to= 'images', default = None)
date = models.CharField(max_length=80, default = None)
time = models.CharField(max_length=80,default = None)
Error I am getting is:
AttributeError: 'str' object has no attribute 'file'
Other methods I have tried:
1) linkobj = Link(link = thumb_file, person = Section.objects.get(section_name = section), date = str(datetime.date.today()), time = datetime.datetime.now().strftime('%H:%M:%S'))
RESULT OF ABOVE METHOD:
1)The thumb_file doesnt throw an error, rather it stores nothing in the database
Points I have noticed:
1)The file is being stored properly into the media folder, ie: I can see the pdf getting stored in the media folder
How do I solve this? Thank you
You don't (basically ever) need to initialize a Storage by yourself. This holds especially true since the storage for the field might not be a FileSystemStorage at all, but could e.g. be backed by S3.
Something like
import datetime
import io
from PIL import Image
from django.core.files.base import ContentFile
def convert_image_to_pdf_data(image):
img = Image.open(io.BytesIO(image.read()))
imgc = img.convert("RGB")
pdfdata = io.BytesIO()
imgc.save(pdfdata, format="PDF")
return pdfdata.getvalue()
def formsubmit(request): # submits the form
photos = request.FILES.getlist("photos") # list of UploadedFiles
section = request.POST["section"]
person = Section.objects.get(section_name=section)
date = str(datetime.date.today())
time = datetime.datetime.now().time("%H:%M:%S")
count = 0
size = 0
for image in photos:
pdfdata = convert_image_to_pdf_data(image)
thumb_file = ContentFile(pdfdata, name="photo.pdf")
Link.objects.create(
link=thumb_file,
person=person,
date=date,
time=time,
)
count += 1
size += image.size
return redirect("index")
should be enough here, i.e. using a ContentFile for the converted PDF content; the field should deal with saving it into the storage.
(As an aside, why are date and time stored separately as strings? Your database surely has a datetime type...)
Ok so I found an answer, to be fair I wont accept my own answer as it doesn't provide an exact answer to the question I asked, rather its a different method, so if anyone does know , please do share so that the community can benefit:
My Solution:
Instead of using ContentFile, I used InMemoryUploadedFile, to store the converted pdf and then moved it into the database( in an ImageField)
I am going to be honest, I am not completely sure about why ContentFile was not working, but when going through the documentation I found out that :
The ContentFile class inherits from File, but unlike File it operates on string content (bytes also supported), rather than an actual file.
Any detailed explanation is welcome
My new views.py
from django.core.files.storage import FileSystemStorage
from PIL import Image
import io
from io import BytesIO
from django.core.files.uploadedfile import InMemoryUploadedFile
from django.core.files.base import ContentFile
import sys
def formsubmit(request): #submits the form
docs = request.FILES.getlist('photos')
print(docs)
section = request.POST['section']
for x in docs:
fs = FileSystemStorage()
print(type(x.size))
img = Image.open(io.BytesIO(x.read()))
imgc = img.convert('RGB')
pdfdata = io.BytesIO()
imgc.save(pdfdata,format='PDF')
thumb_file = InMemoryUploadedFile(pdfdata, None, 'photo.pdf', 'pdf',sys.getsizeof(pdfdata), None)
linkobj = Link(link = thumb_file, person = Section.objects.get(section_name = section), date = str(datetime.date.today()), time = datetime.datetime.now().strftime('%H:%M:%S'))
linkobj.save()
count += 1
size += x.size
return redirect('index')
If you have a question, you can leave it in the comments and ill try to answer it, Good luck!!!
I have a multiple select widget, pn.widgets.MultiSelect, and I am trying to get that when its value is selected it updates the options of another pn.widgets.MultiSelect. This is very similar to How do i automatically update a dropdown selection widget when another selection widget is changed? (Python panel pyviz), however, I have not had much success implementing it.
Using the data from the above example, but as a pandas dataframe, when I try to update the options nothing happens:
outerType = pn.widgets.MultiSelect(name="Outer",
value=["Africa"],
options=np.unique(df.cont).tolist())
innerType = pn.widgets.MultiSelect(name="Inner",
options=np.unique(df.loc[df['cont'].isin(outerType.value)].country).tolist())
#pn.depends(outerType, watch=True)
def _update(outerType):
_values = np.unique(df.loc[df['cont'].isin(outerType.value)].country).tolist()
innerType.options = _values
innerType.value = _values
pn.Row(outerType, innerType)
The same happens if I use
#pn.depends(outerType.param.value, watch=True)
def _update(outerType):
However, it does execute as I expect if I use the code below; this uses AnyOldDummy as the argument in def. How can I do this properly please?
outerType = pn.widgets.MultiSelect(name="Outer",
value=["Africa"],
options=np.unique(df.cont).tolist())
innerType = pn.widgets.MultiSelect(name="Inner",
options=np.unique(df.loc[df['cont'].isin(outerType.value)].country).tolist() )
#pn.depends(outerType, watch=True)
def _update(AnyOldDummy):
_values = np.unique(df.loc[df['cont'].isin(outerType.value)].country).tolist()
innerType.options = _values
innerType.value = _values
pn.Row(outerType, innerType)
Data:
import numpy as np
import pandas as pd
import panel as pn
pn.extension()
_countries = {
'Africa': ['Ghana', 'Togo', 'South Africa'],
'Asia' : ['China', 'Thailand', 'Japan'],
'Europe': ['Austria', 'Bulgaria', 'Greece']
}
df = []
for cont in _countries.keys():
co = _countries.get(cont)
df.append(np.c_[np.repeat(cont, len(co)), co])
df = pd.DataFrame(np.vstack(df), columns=['cont', 'country'])
I need help iterating through input to a webapp I'm writing, which looks like:
The users will be inputting several hundred (or thousands) of urls pasted from excel documents, each on a new line like this. Thus far, as you can see, I've created the input page, an output page, and written the code to query the database.
from flask import Flask,render_template, request
from flask_sqlalchemy import SQLAlchemy
from urllib.parse import urlparse
from sqlalchemy.ext.declarative import declarative_base
app = Flask(__name__)
app.config["DEBUG"] = True
app.config["SECRET_KEY"] = "secret_key_here"
db = SQLAlchemy(app)
SQLALCHEMY_DATABASE_URI = db.create_engine(connector_string_here))
app.config[SQLALCHEMY_DATABASE_URI] = SQLALCHEMY_DATABASE_URI
app.config["SQLALCHEMY_POOL_RECYCLE"] = 299
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
db.Model = declarative_base()
class Scrapers(db.Model):
__tablename__ = "Scrapers"
id = db.Column(db.Integer, primary_key = True)
scraper_dom = db.Column(db.String(255))
scraper_id = db.Column(db.String(128))
db.Model.metadata.create_all(SQLALCHEMY_DATABASE_URI)
Session = db.sessionmaker()
Session.configure(bind=SQLALCHEMY_DATABASE_URI)
session = Session()
scrapers = session.query(Scrapers.scraper_dom, Scrapers.scraper_id).all()
#app.route("/", methods=["GET","POST"])
def index():
if request.method == "Get":
return render_template("url_page.html")
else:
return render_template("url_page.html")
#app.route("/submit", methods=["GET","POST"])
def submit():
sites = [request.form["urls"]]
for site in sites:
que = urlparse(site).netloc
return render_template("submit.html", que=que)
#scrapers.filter(Scrapers.scraper_dom.in_(
#next(x.scraper_id for x in scrapers if x.matches(self.fnetloc))
As is apparent, this is incomplete. I've omitted previous attempts at matching the input, as I realized I had issues iterating through the input. At first, I could only get it to print all of the input instead of iterating over it. And now, it prints like this:
Which is just repeating the urlparse(site).netloc for the first line of input, some random number of times. It is parsing correctly and returning the actual value I will need to use later (for each urlparse(site).netloc match scraper_dom and return associated scraper_id). Now, though, I've tried using input() but kept getting errors with [request.form["urls"]] not being an iterable.
Please help, it'd be much appreciated.
Output of sites:
New output with:
que = [urlparse(site).netloc for site in request.form["urls"].split('\n')]
I am pulling from a mysql database table using python3.4. I use the csv module to write the rows of data from the database into .CSV format. Now I am trying toros figure out how I can vet the rows of data by keywords that may show up in the fourth column of data (row[3]). I was thinking of using the re moduleas below but I keep getting errors. Is it not possible to search keywords in a field that is string type and to filter those results if they have those keywords? I keep getting an error. Please help
import re
import csv
userdate = input('What date do you want to look at?')
query = ("SELECT *FROM sometable WHERE timestamp LIKE %s", userdate)
keywords = 'apples', 'bananas', 'cocoa'
# Execute sql Query
cursor.execute(query)
result = cursor.fetchall()
#Reads a CSV file and return it as a list of rows
def read_csv_file(filename):
"""Reads a CSV file and return it as a list of rows."""
for row in csv.reader(open(filename)):
data.append(row)
return data
f = open(path_in + data_file)
read_it = read_csv_file(path_in + data_file)
with open('file.csv', 'wb') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=' ',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
for row in data:
match = re.search('keywords, read_it)
if match:
spamwriter.writerow(row)
I gave up on the regular expressions and used
for row in data:
found_it = row.find(keywords)
if found_it != -1:
spamwriter.writerow(row)
I've been stuck for hours on this csv problem. The following code is run after a form is posted :
fichier_en_lecture = request.FILES['fichier_csv'].read()
nom_du_fichier = request.FILES['fichier_csv'].name
importateur = request.user
traitement_du_fichier(fichier_en_lecture, importateur)
And the "traitement_du_fichier" function goes like this :
def traitement_du_fichier(fichier_en_lecture, nom_du_fichier, importateur):
nouveau_fichier = FichierAdhérents(importateur=importateur, fichier_csv=nom_du_fichier)
nouveau_fichier.save()
import csv
lecteur = csv.reader(fichier_en_lecture, delimiter=",", quotechar='|')
for row in lecteur:
nouvel_adhérent = AdhérentDuFichier()
nouvel_adhérent['fichier_adhérents'] = nouveau_fichier
column_counter = 0
nouvel_adhérent['fédération'] = row[column_counter]
column_counter += 1
nouvel_adhérent['date_première_adhésion'] = row[column_counter]
column_counter += 1
nouvel_adhérent['date_dernière_cotisation'] = row[column_counter]
I get the following error :
iterator should return strings, not int (did you open the file in text mode?)
I've tried to use open() but from what I understand, open() only works with a direct path to the uploaded file. However, I need to do this from memory.
In python 3,
I used:
import csv
from io import StringIO
csvf = StringIO(xls_file.read().decode())
reader = csv.reader(csvf, delimiter=',')
xls_file being the file got from the POST form.
I hope it helps.