Pygame, copying the screen into an array - pygame

How do you get the screen contents into an array in pygame? I have tried this from the documentation:
self.screen.lock()
tmp_frame = pygame.surfarray.array3d(self.screen)
self.screen.unlock()
I have tried all sorts of things, such as using pixelcopy to get a copy of the surface first, but I always get a segmentation fault.
Fatal Python error: (pygame parachute) Segmentation Fault Aborted
(core dumped)
Is it because I am trying to copy from the screen directly?
This is what screen contains:
self.screen = pygl2d.display.set_mode((self.SCREEN_WIDTH, self.SCREEN_HEIGHT), pygame.DOUBLEBUF, depth=24)
And this is the definition of set_mode:
def set_mode(resolution=(0,0), flags=0, depth=0):
flags |= pygame.OPENGL
screen = pygame.display.set_mode(resolution, flags, depth)
init_gl()
return screen
Edit followup:
I also tried copying the screen surface into another surface first with
tmp_surface= self.screen.copy()
But I get
pygame.error: Cannot copy opengl display
So really, I suppose the question is how do you copy this opengl display contents into an array?

To anyone who might experience something similar:
I was not able to find a direct solution, all methods of accessing the hardware accelerated surface resulted in a segmentation error. (array3d,array2d,accessing a reference array pixels3d etc).
However, I was able to figure out a workaround. It appears that you are able to save images with
pygame.image.save(self.screen, 'output.png')
Similarly, you can do
string_image = pygame.image.tostring(self.screen, 'RGB')
temp_surf = pygame.image.fromstring(string_image,(self.SCREEN_WIDTH, self.SCREEN_HEIGHT),'RGB' )
tmp_arr = pygame.surfarray.array3d(temp_surf)
This should get you a numpy array of the screen contents.

import pygame
import numpy as np
import time
from pandas import *
pygame.init()
display = pygame.display.set_mode((350, 350))
x = np.arange(0, 300)
y = np.arange(0, 300)
X, Y = np.meshgrid(x, y)
Z = X + Y
Z = 255*Z/Z.max()
surf = pygame.surfarray.make_surface(Z)
running = True
while running:
for event in pygame.event.get():
if event.type == pygame.QUIT:
running = False
display.blit(surf, (0, 0))
pygame.display.update()
# Convert the window in black color(2D) into a matrix
window_pixel_matrix = pygame.surfarray.array2d(display)
print(window_pixel_matrix)
time.sleep(10)
pygame.quit()
COMMENT :
"pygame.surfarray.array2d()" is what you're looking for I guess.
Of course, you can use "pygame.surfarray.array3d()" function as well.
You can refer to the official website : "https://www.pygame.org/docs/ref/surfarray.html#pygame.surfarray.array2d"

Related

Pygame get coordinates of colour on image [duplicate]

I would like to get an array which would consist of RGBA code of every pixel in the pygame display
I tried this:
for i in range(SCREEN_WIDTH):
for j in range(SCREEN_HEIGHT):
Pixels.append(pygame.Surface.get_at((i, j)))
But I got an error message that Surface.get_at does not work for tuples so I removed one set of bracket and then it told me that Surface.get_at does not work with integers, so I am confused, how can I get the RGBA value of all pixels? Thank you
EDIT, Ok after a comment I post full runable code:
import pygame
pygame.init()
PPM = 15
SCREEN_WIDTH, SCREEN_HEIGHT = 640, 480
pos_X = SCREEN_WIDTH/PPM/3
pos_Y = SCREEN_HEIGHT/PPM/3
screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
FPS = 24
TIME_STEP = 1.0 / FPS
running = True
lead_x = pos_X*PPM
lead_y = pos_Y*PPM
k = 0
Pixels = []
while running:
screen.fill((255, 255, 255, 255))
for event in pygame.event.get():
if event.type == pygame.KEYDOWN and event.key == K_ESCAPE:
running = False
if k == 0:
for i in range(SCREEN_WIDTH):
for j in range(SCREEN_HEIGHT):
Pixels.append(pygame.Surface.get_at((i, j)))
k +=1
pygame.draw.rect(screen, (128,128,128), [lead_x, lead_y,50,50])
pygame.display.update()
pygame.display.flip() # Update the full display Surface to the screen
pygame.time.Clock().tick(FPS)
pygame.quit()
And I got these exact error, nothing less and nothing more:
Exception has occurred: TypeError
descriptor 'get_at' for 'pygame.Surface' objects doesn't apply to 'tuple' object
.get_at is a instance function method (see Method Objects) of pygame.Surface. So it has to be called on an instance of pygame.Surface. screen is the Surface object, which represents the window. So it has to be:
Pixels.append(pygame.Surface.get_at((i, j)))
Pixels.append(screen.get_at((i, j)))
respectively
Pixels.append(pygame.Surface.get_at(screen, (i, j)))
To get all pixels value as bytes, you can use
pygame.Surface.get_buffer(screen).raw

How can I load python wikipedia image in pygame display?

Alright, I am making as title, and here is this is part of codes.
black = (0,0,0)
display = pygame.display.set_mode((1000, 600))
def main():
search_word = "apple inc"
result = ""
main = True
while main:
for event in pygame.event.get():
if event.type == pygame.QUIT:
pygame.quit()
quit()
display.blit(black)
result = wikipedia.page(search_word)
display.blit(result.images[0], (100, 100)
pygame.display.update()
And, I got error:
Traceback (most recent call last):
display.blit(result.images[0], (100, 100)) AttributeError: 'str' object has no attribute 'images'
How can I fix it?
How can I fix it?
You can fix it by passing a Surface object to the blit function, not a string.
Pygame does not automatically download images from the internet for you. You have to download the images yourself; then you can use the functions in pygame.image to create Surface instances.

Getting different accuracies using different caffe classes(98.65 vs 98.1 vs 98.20)

When I train and then test my model using Caffe's command line interface, I get e.g. 98.65% whereas when I myself write code(given below) to calculate accuracy from the same pre-trained model, I get e.g 98.1% using Caffe.Net.
Everything is straight forward and I have no idea what is causing the issue.
I also tried using Caffe.Classifier and its predict method, and yet get another lesser accuracy(i.e. 98.20%!)
Here is the snippet of code I wrote:
import sys
import caffe
import numpy as np
import lmdb
import argparse
from collections import defaultdict
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
import itertools
from sklearn.metrics import roc_curve, auc
import random
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--proto', help='path to the network prototxt file(deploy)', type=str, required=True)
parser.add_argument('--model', help='path to your caffemodel file', type=str, required=True)
parser.add_argument('--mean', help='path to the mean file(.binaryproto)', type=str, required=True)
#group = parser.add_mutually_exclusive_group(required=True)
parser.add_argument('--db_type', help='lmdb or leveldb', type=str, required=True)
parser.add_argument('--db_path', help='path to your lmdb/leveldb dataset', type=str, required=True)
args = parser.parse_args()
predicted_lables=[]
true_labels = []
misclassified =[]
class_names = ['unsafe','safe']
count=0
correct = 0
batch=[]
plabe_ls=[]
batch_size = 50
cropx = 224
cropy = 224
i = 0
multi_crop = False
use_caffe_classifier = True
caffe.set_mode_gpu()
# Extract mean from the mean image file
mean_blobproto_new = caffe.proto.caffe_pb2.BlobProto()
f = open(args.mean, 'rb')
mean_blobproto_new.ParseFromString(f.read())
mean_image = caffe.io.blobproto_to_array(mean_blobproto_new)
f.close()
net = caffe.Classifier(args.proto, args.model,
mean = mean_image[0].mean(1).mean(1),
image_dims = (224, 224))
net1 = caffe.Net(args.proto, args.model, caffe.TEST)
net1.blobs['data'].reshape(batch_size, 3,224, 224)
data_blob_shape = net1.blobs['data'].data.shape
#check and see if its lmdb or leveldb
if(args.db_type.lower() == 'lmdb'):
lmdb_env = lmdb.open(args.db_path)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
for key, value in lmdb_cursor:
count += 1
datum = caffe.proto.caffe_pb2.Datum()
datum.ParseFromString(value)
label = int(datum.label)
image = caffe.io.datum_to_array(datum).astype(np.float32)
#key,image,label
#buffer n image
if(count % 5000 == 0):
print('{0} samples processed so far'.format(count))
if(i < batch_size):
i+=1
inf= key,image,label
batch.append(inf)
#print(key)
if(i >= batch_size):
#process n image
ims=[]
for x in range(len(batch)):
img = batch[x][1]
#img has c,w,h shape! its already gone through transpose and channel swap when it was being saved into lmdb!
#Method III : use center crop just like caffe does in test time
if (use_caffe_classifier != True):
#center crop
c,w,h = img.shape
startx = h//2 - cropx//2
starty = w//2 - cropy//2
img = img[:, startx:startx + cropx, starty:starty + cropy]
#transpose the image so we can subtract from mean
img = img.transpose(2,1,0)
img -= mean_image[0].mean(1).mean(1)
#transpose back to the original state
img = img.transpose(2,1,0)
ims.append(img)
else:
ims.append(img.transpose(2,1,0))
if (use_caffe_classifier != True):
net1.blobs['data'].data[...] = ims[:]
out_1 = net1.forward()
plabe_ls = out_1['pred']
else:
out_1 = net.predict(np.asarray(ims), oversample=multi_crop)
plabe_ls = out_1
plbl = np.asarray(plabe_ls)
plbl = plbl.argmax(axis=1)
for j in range(len(batch)):
if (plbl[j] == batch[j][2]):
correct+=1
else:
misclassified.append(batch[j][0])
predicted_lables.append(plbl[j])
true_labels.append(batch[j][2])
batch.clear()
i = 0
sys.stdout.write("\rAccuracy: %.2f%%" % (100.*correct/count))
sys.stdout.flush()
print(", %i/%i corrects" % (correct, count))
What is causing this difference in accuracies ?
More information :
I am using Python3.5 on windows.
I read images from an lmdb dataset.
The images have 256x256 and center cropped with the size 224x224.
It is finetuned on GoogleNet.
For the Caffe.predict to work well I had to change classify.py
In training, I just use Caffes defaults, such as random crops at training and center crop at test-time.
Changes:
changed line 35 to:
self.transformer.set_transpose(in_, (2, 1, 0))
and line 99 to :
predictions = predictions.reshape((len(predictions) // 10, 10, -1))
1) First off, you need to revert Line 35 (32?) of classify.py: self.transformer.set_transpose(in_, (2, 1, 0)) back to the original
self.transformer.set_transpose(in_, (2, 0, 1)). So it expects HWC and transforms internally to CHW for downstream processing.
2) Run your Classifier branch as it is. You're likely to get a bad result. Please check this. If so, it means the image database is not CWH as you've commented, but actually CHW. After you've confirmed this, make the change to your Classifier branch: ims.append(img.transpose(2,1,0)) to become ims.append(img.transpose(1,2,0)). Re-test your Classifier branch. The result should be 98.2% (goto Step 3) or 98.65% (try Step 4).
3) If your result in Step 3 is 98.2%, also undo your the second change to classify.py. Theoretically, as your images have even height/width so // and / should have no difference. If it does differ or crashes, something is seriously wrong with your image database -- your assumption of the image size is incorrect. You need to check these. They could be off by a pixel or so, and could explain the slight discrepancies in accuracy.
4) If your result in Step 3 is 98.65%, then you need to make changes to the Caffe.Net branch of your code. The database images are CHW, so you need to make the first transpose: img = img.transpose(1,2,0) and the second transpose after mean subtraction to img = img.transpose(2,0,1). Then run your Caffe.Net branch. If you still get 98.1% as before, you should check that mean subtraction is performed correctly by your network.
In Steps (2) and (4), it's possible to get worse results, which means that the problem is likely a difference in mean subtraction for your trained Net vs your expectations in Python code. Check this.
About your 98.2% for the caffe.Classifier:
If you look at lines 78 - 80, the center crop is done along crop_dims , not img_dims. If you further look at line 42 on the caffe.Classifier constructor, the crop_dims are never user-determined. It's determined by the size of the Net's input blobs. Lastly, it you look at line 70, the img_dims are used to resize the images prior to center cropping. So what's happening with your setup is: a) The images are first getting resized to 224 x 224, then uselessly getting center cropped to 224 x 224 ( I assume this is the HxW for your Net ). You obviously will get results poorer than 98.65%. What you need to do is to change the img_dims = (256, 256). That prevents resizing. The crop will be picked up automatically from your Net and you should get your 98.65%.

FiPy Simple Convection

I am trying to understand how FiPy works by working an example, in particular I would like to solve the following simple convection equation with periodic boundary:
$$\partial_t u + \partial_x u = 0$$
If initial data is given by $u(x, 0) = F(x)$, then the analytical solution is $u(x, t) = F(x - t)$. I do get a solution, but it is not correct.
What am I missing? Is there a better resource for understanding FiPy than the documentation? It is very sparse...
Here is my attempt
from fipy import *
import numpy as np
# Generate mesh
nx = 20
dx = 2*np.pi/nx
mesh = PeriodicGrid1D(nx=nx, dx=dx)
# Generate solution object with initial discontinuity
phi = CellVariable(name="solution variable", mesh=mesh)
phiAnalytical = CellVariable(name="analytical value", mesh=mesh)
phi.setValue(1.)
phi.setValue(0., where=x > 1.)
# Define the pde
D = [[-1.]]
eq = TransientTerm() == ConvectionTerm(coeff=D)
# Set discretization so analytical solution is exactly one cell translation
dt = 0.01*dx
steps = 2*int(dx/dt)
# Set the analytical value at the end of simulation
phiAnalytical.setValue(np.roll(phi.value, 1))
for step in range(steps):
eq.solve(var=phi, dt=dt)
print(phi.allclose(phiAnalytical, atol=1e-1))
As addressed on the FiPy mailing list, FiPy is not great at handling convection only PDEs (absent diffusion, pure hyperbolic) as it's missing higher order convection schemes. It is better to use CLAWPACK for this class of problem.
FiPy does have one second order scheme that might help with this problem, the VanLeerConvectionTerm, see an example.
If the VanLeerConvectionTerm is used in the above problem, it does do a better job of preserving the shock.
import numpy as np
import fipy
# Generate mesh
nx = 20
dx = 2*np.pi/nx
mesh = fipy.PeriodicGrid1D(nx=nx, dx=dx)
# Generate solution object with initial discontinuity
phi = fipy.CellVariable(name="solution variable", mesh=mesh)
phiAnalytical = fipy.CellVariable(name="analytical value", mesh=mesh)
phi.setValue(1.)
phi.setValue(0., where=mesh.x > 1.)
# Define the pde
D = [[-1.]]
eq = fipy.TransientTerm() == fipy.VanLeerConvectionTerm(coeff=D)
# Set discretization so analytical solution is exactly one cell translation
dt = 0.01*dx
steps = 2*int(dx/dt)
# Set the analytical value at the end of simulation
phiAnalytical.setValue(np.roll(phi.value, 1))
viewer = fipy.Viewer(phi)
for step in range(steps):
eq.solve(var=phi, dt=dt)
viewer.plot()
raw_input('stopped')
print(phi.allclose(phiAnalytical, atol=1e-1))

Getting transparency to work in glumpy with pygame

I also have been having difficulties getting glumpy to work well with transparent colormaps, and transparency in general.
Below is a simple script that should draw two transparent glumpy Images, one on top of the other. The second simply overwrites the first, so you can't see the one behind the other.
I have experimented with different opengl enable/disable commands, but with no joy! Any help or suggestions would be much appreciated.
from pylab import *
import pygame
from pygame.locals import *
import glumpy
import OpenGL.GL as gl
import OpenGL.GLU as glu
# Set the width and height of the screen [width,height]
size=[256,256]
screen=pygame.display.set_mode(size,OPENGL|DOUBLEBUF)
## I think something here might get this to work, but I've
## experimented turining these off and on. Any suggestions?
gl.glDisable(gl.GL_DEPTH_TEST)
gl.glDisable(gl.GL_LIGHTING)
gl.glEnable(gl.GL_BLEND)
gl.glBlendFunc (gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
a = np.random.rand(256,256).astype('f')
a[0:128,:] = 1.0
b = np.random.rand(256,256).astype('f')
b[:,0:128] = 1.0
# cmaps
aCM = glumpy.colormap.Colormap("Rd", (0., (1.,1.,0.,0.0)), (1., (1.0,0.0,0.0,1.0)))
bCM = glumpy.colormap.Colormap("Bl", (0., (0.,1.0,1.,0.0)), (1., (0.0,0.0,1.0,1.0)))
# glumpy images
ai = glumpy.image.Image(a,colormap=aCM)
bi = glumpy.image.Image(b,colormap=bCM)
print('press q or ESC to quit')
while True :
gl.glPushMatrix()
gl.glLoadIdentity()
gl.glViewport(0, 0, size[0],size[1])
# draw a diagonal line underneath everything
gl.glColor4d(1,0,0,1)
gl.glBegin(gl.GL_LINES)
gl.glVertex3d(0,0,-0.1)
gl.glVertex3d(1,1.0,-0.1)
gl.glEnd()
# draw the glumpy images, with ai beneath bi
ai.draw(x=-1,y=-1,z=0,width=2.0,height=2.0)
bi.draw(x=-.9,y=-.9,z=0.1,width=1.8,height=2.0)
gl.glPopMatrix()
pygame.display.flip()
gl.glClearColor(0.1, 0.1, 0.1, 0.1);
gl.glClear(gl.GL_COLOR_BUFFER_BIT);
for event in pygame.event.get(): # User did something
if event.type == pygame.KEYDOWN:
if event.key in [pygame.K_q, pygame.K_ESCAPE] :
pygame.quit()
sys.exit()
I believe that the problem above was caused by a bug in glumpy. I have outlined a very simple patch that appears to rectify the problem here: https://groups.google.com/forum/#!topic/glumpy-users/UUi4yzl0ktU