I am working on a Spark Application and I want to create a rest API in Django, below is my code
from django.shortcuts import render
from django.http import Http404
from rest_framework.views import APIView
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework import status
from django.http import JsonResponse
from django.core import serializers
from django.conf import settings
import json
from pyspark import SparkContext, SparkConf, SQLContext
sc = SparkContext()
sql = SQLContext(sc)
df = Sql.read.format("jdbc").options(
url = "jdbc:mysql://127.0.0.1:3306/demo",
driver = "com.mysql.cj.jdbc.Driver",
dbtable = "tablename",
user = "xyz",
password = "abc"
).load()
totalrecords = df.count()
# Create your views here.
#api_view(["GET"])
def Demo(self):
try:
a = str(totalrecords)
return JsonResponse(a,safe=False)
except ValueError as e:
return Response(e.args[0],status.HTTP_400_BAD_REQUEST)
I want to know how will I run this code, as I have directly tried "python manage.py runserver" which is not working, so how to run this spark and django with django api and spark-submit with all required spark jar file?
To run this code you have to use spark submit only,
spark-submit --jars mysql.jar manage.py runserver 0.0.0.0:8000
or
spark-submit manage.py runserver
Related
I am trying to connect to Mysql. I have uploaded the corresponding java jar we are using , which is mysql-connector-java-5.1.49.jar , I uploaded to s3 bucket. I am using the following code to access to Mysql and I am failing with the Error
An error occurred while calling o93.getDynamicFrame.
com.mysql.cj.jdbc.Driver
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext, SparkConf
from awsglue.context import GlueContext
from awsglue.job import Job
import time
from pyspark.sql.types import StructType, StructField, IntegerType, StringType
import boto3
import json
sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
spark.conf.set("jars", "s3://xxxxx/jdbc-drivers/mysql-connector-java-5.1.49.jar")
client = boto3.client("secretsmanager" , region_name = "eu-west-1")
get_secret_value_response = client.get_secret_value(SecretId = "etl-1")
secret = get_secret_value_response["SecretString"]
secret=json.loads(secret)
username = secret.get("mysql_username")
password = secret.get("mysql_password")
url = secret.get("mysql_url")
table = secret.get("mysql_table")
connection_mysql_options_source_session = {
"url": url,
"dbtable": table,
"user": username,
"password": password,
"customJdbcDriverS3Path": "s3://xxxxx/jdbc-drivers/mysql-connector-java-5.1.49.jar",
"customJdbcDriverClassName": "com.mysql.cj.jdbc.Driver"}
# Read from JDBC databases with custom driver
df_session = glueContext.create_dynamic_frame.from_options(connection_type="mysql", connection_options=connection_mysql_options_source_session)
df_session.printSchema()
In job details section, I have referred to jar libs:
And I didn't define any connection in the connection section of job properties. I can't figure out why I am getting this error.
The strange thing is that I can connect with Crawler, data catalogue and also direct connection to the same server, but via Notebook & script I can't.
I want to pass a SQL query as parameter in a POST request. Ideally this SQL query will be further passed into MySQL connection, and then it can fetch data back.
Here is what I did:
These are basic modules and settings:
from flask import Flask
from flask_cors import CORS, cross_origin
app = Flask(__name__)
CORS(app)
from app import app
from flaskext.mysql import MySQL
mysql = MySQL()
app.config['MYSQL_DATABASE_USER'] = 'root'
app.config['MYSQL_DATABASE_PASSWORD'] = 'root'
app.config['MYSQL_DATABASE_DB'] = 'hibernate1'
app.config['MYSQL_DATABASE_HOST'] = 'localhost'
mysql.init_app(app)
This is the main.py, where I want to pass a SQL query:
from urllib import response
import pymysql
from app import app
from config import mysql
from flask import jsonify
from flask import flash, request
#app.route('/SQL/<query>', methods=['POST'])
def return_query(query):
conn = mysql.connect()
cursor = conn.cursor(pymysql.cursors.DictCursor)
cursor.execute(query)
Rows = cursor.fetchall()
respone = jsonify(Rows)
return respone
if __name__ == "__main__":
app.run()
This is test.py:
import requests
dictToSend = {"query": "select * from student"}
res = requests.post('http://localhost:5000/SQL', json = dictToSend)
print ('response from server:', res.text)
dictFromServer = res.json()
print(dictFromServer)
However, I got the following errors:
response from server: <!doctype html>
<html lang=en>
<title>404 Not Found</title>
<h1>Not Found</h1>
<p>The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.</p>
It says the requested URL was not found. But the following GET request works fine:
#app.route('/', methods=['GET'])
def return_hello():
return {"data": "hello"}
Could anyone tell me what I missed in the POST request? Thanks.
I have a selenium python automation test, it works fine, now I want to generate Html and JSON reports and have screenshots in the report using pytest. I am new to automation and python so I am not much aware of how its done.
following is my code
test_screenshot.py
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pytest_html
from selenium.common.exceptions import InvalidSessionIdException
def test_Openurl(setup):
driver = setup["driver"]
url = setup["url"]
try:
driver.get(url)
except Exception as e:
print(e.message)
assert driver.current_url == URL
driver.save_screenshot("ss.png")
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
driver.save_screenshot("ss1.png")
driver.close()
conftest.py
import pytest
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
def pytest_addoption(parser):
parser.addoption("--url", action="store", default="https://google.com/")
#pytest.fixture()
def setup(pytestconfig):
s = Service("C:/Users/Yash/Downloads/chromedriver_win32/chromedriver.exe")
driver = webdriver.Chrome(service=s)
driver.maximize_window()
yield {"driver":driver, "url": pytestconfig.getoption("url")}
I ran this using
pytest test_screenshot.py --url "https://www.netflix.com/in/"
Test case is passed. How do I generate HTML and JSON report?
I tried this
pytest -v -s --json-report --json-report-indent=4 --json-report-file=report/report.json --html=report/report.html test_screenshot.py
but got this error
ERROR: usage: pytest [options] [file_or_dir] [file_or_dir] [...]
pytest: error: unrecognized arguments: --json-report --json-report-indent=4 --json-report-file=report/report.json
inifile: None
You need to install these two libraries : https://pypi.org/project/pytest-json-report/ & https://pypi.org/project/pytest-html/
I am reading the tutorial here
But this tutorial uses sqlalchemy as following:
from datetime import datetime, timedelta
import unittest
from app import app, db
from app.models import User, Post
class UserModelCase(unittest.TestCase):
def setUp(self):
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://'
db.create_all()
def tearDown(self):
db.session.remove()
db.drop_all()
in fact, I already wrote a web app that uses from flask_mysqldb import MySQL
the __init__.py in the tutorial looks like this:
from flask import Flask
from config import Config
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
app = Flask(__name__)
app.config.from_object(Config)
db = SQLAlchemy(app)
migrate = Migrate(app, db)
from app import routes, models
I have mysql=MySQL(app) in my own __init__.py
can I replace db by mysql in the first part of the code to test my app?
and what about app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://'?
I haven't use the sqlalchemy,just use flask_mysqldb
Sol:
use testing.mysqld module
here
Reference:
Mock a MySQL database in Python
I'm trying to debug my aiopg connection settings in production environment using aiohttp logger and gunicorn.
I'm trying to log my database credentials:
models.py:
async def init_pg(app):
logger = logging.getLogger('aiohttp.access')
logger.error("POSTGRES_USER = %s" % app['settings'].POSTGRES_USER)
logger.error("POSTGRES_DATABASE = %s" % app['settings'].POSTGRES_DATABASE)
logger.error("POSTGRES_HOST = %s" % app['settings'].POSTGRES_HOST)
logger.error("POSTGRES_PASSWORD = %s" % app['settings'].POSTGRES_PASSWORD)
app['engine'] = await create_engine(
user=app['settings'].POSTGRES_USER,
database=app['settings'].POSTGRES_DATABASE,
host=app['settings'].POSTGRES_HOST,
password=app['settings'].POSTGRES_PASSWORD
)
This doesn't add any output to /var/log/gunicorn/error_log, although it is expected to.
Here is how I start gunicorn:
/usr/local/bin/gunicorn producer.main:app --daemon --bind 0.0.0.0:8002 --worker-class aiohttp.worker.GunicornWebWorker --access-logfile /var/log/gunicorn/access_log --error-logfile /var/log/gunicorn/error_log --env ENVIRONMENT=PRODUCTION --timeout 120
Here is how I create aiohttp app:
main.py:
import logging
import aiohttp_jinja2
import jinja2
from aiojobs.aiohttp import setup as setup_aiojobs
from aiohttp_swagger import setup_swagger
from aiohttp import web, web_middlewares
from . import settings
from .models import init_pg
from .urls import setup_routes
"""
Run either of the following commands from the parent of current directory:
adev runserver producer --livereload
python3 -m producer.main
"""
def create_app():
logging.basicConfig(level=logging.DEBUG)
app = web.Application(middlewares=[
web_middlewares.normalize_path_middleware(append_slash=True),
], client_max_size=2048**2)
app.update(name='producer', settings=settings)
# setup Jinja2 template renderer
aiohttp_jinja2.setup(app, loader=jinja2.PackageLoader('producer', 'templates'))
# create db connection on startup, shutdown on exit
app.on_startup.append(init_pg)
# app.on_cleanup.append(close_pg)
# setup views and routes
setup_routes(app)
# setup middlewares
# setup_middlewares(app)
# setup aiojobs scheduler
setup_aiojobs(app)
# setup swagger documentation
setup_swagger(app, swagger_url="api/doc")
return app
if __name__ == '__main__':
app = create_app()
web.run_app(app, host=app['settings'].HOST, port=app['settings'].PORT)