import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.model.{HttpMethods, HttpRequest, Uri}
import akka.stream.scaladsl.{Flow, Source}
import akka.stream.{ActorMaterializer, OverflowStrategy}
import spray.json._
import java.util.UUID
import scala.concurrent.duration.DurationInt
import scala.language.postfixOps
import scala.util.{Failure, Success}
object SoftwareRegistry extends App with Formatter {
implicit val system = ActorSystem("NPMRegistry")
implicit val materializer = ActorMaterializer()
case class NPMPackage(name: String)
// reading the packages
val filename = "B:\\Scala\\NPMRegistry\\src\\main\\resources\\packages.txt"
val bufferedSource = scala.io.Source.fromFile(filename)
val listOfPackages: List[NPMPackage] = (for (line <- bufferedSource.getLines) yield {
NPMPackage(line.trim)
}).toList
bufferedSource.close()
// requests
val serverHttpRequests = listOfPackages.map(pkg =>
(HttpRequest(
HttpMethods.GET,
uri = Uri(s"/${pkg.name}")
),
UUID.randomUUID().toString)
)
// source
val sourceList = Source(serverHttpRequests)
val bufferedFlow = Flow[(HttpRequest, String)]
.buffer(10, overflowStrategy = OverflowStrategy.backpressure)
.throttle(1, 3 seconds)
val dd = sourceList
.via(bufferedFlow).async
.via(Http().cachedHostConnectionPoolHttps[String]("registry.npmjs.org"))
.runForeach {
case (Success(response), oId) =>
println(s"$oId $response")
case (Failure(ex), oId) => println(ex)
}
In the above code, I can print the response to the console and I want to know how to consume entity and access the data from the response in a streamed way, not in a future.
Following is the result of the existing code
You basically need to keep you logic within Akka Streams API and not terminating it with runForEach like you did.
A simplified example of this can look like this:
.via(Http().cachedHostConnectionPoolHttps[String]("registry.npmjs.org"))
.flatMapConcat {
case (Success(response), _) => Source.single(response)
case (Failure(_), _) => Source.empty //warning, ignoring errors
}
.map(httpResponse => httpResponse.entity)
.flatMapConcat(e => e.getDataBytes().map(bytes => ???))
.runWith(Sink.ignore)
Instead of runforEach I am flatMapConcatenating to get the HttpRespnse ignoring errors and the context of the request for simplicity of the example. Then mapping to get HttpEntity and then flatMapConcatenating again to get the ByteStrings representing the response body. There could be multiple of those coming form every HttpRequest and that's, what I am guessing you're referring to by "streamed way".
I need some help with the following situation. I have a JSON API with a registration request. The request takes 4 parameters: Standard phone, password, email, and a person parameter where a customer chooses a type. Based on the chosen type, they have to specify one more parameter: Name or age.
When I get an HTTP request, I need to decode it into my request object. Doing it with the code below results in a compilation error of type:
type mismatch;
found : Unit
required: io.circe.Decoder[RegistrationReq]
Any ideas why that can be the case and how to fix it? Thanks in advance!
import cats.effect.Sync
import cats.syntax.either._
import circe.jsonOf
import io.circe.{Decoder, DecodingFailure, HCursor}
import io.circe.generic.semiauto._
import io.circe.refined._
import io.circe.Decoder.Result
import org.http4s.EntityDecoder
trait RegJsonCodec {
import JsonCodec._ //all common encoders and decoders come from there
implicit val RegistrationReqDecoder: Decoder[RegistrationReq] = {
def apply(c: HCursor): Result[RegistrationReq] =
for {
phone <- getPhone(c)
password <- c.downField("password").as[Password]
email <- c.downField("email").as[Email] //Password and Email are refined Strings
person <- getPerson(c)
} yield RegistrationReq(phone, password, email, person)
}
private def getPhone(c: HCursor): Either[DecodingFailure, Long] =
c.downField("phone").as[Long].orElse {
c.downField("phone").as[String].flatMap {
str =>
Either.catchNonFatal(str.toLong)
.leftMap(e => DecodingFailure(e.getMessage, c.history))
}
}.flatMap(phone => Either.right(phone))
private def getPerson(c: HCursor): Either[DecodingFailure, Person] = {
c.downField("person").downField("type").as[String].flatMap { type =>
c.downField("person").downField("name").as[String]
.orElse(c.downField("person").downField("age").as[String]).flatMap { extra =>
Either.catchNonFatal(Person.fromPair(type, extra))
.leftMap(e => DecodingFailure(e.getMessage, c.history))
.flatMap(person => Either.right(person))
}
}
}
implicit def requestEntityDecoder[F[_] : Sync]: EntityDecoder[F, RegistrationReq] = jsonOf[F, RegistrationReq]
}
object RegJsonCodec extends RegJsonCodec
First to say, I'm newcomer in Scala and really need a little help. I need to build a web api, and I'll try to insert one record into database, but have some problems with mapping the entity (db table) into a model (class). I worked with .Net Core Web API (there I used Entity Framework Core, here in Scala use Slick) and try to keep same arhitecture in Scala, but need some more informations, because on the internet I find a lot of versions, and can not choose the best.
As database, MySQL is used.
User.scala
case class User(
id: Int = 0,
userName: String,
firstName: String,
lastName: String
) {
override def equals(that: Any): Boolean = true
}
object User {
implicit object UserFormat extends Format[User] {
def writes(user: User): JsValue = {
val userSeq = Seq(
"id" -> JsNumber(user.id),
"userName" -> JsString(user.userName),
"firstName" -> JsString(user.firstName),
"lastName" -> JsString(user.lastName)
)
JsObject(userSeq)
}
def reads(json: JsValue): JsResult[User] = {
JsSuccess(User(
(json \ "id").as[Int].value,
(json \ "userName").as[String].value,
(json \ "firstName").as[String].value,
(json \ "lastName").as[String].value)
)
}
}
def tupled = (this.apply _).tupled
}
class UserMap #Inject()(protected val dbConfigProvider: DatabaseConfigProvider)(implicit ex: ExecutionContext) {
val dbConfig: DatabaseConfig[JdbcProfile] = dbConfigProvider.get[JdbcProfile]
val db: JdbcBackend#DatabaseDef = dbConfig.db
val dbUsers = TableQuery[UserDef]
def getAll(): Unit = {
val action = sql"SELECT Id, UserName, FirstName, LastName FROM Users".as[(Int, String, String, String)]
return db.run(action)
}
def add(user: User): Future[Seq[User]] = {
dbUsers += user
db.run(dbUsers.result)
}
}
UserDef.scala (which is a mapper of db table / entity)
class UserDef(tag: Tag) extends Table[User](tag, "Users") {
def id = column[Int]("Id", O.PrimaryKey, O.AutoInc)
def userName = column[String]("UserName")
def firstName = column[String]("FirstName")
def lastName = column[String]("LastName")
override def * = (id, userName, firstName, lastName) <> (create, extract)
def create(user: (Int, String, String, String)): User = User(user._1, user._2, user._3, user._4)
def extract(user: User): Option[(Int, String, String, String)] = Some((user.id, user.userName,user.firstName,user.lastName))
}
UsersController.scala
def createUser = Action(parse.json) { implicit request => {
val userJson = request.body
var user = new User(
-1,
(userJson \ "userName").as[String].value,
(userJson \ "firstName").as[String].value,
(userJson \ "lastName").as[String].value
)
var users = TableQuery[UserDef]
Await.result(db.run(DBIO.seq(
users += user,
users.result.map(println))), Duration.Inf
)
Ok(Json.toJson(user))
}
}
How I see the problem:
UserDef is an Entity and must remain clean, only table columns definitions
UserMap is the bridge between User class and UserDef (entity), can be used as a repository with crud methods (getAll(), getById(id), create(user), update(user), delete(id)). This is in same file as User class, but probably must be moved in another.
User class is the model and need to contain only their parameters and writes/reads (Scala specifics)
and now in the controller:
If I try to insert a record into database, with current method, first I need to get all rows from table, and then to add the new record in the list. What happening if I have 3 4mil records in this table? Will get all these rows useless to insert only a new row.
Then, after inserting this new row, I need to return it into client, but how I can get it updated (Id is every time -1, but if I get entire list to see what it contain, I can see the correct id for the newest entity)
thx
Finally, I found a good solution and post it here, maybe somebody need this:
UserMap, for me at least will become UserRepository. There I have CRUD operations and maybe some extra :
def getAll(): Future[Seq[User]] = {
db.run(dbUsers.result)
}
def getById(id: Int): Future[Option[User]] ={
val action = dbUsers.filter(_.id === id).result.headOption
db.run(action)
}
def create(user: User): Future[User] = {
val insertQuery = dbUsers returning dbUsers.map(_.id) into ((x, id) => x.copy(id = id))
val action = insertQuery += user
db.run(action)
}
def update(user: User) {
Try( dbUsers.filter(_.id === user.id).update(user)) match {
case Success(response) => db.run(response)
case Failure(_) => println("An error occurred!")
}
}
def delete(id: Int) {
Try( dbUsers.filter(_.id === id).delete) match {
case Success(response) => db.run(response)
case Failure(_) => println("An error occurred!")
}
}
and UsersController:
def getAll() = Action {
var users = Await.result(usersRepository.getAll(), Duration.Inf)
Ok(Json.toJson(users))
}
def getById(id: Int) = Action { implicit request => {
val user = Await.result(usersRepository.getById(id), Duration.Inf)
Ok(Json.toJson(user))
}
}
def create = Action(parse.json) { implicit request => {
val userJson = request.body
var user = new User(
-1,
(userJson \ "userName").as[String].value,
(userJson \ "firstName").as[String].value,
(userJson \ "lastName").as[String].value
)
var createdUser = Await.result(usersRepository.create((user)), Duration.Inf)
Ok(Json.toJson(createdUser))
}
}
def update(id: Int) = Action(parse.json) { implicit request => {
val userJson = request.body
var user = new User(
(userJson \ "id").as[Int].value,
(userJson \ "userName").as[String].value,
(userJson \ "firstName").as[String].value,
(userJson \ "lastName").as[String].value
)
var updatedUser = usersRepository.update(user)
Ok(Json.toJson(user))
}
}
def delete(id: Int) = Action {
usersRepository.delete(id)
Ok("true")
}
Anyway, I know I have some bad blocks of code there...especially in create & update methods, where convert json to User.
I wanted to give it a try, and here is a full working example of a Play 2.7/Scala 2.13/Slick 4.0.2 REST-API controller bound to a MySQL database.
Since you are starting with Scala, maybe it is a bit overwhelming at first to get eased with Play, Slick, etc...
So here is an humble skeleton (derived from Play-Slick GitHub)
So first, since we want to write an API, here is the conf/routes file:
GET /users controllers.UserController.list()
GET /users/:uuid controllers.UserController.get(uuid: String)
POST /users controllers.UserController.create()
PUT /users controllers.UserController.update()
DELETE /users/:uuid controllers.UserController.delete(uuid: String)
Nothing to fancy here, we just bind routes to functions in the upcoming controller.
Just notice that the 2nd GET and the DELETE expect an UUID as query param, while Json bodies with be used for the POST and PUT.
It would be nice to see the model right now, in app/models/User.scala:
package models
import java.util.UUID
import play.api.libs.json.{Json, OFormat}
case class User(
uuid: UUID,
username: String,
firstName: String,
lastName: String
) {
}
object User {
// this is because defining a companion object shadows the case class function tupled
// see: https://stackoverflow.com/questions/22367092/using-tupled-method-when-companion-object-is-in-class
def tupled = (User.apply _).tupled
// provides implicit json mapping
implicit val format: OFormat[User] = Json.format[User]
}
I used an uuid instead using a numerical id, but basically, it is the same.
Notice that a Json serializer/deserializer can be written in just one line (you don't need to detail it with case classes). I think it is also a good practice to not override it to produce Seq as found on your code, since this serializer will be very usefull when converting objects to Json on the controller.
Now the tupled definition is most likelly a hack (see comment) that will be required later on the DAO...
Next, we need a controller in app/controllers/UserController.scala:
package controllers
import java.util.UUID
import forms.UserForm
import javax.inject.Inject
import play.api.Logger
import play.api.data.Form
import play.api.i18n.I18nSupport
import play.api.libs.json.Json
import play.api.mvc._
import services.UserService
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}
class UserController #Inject()(userService: UserService)
(implicit ec: ExecutionContext) extends InjectedController with I18nSupport {
lazy val logger: Logger = Logger(getClass)
def create: Action[AnyContent] = Action.async { implicit request =>
withFormErrorHandling(UserForm.create, "create failed") { user =>
userService
.create(user)
.map(user => Created(Json.toJson(user)))
}
}
def update: Action[AnyContent] = Action.async { implicit request =>
withFormErrorHandling(UserForm.create, "update failed") { user =>
userService
.update(user)
.map(user => Ok(Json.toJson(user)))
}
}
def list: Action[AnyContent] = Action.async { implicit request =>
userService
.getAll()
.map(users => Ok(Json.toJson(users)))
}
def get(uuid: String): Action[AnyContent] = Action.async { implicit request =>
Try(UUID.fromString(uuid)) match {
case Success(uuid) =>
userService
.get(uuid)
.map(maybeUser => Ok(Json.toJson(maybeUser)))
case Failure(_) => Future.successful(BadRequest(""))
}
}
def delete(uuid: String): Action[AnyContent] = Action.async {
Try(UUID.fromString(uuid)) match {
case Success(uuid) =>
userService
.delete(uuid)
.map(_ => Ok(""))
case Failure(_) => Future.successful(BadRequest(""))
}
}
private def withFormErrorHandling[A](form: Form[A], onFailureMessage: String)
(block: A => Future[Result])
(implicit request: Request[AnyContent]): Future[Result] = {
form.bindFromRequest.fold(
errors => {
Future.successful(BadRequest(errors.errorsAsJson))
}, {
model =>
Try(block(model)) match {
case Failure(e) => {
logger.error(onFailureMessage, e)
Future.successful(InternalServerError)
}
case Success(eventualResult) => eventualResult.recover {
case e =>
logger.error(onFailureMessage, e)
InternalServerError
}
}
})
}
}
So here:
basically, each of our 5 functions referenced from the routes file check input, and then delegate the work to an injected UserService (more on that later)
for the create and update functions, you can see that we use Play Forms that I think is also a good practice. Their role is to validate the incoming Json, and that Marshall it into a User type.
Also, you can see that we use Action.async: Scala offers a very powerfull leverage with Futures so lets use it! Basically by doing so, you ensure that your code is not-blocking, thus easing the IOPS on your hardware.
Finally for the case of GET (one), GET (all), POST and PUT, since we return users, and have a deseralizer, a simple Json.toJson(user) do the work.
Before jumping to service and dao, lets see the form, in app/forms/UserForm.scala:
package forms
import java.util.UUID
import models.User
import play.api.data.Form
import play.api.data.Forms.{mapping, nonEmptyText, _}
object UserForm {
def create: Form[User] = Form(
mapping(
"uuid" -> default(uuid, UUID.randomUUID()),
"username" -> nonEmptyText,
"firstName" -> nonEmptyText,
"lastName" -> nonEmptyText,
)(User.apply)(User.unapply)
)
}
Nothing too fancy here, just as the doc says, although there is just a trick : when no uuid is defined (in the POST case, then we generate one).
Now, the service... not so much required in this very case, but in practice it might be a good thing to have an extra layer (dealing with acls for example), in app/services/UserService.scala:
package services
import java.util.UUID
import dao.UserDAO
import javax.inject.Inject
import models.User
import scala.concurrent.{ExecutionContext, Future}
class UserService #Inject()(dao: UserDAO)(implicit ex: ExecutionContext) {
def get(uuid: UUID): Future[Option[User]] = {
dao.get(uuid)
}
def getAll(): Future[Seq[User]] = {
dao.all()
}
def create(user: User): Future[User] = {
dao.insert(user)
}
def update(user: User): Future[User] = {
dao.update(user)
}
def delete(uuid: UUID): Future[Unit] = {
dao.delete(uuid)
}
}
As you can see, here, it is just a wrapper around the dao, and finnally the dao in app/dao/UserDao.scala:
package dao
import java.util.UUID
import javax.inject.Inject
import models.User
import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider}
import play.db.NamedDatabase
import slick.jdbc.JdbcProfile
import scala.concurrent.{ExecutionContext, Future}
class UserDAO #Inject()(#NamedDatabase("mydb") protected val dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends HasDatabaseConfigProvider[JdbcProfile] {
import profile.api._
private val users = TableQuery[UserTable]
def all(): Future[Seq[User]] = db.run(users.result)
def get(uuid: UUID): Future[Option[User]] = {
db.run(users.filter(_.uuid === uuid).result.headOption)
}
def insert(user: User): Future[User] = {
db.run(users += user).map(_ => user)
}
def update(user: User): Future[User] = {
db.run(users.filter(_.uuid === user.uuid).update(user)).map(_ => user)
}
def delete(uuid: UUID): Future[Unit] = {
db.run(users.filter(_.uuid === uuid).delete).map(_ => ())
}
private class UserTable(tag: Tag) extends Table[User](tag, "users") {
def uuid = column[UUID]("uuid", O.PrimaryKey)
def username = column[String]("username")
def firstName = column[String]("firstName")
def lastName = column[String]("lastName")
def * = (uuid, username, firstName, lastName) <> (User.tupled, User.unapply)
}
}
So, here I have just adapted the code from the official play-slick example, so I guess, I do not have better comment than theirs...
Hope, the whole things helps to get a better picture :)
If something is unclear, feel free to ask!
I want to write a polling client in akka-http that converts all response bodies into a Play JsObject. What I have so far is to code below that uses this library wich should make things simple (i think?). However when I try to run the code below I get the following error:
Error:(26, 56) type mismatch;
found : akka.http.scaladsl.unmarshalling.FromEntityUnmarshaller[play.api.libs.json.JsObject]
(which expands to) akka.http.scaladsl.unmarshalling.Unmarshaller[akka.http.scaladsl.model.HttpEntity,play.api.libs.json.JsObject]
required: akka.http.scaladsl.unmarshalling.Unmarshaller[akka.http.scaladsl.model.HttpResponse,play.api.libs.json.JsObject]
Unmarshaller.byteStringUnmarshaller.mapWithCharset { (data, charset) =>
What do I need to change to get thing working as expected?
import java.util.UUID
import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.model.{HttpEntity, HttpRequest, HttpResponse}
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.Source
import akka.http.scaladsl.unmarshalling.{Unmarshal, Unmarshaller}
import de.heikoseeberger.akkahttpplayjson.PlayJsonSupport._
import play.api.libs.json.{JsObject, Json}
import scala.concurrent.duration._
import scala.util.{Success, Try}
object Main extends App {
implicit val system = ActorSystem("TestSys")
implicit val ec = system.dispatcher
implicit val materializer = ActorMaterializer()
implicit val um:Unmarshaller[HttpResponse, JsObject] = {
Unmarshaller.byteStringUnmarshaller.mapWithCharset { (data, charset) =>
Json.parse(data.toArray).as[JsObject]
}
}
val request = HttpRequest(uri="https://www.google.com/finance/info?q=INDEXDB%3ADAX") -> UUID.randomUUID()
val pool = Http().superPool[UUID]()
val googleFinanceFlow =
Source.tick(0 milliseconds,10000 milliseconds,request)
.via(pool)
.runForeach {
case (Success(response),_) =>
val json = Unmarshal(response).to[JsObject]
println(json.onSuccess{case r => println(r.toString())})
case _ => Json.obj()
}
}
Just delete the explicit implicit (wow, that sounds nice, eh?) definition of the Unmarshaller[HttpResponse, JsObject]. That's not needed, because a suitable unmarshaller is provided by PlayJsonSupport.
I'll appreciate if someone can throw pointers on how to modify the following play framework logging filter (ref. play filters) to achieve the following:
Print and modify the incoming json request body and http headers (e.g., for POST, PUT, & PATCH)
Print and modify the outgoing json response body and http headers
A modification example can be injecting/replacing some token strings in the request and response body, e.g,
REQUEST Json: {'a': 'REPLACE_ME', 'b': 'REPLACE_ME_TOO', 'c':'something'}
RESPONSE Json: {'A': 'REPLACE_ME', 'Bb': 'REPLACE_ME_TOO', 'C':'anything'}
import play.api.Logger
import play.api.mvc._
import play.api.libs.concurrent.Execution.Implicits.defaultContext
object LoggingFilter extends EssentialFilter {
def apply(nextFilter: EssentialAction) = new EssentialAction {
def apply(requestHeader: RequestHeader) = {
val startTime = System.currentTimeMillis
nextFilter(requestHeader).map { result =>
val endTime = System.currentTimeMillis
val requestTime = endTime - startTime
Logger.info(s"${requestHeader.method} ${requestHeader.uri}" +
s" took ${requestTime}ms and returned ${result.header.status}")
result.withHeaders("Request-Time" -> requestTime.toString)
}
}
}
}
So far I have tried the following solution which is clearly ugly and brutal as it contains blocking calls and cryptic operators. I am still not sure how to re-inject the modified request body. (The presented solution incorporates code from 2 and 3.)
import play.api.libs.iteratee._
import play.api.mvc._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.{Await, Future}
import scala.concurrent.duration.Duration
class ReqResFilter extends EssentialFilter {
def apply(next: EssentialAction) = new EssentialAction {
def apply(requestHeader: RequestHeader): Iteratee[Array[Byte], Result] = {
modifyRequest(next, requestHeader).map { result => modifyResponse(result)}
}
}
def bytesToString: Enumeratee[Array[Byte], String] = Enumeratee.map[Array[Byte]] { bytes => new String(bytes)}
def modifyRequest(nextA: EssentialAction, request: RequestHeader): Iteratee[Array[Byte], Result] = {
def step(body: Array[Byte], nextI: Iteratee[Array[Byte], Result])(i: Input[Array[Byte]]):
Iteratee[Array[Byte], Result] = i match {
case Input.EOF =>
val requestBody = new String(body, "utf-8")
val modRequestBody = requestBody.replaceAll("REPLACE_ME", "1224")
println(s"modifyRequest:: Here is the request body ${modRequestBody}")
Iteratee.flatten(nextI.feed(Input.EOF))
case Input.Empty =>
Cont[Array[Byte], Result](step(body, nextI) _)
case Input.El(e) =>
val curBody = Array.concat(body, e)
Cont[Array[Byte], Result](step(curBody, Iteratee.flatten(nextI.feed(Input.El(e)))) _)
}
val nextIteratee: Iteratee[Array[Byte], Result] = nextA(request)
Cont[Array[Byte], Result](i => step(Array(), nextIteratee)(i))
}
def modifyResponse(result: Result): Result = {
val responseBodyFuture: Future[String] = result.body |>>> bytesToString &>> Iteratee.consume[String]()
val responseBody = Await.result(responseBodyFuture, Duration.Inf)
val modResponseBody = responseBody.replaceAll("REPLACE_ME", "1224")
println(s"modifyResponse:: Here is the response body ${modResponseBody}")
new Result(result.header, Enumerator(modResponseBody.getBytes)).withHeaders("New-Header" -> "1234")
}
}
Well since there are no solutions posted here let me add one solution. To make it work, I rewrote step() in modifyRequest() as follows:
def step(body: Array[Byte], nextI: Iteratee[Array[Byte], Result])(i: Input[Array[Byte]]):
Iteratee[Array[Byte], Result] = i match {
case Input.EOF =>
val requestBody = new String(body, "utf-8")
val modRequestBody = requestBody.replaceAll("REPLACE_ME", "1224")
println(s"modifyRequest:: Here is the request body ${modRequestBody}")
Iteratee.flatten(nextI.feed(Input.El(modRequestBody.getBytes)))
case Input.Empty =>
Cont[Array[Byte], Result](step(body, nextI) _)
case Input.El(e) =>
val curBody = Array.concat(body, e)
Cont[Array[Byte], Result](step(curBody, nextI) _)
}
The change is still blocking in nature as it buffers the incoming request. If someone has better solution please do post. Thanks.