Upload pdf in HTML and Deserialize json file - html

I'm trying to upload a file in html and then send it to my database via restangular.
My frontend is a combination of angular with typescript but the upload is a form.
<form enctype="multipart/form-data">
<fieldset class="form-group" ng-repeat="field in $ctrl.metadata.fields">
<label ng-if="field.inputType !== 'hidden'" for="{{field.propertyKey}}"><strong>{{field.name}}</strong></label>
<input ng-if="field.inputType !== 'select' && field.inputType !== 'file'" class="form-control" type="{{field.inputType}}" name="{{field.propertyKey}}" id="{{field.propertyKey}}" ng-model="$ctrl.data[field.propertyKey]"/>
<input ng-if="field.inputType === 'file'" class="form-control" ngf-select type="{{field.inputType}}" name="{{field.propertyKey}}" id="{{field.propertyKey}}" ng-model="$ctrl.data[field.propertyKey]"/>
<sp-dropdown ng-if="field.inputType === 'select'" value="$ctrl.data[field.propertyKey]" api-domain="field.linkedObjectApiDomain" linked-object-name="field.linkedObjectName"></sp-dropdown>
</fieldset>
<button class="btn btn-primary" ng-click="$ctrl.save({item: $ctrl.data})">Save</button>
<button ng-if="$ctrl.metadata.buttons.hasOpen" class="btn btn-primary" ng-click="$ctrl.open()">Open</button>
</form>
I did the databinding of the file with ng-file-upload.
Upon saving we enter this typescript save method.
public save(item: any): any {
console.log("item to save is ", item);
console.log("rapport is ", item["rapport"]);
if (item.id === undefined) {
this.restService.save(this.metadata.apiDomain, item).then((addedItem: any) => {
toastr.success(`${addedItem.naam} successfully created.`, `Overzicht Dossiers Created`);
});
} else {
this.restService.update(this.metadata.apiDomain, item).then((updatedItem: any) => {
toastr.success(`${updatedItem.naam} successfully updated.`, `Overzicht Dossiers Updated`);
});
}
}
The second log with the file gives the json:
lastModified:1463402787393
lastModifiedDate:Mon May 16 2016 14:46:27 GMT+0200 (Romance (zomertijd))
name:"Rapport.pdf"
size:83605
type:"application/pdf"
upload:Promise
webkitRelativePath:""
__proto__:File
On server side I'm using a spring project which I didn't set up myself but the important files are my class which should store this data
Dossier
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package be.ugent.lca.data.entities;
import be.ugent.sherpa.entity.BaseEntity;
import java.sql.Date;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.Lob;
import javax.persistence.ManyToOne;
import javax.persistence.OneToOne;
/**
*
* #author Sam
*/
#Entity
//#JsonDeserialize(using = DossierDeserializer.class)
//#JsonSerialize(using = DossierSerializer.class)
public class Dossier extends BaseEntity{
private String externDossierNr;
private String internDossierNr;
private Date datum;
private Boolean doc;
private Date refKlantDatum;
private String refKlantVerwijzing;
private String verantw;
#OneToOne(fetch=FetchType.LAZY, mappedBy="dossier")
private Offerte offerte;
private String status;
#ManyToOne(fetch=FetchType.EAGER)
#JoinColumn(name = "persoon")
private Persoon persoon;
#ManyToOne(fetch = FetchType.LAZY)
#JoinColumn(name = "OrganisatieFirma")
private OrganisatieFirma organisatieFirma;
#ManyToOne(fetch = FetchType.LAZY)
#JoinColumn(name = "OrganisatieIntern")
private OrganisatieIntern organisatieIntern;
#Lob
#Column(length=100000)
private byte[] rapport;
public Offerte getOfferte() {
return offerte;
}
public void setOfferte(Offerte offerte) {
this.offerte = offerte;
}
public byte[] getRapport() {
return rapport;
}
public void setRapport(byte[] rapport) {
this.rapport = rapport;
}
public OrganisatieFirma getOrganisatieFirma() {
return organisatieFirma;
}
public String getExternDossierNr() {
return externDossierNr;
}
public void setExternDossierNr(String externDossierNr) {
this.externDossierNr = externDossierNr;
}
public String getInternDossierNr() {
return internDossierNr;
}
public void setInternDossierNr(String internDossierNr) {
this.internDossierNr = internDossierNr;
}
public void setOrganisatieFirma(OrganisatieFirma organisatieFirma) {
this.organisatieFirma = organisatieFirma;
}
public OrganisatieIntern getOrganisatieIntern() {
return organisatieIntern;
}
public void setOrganisatieIntern(OrganisatieIntern organisatieIntern) {
this.organisatieIntern = organisatieIntern;
}
public Persoon getPersoon() {
return persoon;
}
public void setPersoon(Persoon persoon) {
this.persoon = persoon;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public Date getDatum() {
return datum;
}
public void setDatum(Date datum) {
this.datum = datum;
}
public Date getRefKlantDatum() {
return refKlantDatum;
}
public void setRefKlantDatum(Date refKlantDatum) {
this.refKlantDatum = refKlantDatum;
}
public String getRefKlantVerwijzing() {
return refKlantVerwijzing;
}
public void setRefKlantVerwijzing(String refKlantVerwijzing) {
this.refKlantVerwijzing = refKlantVerwijzing;
}
public String getVerantw() {
return verantw;
}
public void setVerantw(String verantw) {
this.verantw = verantw;
}
public Boolean getDoc() {
return doc;
}
public void setDoc(Boolean doc) {
this.doc = doc;
}
}
and my repository for this class
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package be.ugent.lca.data.repository;
import be.ugent.lca.data.entities.Dossier;
import be.ugent.lca.data.query.DossierQuery;
import be.ugent.sherpa.repository.RestRepository;
import org.springframework.data.rest.core.annotation.RepositoryRestResource;
/**
*
* #author Sam
*/
#RepositoryRestResource(collectionResourceRel = "dossiers", path = "dossiers")
public interface DossierRepository extends RestRepository<Dossier, DossierQuery<?>>{
}
When trying to save a file to my database the server gives this exception
Caused by: com.fasterxml.jackson.databind.JsonMappingException: Can not deserialize instance of byte[] out of START_OBJECT token
This led me to believe that I have to write my own deserializer for Dossier
Thus:
package be.ugent.lca.data.entities.deserializers;
import be.ugent.lca.data.entities.Dossier;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.ObjectCodec;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
public class DossierDeserializer extends JsonDeserializer {
#Override
public Dossier deserialize(JsonParser jsonParser,
DeserializationContext deserializationContext) throws IOException {
ObjectCodec oc = jsonParser.getCodec();
JsonNode root = oc.readTree(jsonParser);
Dossier dossier = new Dossier();
dossier.setExternDossierNr(root.get("externDossierNr").asText());
dossier.setInternDossierNr(root.get("internDossierNr").asText());
return dossier;
}
}
But my problem is that I don't know how exactly to deserialize the file json, since writing out root.get("rapport") gives back an empty string.
Any help would be much appreciated.

I've worked out the file upload.
First of all I split the file upload from the rest of my data so I won't have to rewrite the automatic deserialization for everything that does work.
this.restService.save(this.metadata.apiDomain, item).then((addedItem: any) => {
toastr.success(`${addedItem.naam} successfully created.`, `Overzicht Dossiers Created`);
console.log("created item ", addedItem);
var fd = new FormData();
fd.append("rapport", item["rapport"]);
this.restService.one('dossiers/' + addedItem.id + '/rapport').withHttpConfig({transformRequest: angular.identity}).customPOST(fd, '', undefined, {'Content-Type': undefined}).then(
(addedDossier: any) => {
console.log("posted dossier ", addedDossier);
}
);
});
In the callback of my normal save I do the custom post to dossiers/{id}/rapport for this I need a custom controller.
#BasePathAwareController
#RequestMapping("/dossiers/{id}")
#ExposesResourceFor(Dossier.class)
public class DossierController {
The BasePathAwawareController makes sure that all automatically generated paths that you don't override keep existing.
#Autowired
private DossierRepository dossierRepository;
With this I inject my repository to connect to my database.
#RequestMapping(path = "/rapport", method = RequestMethod.POST)//,headers = "content-type=multipart/form-data")
public #ResponseBody String postRapport(#PathVariable("id") Long id,#RequestParam("rapport") MultipartFile file) {
String name = "rapport";
System.out.println("Entered custom file upload with id " + id);
if (!file.isEmpty()) {
try {
byte[] bytes = file.getBytes();
Dossier dossier = dossierRepository.findOne(id);
dossier.setRapport(bytes);
dossierRepository.save(dossier);
return "You successfully uploaded " + name + " into " + name + "-uploaded !";
} catch (Exception e) {
return "You failed to upload " + name + " => " + e.getMessage();
}
} else {
return "You failed to upload " + name + " because the file was empty.";
}
}
Like this I'm able to successfully upload my file.

Related

Reading Very Complex JSON using Spring Batch

My objective is to read a very complex JSON using Spring Batch. Below is the sample JSON.
{
"order-info" : {
"order-number" : "Test-Order-1"
"order-items" : [
{
"item-id" : "4144769310"
"categories" : [
"ABCD",
"DEF"
],
"item_imag" : "http:// "
"attributes: {
"color" : "red"
},
"dimensions" : {
},
"vendor" : "abcd",
},
{
"item-id" : "88888",
"categories" : [
"ABCD",
"DEF"
],
.......
I understand that I would need to create a Custom ItemReader to parse this JSON.
Kindly provide me some pointers. I am really clueless.
I am now not using CustomItemReader. I am using Java POJOs. My JsonItemReader is as per below:
#Bean
public JsonItemReader<Trade> jsonItemReader() {
ObjectMapper objectMapper = new ObjectMapper();
JacksonJsonObjectReader<Trade> jsonObjectReader =
new JacksonJsonObjectReader<>(Trade.class);
jsonObjectReader.setMapper(objectMapper);
return new JsonItemReaderBuilder<Trade>()
.jsonObjectReader(jsonObjectReader)
.resource(new ClassPathResource("search_data_1.json"))
.name("tradeJsonItemReader")
.build();
}
The exception which I now get is :
java.lang.IllegalStateException: The Json input stream must start with an array of Json objects
From similar posts in this forum I understand that I need to use JsonObjectReader. "You can implement it to read a single json object and use it with the JsonItemReader (either at construction time or using the setter)".
How can I do this either # construction time or using setter? Please share some code snippet for the same.
The delegate of MultiResourceItemReader should still be a JsonItemReader. You just need to use a custom JsonObjectReader with the JsonItemReader instead of JacksonJsonObjectReader. Visually, this would be: MultiResourceItemReader -- delegates to --> JsonItemReader -- uses --> your custom JsonObjectReader.
Could you please share a code snippet for the above?
JacksonJsonItemReader is meant to parse from a root node that is already and array node, so it expects your json to start with '['.
If you desire to parse a complex object - in this case, one that have many parent nodes/properties before it gets to the array - you should write a reader. It is really simple to do it and you can follow JacksonJsonObjectReader's structure. Here follows and example of a generic reader for complex object with respective unit tests.
The unit test
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.BlockJUnit4ClassRunner;
import org.springframework.core.io.ByteArrayResource;
import com.example.batch_experiment.dataset.Dataset;
import com.example.batch_experiment.dataset.GenericJsonObjectReader;
import com.example.batch_experiment.json.InvalidArrayNodeException;
import com.example.batch_experiment.json.UnreachableNodeException;
import com.fasterxml.jackson.databind.ObjectMapper;
#RunWith(BlockJUnit4ClassRunner.class)
public class GenericJsonObjectReaderTest {
GenericJsonObjectReader<Dataset> reader;
#Before
public void setUp() {
reader = new GenericJsonObjectReader<Dataset>(Dataset.class, "results");
}
#Test
public void shouldRead_ResultAsRootNode() throws Exception {
reader.open(new ByteArrayResource("{\"result\":{\"results\":[{\"id\":\"a\"}]}}".getBytes()) {});
Assert.assertTrue(reader.getDatasetNode().isArray());
Assert.assertFalse(reader.getDatasetNode().isEmpty());
}
#Test
public void shouldIgnoreUnknownProperty() throws Exception {
String jsonStr = "{\"result\":{\"results\":[{\"id\":\"a\", \"aDifferrentProperty\":0}]}}";
reader.open(new ByteArrayResource(jsonStr.getBytes()) {});
Assert.assertTrue(reader.getDatasetNode().isArray());
Assert.assertFalse(reader.getDatasetNode().isEmpty());
}
#Test
public void shouldIgnoreNullWithoutQuotes() throws Exception {
String jsonStr = "{\"result\":{\"results\":[{\"id\":\"a\",\"name\":null}]}}";
try {
reader.open(new ByteArrayResource(jsonStr.getBytes()) {});
Assert.assertTrue(reader.getDatasetNode().isArray());
Assert.assertFalse(reader.getDatasetNode().isEmpty());
} catch (Exception e) {
Assert.fail(e.getMessage());
}
}
#Test
public void shouldThrowException_OnNullNode() throws Exception {
boolean exceptionThrown = false;
try {
reader.open(new ByteArrayResource("{}".getBytes()) {});
} catch (UnreachableNodeException e) {
exceptionThrown = true;
}
Assert.assertTrue(exceptionThrown);
}
#Test
public void shouldThrowException_OnNotArrayNode() throws Exception {
boolean exceptionThrown = false;
try {
reader.open(new ByteArrayResource("{\"result\":{\"results\":{}}}".getBytes()) {});
} catch (InvalidArrayNodeException e) {
exceptionThrown = true;
}
Assert.assertTrue(exceptionThrown);
}
#Test
public void shouldReadObjectValue() {
try {
reader.setJsonParser(new ObjectMapper().createParser("{\"id\":\"a\"}"));
Dataset dataset = reader.read();
Assert.assertNotNull(dataset);
Assert.assertEquals("a", dataset.getId());
} catch (Exception e) {
Assert.fail(e.getMessage());
}
}
}
And the reader:
import java.io.IOException;
import java.io.InputStream;
import java.util.logging.Logger;
import org.springframework.batch.item.ParseException;
import org.springframework.batch.item.json.JsonObjectReader;
import org.springframework.core.io.Resource;
import com.example.batch_experiment.json.InvalidArrayNodeException;
import com.example.batch_experiment.json.UnreachableNodeException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
/*
* This class follows the structure and functions similar to JacksonJsonObjectReader, with
* the difference that it expects a object as root node, instead of an array.
*/
public class GenericJsonObjectReader<T> implements JsonObjectReader<T>{
Logger logger = Logger.getLogger(GenericJsonObjectReader.class.getName());
ObjectMapper mapper = new ObjectMapper();
private JsonParser jsonParser;
private InputStream inputStream;
private ArrayNode targetNode;
private Class<T> targetType;
private String targetPath;
public GenericJsonObjectReader(Class<T> targetType, String targetPath) {
super();
this.targetType = targetType;
this.targetPath = targetPath;
}
public JsonParser getJsonParser() {
return jsonParser;
}
public void setJsonParser(JsonParser jsonParser) {
this.jsonParser = jsonParser;
}
public ArrayNode getDatasetNode() {
return targetNode;
}
/*
* JsonObjectReader interface has an empty default method and must be implemented in this case to set
* the mapper and the parser
*/
#Override
public void open(Resource resource) throws Exception {
logger.info("Opening json object reader");
this.inputStream = resource.getInputStream();
JsonNode jsonNode = this.mapper.readTree(this.inputStream).findPath(targetPath);
if (!jsonNode.isMissingNode()) {
this.jsonParser = startArrayParser(jsonNode);
logger.info("Reader open with parser reference: " + this.jsonParser);
this.targetNode = (ArrayNode) jsonNode; // for testing purposes
} else {
logger.severe("Couldn't read target node " + this.targetPath);
throw new UnreachableNodeException();
}
}
#Override
public T read() throws Exception {
try {
if (this.jsonParser.nextToken() == JsonToken.START_OBJECT) {
T result = this.mapper.readValue(this.jsonParser, this.targetType);
logger.info("Object read: " + result.hashCode());
return result;
}
} catch (IOException e) {
throw new ParseException("Unable to read next JSON object", e);
}
return null;
}
/**
* Creates a new parser from an array node
*/
private JsonParser startArrayParser(JsonNode jsonArrayNode) throws IOException {
JsonParser jsonParser = this.mapper.getFactory().createParser(jsonArrayNode.toString());
if (jsonParser.nextToken() == JsonToken.START_ARRAY) {
return jsonParser;
} else {
throw new InvalidArrayNodeException();
}
}
#Override
public void close() throws Exception {
this.inputStream.close();
this.jsonParser.close();
}
}

Keep tags order using SnakeYAML

I'm trying to translate yaml files to json, but the translation re-orders the tags...
Ex, YAML source:
zzzz:
b: 456
a: dfff
aa:
s10: "dddz"
s3: eeee
bbb:
- b1
- a2
snakeYAML produces:
{
"aa": {
"s3": "eeee",
"s10":"dddz"
},
"bbb":[
"b1",
"a2"
],
"zzzz": {
"a": "dfff",
"b":456
}
}
Create following class in your code, this is a tweaked version from the SnakeYAML source that uses LinkedHashMap and LinkedHashSet that keep the insertion order instead of TreeMap and TreeSet which auto-sort them.
import java.beans.FeatureDescriptor;
import java.beans.IntrospectionException;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.*;
import org.yaml.snakeyaml.error.YAMLException;
import org.yaml.snakeyaml.introspector.*;
import org.yaml.snakeyaml.util.PlatformFeatureDetector;
public class CustomPropertyUtils extends PropertyUtils {
private final Map<Class<?>, Map<String, Property>> propertiesCache = new HashMap<Class<?>, Map<String, Property>>();
private final Map<Class<?>, Set<Property>> readableProperties = new HashMap<Class<?>, Set<Property>>();
private BeanAccess beanAccess = BeanAccess.DEFAULT;
private boolean allowReadOnlyProperties = false;
private boolean skipMissingProperties = false;
private PlatformFeatureDetector platformFeatureDetector;
public CustomPropertyUtils() {
this(new PlatformFeatureDetector());
}
CustomPropertyUtils(PlatformFeatureDetector platformFeatureDetector) {
this.platformFeatureDetector = platformFeatureDetector;
/*
* Android lacks much of java.beans (including the Introspector class, used here), because java.beans classes tend to rely on java.awt, which isn't
* supported in the Android SDK. That means we have to fall back on FIELD access only when SnakeYAML is running on the Android Runtime.
*/
if (platformFeatureDetector.isRunningOnAndroid()) {
beanAccess = BeanAccess.FIELD;
}
}
protected Map<String, Property> getPropertiesMap(Class<?> type, BeanAccess bAccess) {
if (propertiesCache.containsKey(type)) {
return propertiesCache.get(type);
}
Map<String, Property> properties = new LinkedHashMap<String, Property>();
boolean inaccessableFieldsExist = false;
switch (bAccess) {
case FIELD:
for (Class<?> c = type; c != null; c = c.getSuperclass()) {
for (Field field : c.getDeclaredFields()) {
int modifiers = field.getModifiers();
if (!Modifier.isStatic(modifiers) && !Modifier.isTransient(modifiers)
&& !properties.containsKey(field.getName())) {
properties.put(field.getName(), new FieldProperty(field));
}
}
}
break;
default:
// add JavaBean properties
try {
for (PropertyDescriptor property : Introspector.getBeanInfo(type)
.getPropertyDescriptors()) {
Method readMethod = property.getReadMethod();
if ((readMethod == null || !readMethod.getName().equals("getClass"))
&& !isTransient(property)) {
properties.put(property.getName(), new MethodProperty(property));
}
}
} catch (IntrospectionException e) {
throw new YAMLException(e);
}
// add public fields
for (Class<?> c = type; c != null; c = c.getSuperclass()) {
for (Field field : c.getDeclaredFields()) {
int modifiers = field.getModifiers();
if (!Modifier.isStatic(modifiers) && !Modifier.isTransient(modifiers)) {
if (Modifier.isPublic(modifiers)) {
properties.put(field.getName(), new FieldProperty(field));
} else {
inaccessableFieldsExist = true;
}
}
}
}
break;
}
if (properties.isEmpty() && inaccessableFieldsExist) {
throw new YAMLException("No JavaBean properties found in " + type.getName());
}
System.out.println(properties);
propertiesCache.put(type, properties);
return properties;
}
private static final String TRANSIENT = "transient";
private boolean isTransient(FeatureDescriptor fd) {
return Boolean.TRUE.equals(fd.getValue(TRANSIENT));
}
public Set<Property> getProperties(Class<? extends Object> type) {
return getProperties(type, beanAccess);
}
public Set<Property> getProperties(Class<? extends Object> type, BeanAccess bAccess) {
if (readableProperties.containsKey(type)) {
return readableProperties.get(type);
}
Set<Property> properties = createPropertySet(type, bAccess);
readableProperties.put(type, properties);
return properties;
}
protected Set<Property> createPropertySet(Class<? extends Object> type, BeanAccess bAccess) {
Set<Property> properties = new LinkedHashSet<>();
Collection<Property> props = getPropertiesMap(type, bAccess).values();
for (Property property : props) {
if (property.isReadable() && (allowReadOnlyProperties || property.isWritable())) {
properties.add(property);
}
}
return properties;
}
public Property getProperty(Class<? extends Object> type, String name) {
return getProperty(type, name, beanAccess);
}
public Property getProperty(Class<? extends Object> type, String name, BeanAccess bAccess) {
Map<String, Property> properties = getPropertiesMap(type, bAccess);
Property property = properties.get(name);
if (property == null && skipMissingProperties) {
property = new MissingProperty(name);
}
if (property == null) {
throw new YAMLException(
"Unable to find property '" + name + "' on class: " + type.getName());
}
return property;
}
public void setBeanAccess(BeanAccess beanAccess) {
if (platformFeatureDetector.isRunningOnAndroid() && beanAccess != BeanAccess.FIELD) {
throw new IllegalArgumentException(
"JVM is Android - only BeanAccess.FIELD is available");
}
if (this.beanAccess != beanAccess) {
this.beanAccess = beanAccess;
propertiesCache.clear();
readableProperties.clear();
}
}
public void setAllowReadOnlyProperties(boolean allowReadOnlyProperties) {
if (this.allowReadOnlyProperties != allowReadOnlyProperties) {
this.allowReadOnlyProperties = allowReadOnlyProperties;
readableProperties.clear();
}
}
public boolean isAllowReadOnlyProperties() {
return allowReadOnlyProperties;
}
/**
* Skip properties that are missing during deserialization of YAML to a Java
* object. The default is false.
*
* #param skipMissingProperties
* true if missing properties should be skipped, false otherwise.
*/
public void setSkipMissingProperties(boolean skipMissingProperties) {
if (this.skipMissingProperties != skipMissingProperties) {
this.skipMissingProperties = skipMissingProperties;
readableProperties.clear();
}
}
public boolean isSkipMissingProperties() {
return skipMissingProperties;
}
}
Then, create your Yaml instance like this:
DumperOptions options = new DumperOptions();
CustomPropertyUtils customPropertyUtils = new CustomPropertyUtils();
Representer customRepresenter = new Representer();
customRepresenter.setPropertyUtils(customPropertyUtils);
Yaml yaml = new Yaml(customRepresenter, options);
Profit!
The keeping of property order depends on java implementation and is not guaranteed.
In order to control the yaml generation you will need to implement your CustomRepresenter overriding the getProperties , see the example below:
package io.github.rockitconsulting.test.rockitizer.configuration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.introspector.Property;
import org.yaml.snakeyaml.representer.Representer;
/**
* Custom implementation of {#link Representer} and {#link Comparator}
* to keep the needed order of javabean properties of model classes,
* thus generating the understandable yaml
*
*/
public class ConfigurationModelRepresenter extends Representer {
public ConfigurationModelRepresenter() {
super();
}
public ConfigurationModelRepresenter(DumperOptions options) {
super(options);
}
protected Set<Property> getProperties(Class<? extends Object> type) {
Set<Property> propertySet;
if (typeDefinitions.containsKey(type)) {
propertySet = typeDefinitions.get(type).getProperties();
}
propertySet = getPropertyUtils().getProperties(type);
List<Property> propsList = new ArrayList<>(propertySet);
Collections.sort(propsList, new BeanPropertyComparator());
return new LinkedHashSet<>(propsList);
}
class BeanPropertyComparator implements Comparator<Property> {
public int compare(Property p1, Property p2) {
if (p1.getType().getCanonicalName().contains("util") && !p2.getType().getCanonicalName().contains("util")) {
return 1;
} else if(p2.getName().endsWith("Name")|| p2.getName().equalsIgnoreCase("name")) {
return 1;
} else {
return -1;
} // returning 0 would merge keys
}
}
}
The below snippet shows the usage of the newly created class to generate the yaml structure:
DumperOptions options = new DumperOptions();
ConfigurationModelRepresenter customRepresenter = new ConfigurationModelRepresenter();
Yaml yaml = new Yaml(customRepresenter, options);
StringWriter writer = new StringWriter();
yaml.dump(suite, writer);
FileWriter fw = new FileWriter(rootPathTestSrc + "config_gen.yml", false);
fw.write(writer.toString());
fw.close();
This approach is much cleaner, comparing to the one suggested above.

how to save apache spark schema output in mysql database

Can anyone please tell me if there is any way in apache spark to store a JavaRDD on mysql database? I am taking input from 2 csv files and then after doing join operations on their contents I need to save the output(the output JavaRDD) in the mysql database. I am already able to save the output successfully on hdfs but I am not finding any information related to apache Spark-MYSQL connection. Below I am posting the code for spark sql. This might serve as a reference to those who are looking for an example for spark-sql.
package attempt1;
import java.io.Serializable;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.api.java.JavaSQLContext;
import org.apache.spark.sql.api.java.JavaSchemaRDD;
import org.apache.spark.sql.api.java.Row;
public class Spark_Mysql {
#SuppressWarnings("serial")
public static class CompleteSample implements Serializable {
private String ASSETNUM;
private String ASSETTAG;
private String CALNUM;
public String getASSETNUM() {
return ASSETNUM;
}
public void setASSETNUM(String aSSETNUM) {
ASSETNUM = aSSETNUM;
}
public String getASSETTAG() {
return ASSETTAG;
}
public void setASSETTAG(String aSSETTAG) {
ASSETTAG = aSSETTAG;
}
public String getCALNUM() {
return CALNUM;
}
public void setCALNUM(String cALNUM) {
CALNUM = cALNUM;
}
}
#SuppressWarnings("serial")
public static class ExtendedSample implements Serializable {
private String ASSETNUM;
private String CHANGEBY;
private String CHANGEDATE;
public String getASSETNUM() {
return ASSETNUM;
}
public void setASSETNUM(String aSSETNUM) {
ASSETNUM = aSSETNUM;
}
public String getCHANGEBY() {
return CHANGEBY;
}
public void setCHANGEBY(String cHANGEBY) {
CHANGEBY = cHANGEBY;
}
public String getCHANGEDATE() {
return CHANGEDATE;
}
public void setCHANGEDATE(String cHANGEDATE) {
CHANGEDATE = cHANGEDATE;
}
}
#SuppressWarnings("serial")
public static void main(String[] args) throws Exception {
JavaSparkContext ctx = new JavaSparkContext("local[2]", "JavaSparkSQL");
JavaSQLContext sqlCtx = new JavaSQLContext(ctx);
JavaRDD<CompleteSample> cs = ctx.textFile("C:/Users/cyg_server/Documents/bigDataExample/AssetsImportCompleteSample.csv").map(
new Function<String, CompleteSample>() {
public CompleteSample call(String line) throws Exception {
String[] parts = line.split(",");
CompleteSample cs = new CompleteSample();
cs.setASSETNUM(parts[0]);
cs.setASSETTAG(parts[1]);
cs.setCALNUM(parts[2]);
return cs;
}
});
JavaRDD<ExtendedSample> es = ctx.textFile("C:/Users/cyg_server/Documents/bigDataExample/AssetsImportExtendedSample.csv").map(
new Function<String, ExtendedSample>() {
public ExtendedSample call(String line) throws Exception {
String[] parts = line.split(",");
ExtendedSample es = new ExtendedSample();
es.setASSETNUM(parts[0]);
es.setCHANGEBY(parts[1]);
es.setCHANGEDATE(parts[2]);
return es;
}
});
JavaSchemaRDD complete = sqlCtx.applySchema(cs, CompleteSample.class);
complete.registerAsTable("cs");
JavaSchemaRDD extended = sqlCtx.applySchema(es, ExtendedSample.class);
extended.registerAsTable("es");
JavaSchemaRDD fs= sqlCtx.sql("SELECT cs.ASSETTAG, cs.CALNUM, es.CHANGEBY, es.CHANGEDATE FROM cs INNER JOIN es ON cs.ASSETNUM=es.ASSETNUM;");
JavaRDD<String> result = fs.map(new Function<Row, String>() {
public String call(Row row) {
return row.getString(0);
}
});
result.saveAsTextFile("hdfs://path/to/hdfs/dir-name"); //instead of hdfs I need to save it on mysql database, but I am not able to find any Spark-MYSQL connection
}
}
Here at the end I am saving the result successfully in HDFS. But now I want to save into MYSQL database. Kindly help me out. Thanks
There are two approaches you can use for writing your results back to the database. One is to use something like DBOutputFormat and configure that, and the other is to use foreachPartition on the RDD you want to save and pass in a function which creates a connection to MySQL and writes the result back.
Here is an example using DBOutputFormat.
Create a class that represents your table row -
public class TableRow implements DBWritable
{
public String column1;
public String column2;
#Override
public void write(PreparedStatement statement) throws SQLException
{
statement.setString(1, column1);
statement.setString(2, column2);
}
#Override
public void readFields(ResultSet resultSet) throws SQLException
{
throw new RuntimeException("readFields not implemented");
}
}
Then configure your job and write a mapToPair function. The value doesn't appear to be used. If anyone knows, please post a comment.
String tableName = "YourTableName";
String[] fields = new String[] { "column1", "column2" };
JobConf job = new JobConf();
DBConfiguration.configureDB(job, "com.mysql.jdbc.Driver", "jdbc:mysql://localhost/DatabaseNameHere", "username", "password");
DBOutputFormat.setOutput(job, tableName, fields);
// map your rdd into a table row
JavaPairRDD<TableRow, Object> rows = rdd.mapToPair(...);
rows.saveAsHadoopDataset(job);

The same "Error parsing data org.json.JSONException" bug still following me

I'm using AsyncTask in order to display data provided from database through PHP and JSON, so when I try to run out my application I got that errors :
09-20 15:31:51.330: E/Buffer Error(4484): Error converting result java.lang.NullPointerException
09-20 15:31:51.330: E/JSON Parser(4484): Error parsing data org.json.JSONException: End of input at character 0 of
This is my Java Class :
package com.androidhive.dashboard;
import android.app.Activity;
import android.app.ListActivity;
import android.os.Bundle;
import androidhive.dashboard.R;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import org.apache.http.NameValuePair;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import android.app.ProgressDialog;
import android.content.Intent;
import android.os.AsyncTask;
import android.util.Log;
import android.view.View;
import android.widget.AdapterView;
import android.widget.AdapterView.OnItemClickListener;
import android.widget.ListAdapter;
import android.widget.ListView;
import android.widget.SimpleAdapter;
import android.widget.TextView;
public class PlacesActivity extends ListActivity {
// Progress Dialog
private ProgressDialog pDialog;
// Creating JSON Parser object
JSONParser jParser = new JSONParser();
ArrayList<HashMap<String, String>> productsList;
// url to get all products list
private static String url_all_products = "http://192.168.1.74/test/focus.php";
// JSON Node names
private static final String TAG_SUCCESS = "success";
private static final String TAG_PRODUCTS = "products";
private static final String TAG_PID = "pid";
private static final String LIB_ART = "LibArt";
private static final String COD_ART = "CodArt";
// products JSONArray
JSONArray products = null;
#Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.places_layout);
// Hashmap for ListView
productsList = new ArrayList<HashMap<String, String>>();
// Loading products in Background Thread
new LoadAllProducts().execute();
// Get listview
ListView lv = getListView();
}//onCreate finish
// Response from Edit Product Activity
#Override
protected void onActivityResult(int requestCode, int resultCode, Intent data) {
super.onActivityResult(requestCode, resultCode, data);
// if result code 100
if (resultCode == 100) {
// if result code 100 is received
// means user edited/deleted product
// reload this screen again
Intent intent = getIntent();
finish();
startActivity(intent);
}
}
/**
* Background Async Task to Load all product by making HTTP Request
* */
class LoadAllProducts extends AsyncTask<String, String, String> {
/**
* Before starting background thread Show Progress Dialog
* */
#Override
protected void onPreExecute() {
super.onPreExecute();
pDialog = new ProgressDialog(PlacesActivity.this);
pDialog.setMessage("Loading products. Please wait...");
pDialog.setIndeterminate(false);
pDialog.setCancelable(false);
pDialog.show();
}
/**
* getting All products from url
* */
protected String doInBackground(String... args) {
// Building Parameters
List<NameValuePair> params = new ArrayList<NameValuePair>();
// getting JSON string from URL
JSONObject json = jParser.makeHttpRequest(url_all_products, "GET", params);
// Check your log cat for JSON reponse
Log.d("All Products: ", json.toString());
try {
// Checking for SUCCESS TAG
int success = json.getInt(TAG_SUCCESS);
if (success == 1) {
// products found
// Getting Array of Products
products = json.getJSONArray(TAG_PRODUCTS);
// looping through All Products
for (int i = 0; i < products.length(); i++) {
JSONObject c = products.getJSONObject(i);
// Storing each json item in variable
String id = c.getString(TAG_PID);
String LibArt = c.getString(LIB_ART);
String CodArt = c.getString(COD_ART);
// creating new HashMap
HashMap<String, String> map = new HashMap<String, String>();
// adding each child node to HashMap key => value
map.put(TAG_PID, id);
map.put(LIB_ART, LibArt);
map.put(COD_ART,CodArt);
// adding HashList to ArrayList
productsList.add(map);
}
}
} catch (JSONException e) {
e.printStackTrace();
}
return null;
}
/**
* After completing background task Dismiss the progress dialog
* **/
protected void onPostExecute(String file_url) {
// dismiss the dialog after getting all products
pDialog.dismiss();
// updating UI from Background Thread
runOnUiThread(new Runnable() {
public void run() {
/**
* Updating parsed JSON data into ListView
* */
ListAdapter adapter = new SimpleAdapter(
PlacesActivity.this, productsList,
R.layout.list_item, new String[] { COD_ART,TAG_PID,
LIB_ART},
new int[] { R.id.codart ,R.id.pid, R.id.libart });
// updating listview
setListAdapter(adapter);
}
});
}
}
}
And here it is the PHP File :
<?php
require 'FastJSON.class.php';
$db = mssql_connect ('HPWALID', '', '');
$ret = mssql_select_db ('Focus', $db) or die ('Echec lors de la connexion: '.mysql_error ());
$result = mssql_query("SELECT * FROM TabStock");
if (mssql_num_rows($result) > 0) {
// looping through all results
// products node
$response["products"] = array();
while ($row = mssql_fetch_array($result)) {
// temp user array
$product = array();
$product["CodArt"] = $row["CodArt"];
$product["LibArt"] = $row["LibArt"];
$product["pid"] = $row["pid"];
// push single product into final response array
array_push($response["products"], $product);
}
// success
$response["success"] = 1;
// echoing JSON response
$var = FastJSON::encode($response);
echo $var;
} else {
// no products found
$response["success"] = 0;
$response["message"] = "No products found";
// echo no users JSON
// echo json_encode($response);
$var = FastJSON::encode($response);
echo $var;
}
?>
change your IP address(192.168.1.74) to (10.0.2.2) if you are using emualtor

Anyone able to get directory listing feature work in Android

It seems that, recent released Google Drive SDK supports directory listing.
https://developers.google.com/drive/v2/reference/files/list
I try to integrate it into my Android app.
package com.jstock.cloud;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import android.util.Log;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.services.GoogleKeyInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.gson.GsonFactory;
import com.google.api.client.extensions.android2.AndroidHttp;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.Drive.Files;
import com.google.api.services.drive.model.File;
import com.google.api.services.drive.model.FileList;
import com.jstock.gui.Utils;
public class CloudFile {
public final java.io.File file;
public final long checksum;
public final long date;
public final int version;
private CloudFile(java.io.File file, long checksum, long date, int version) {
this.file = file;
this.checksum = checksum;
this.date = date;
this.version = version;
}
public static CloudFile newInstance(java.io.File file, long checksum, long date, int version) {
return new CloudFile(file, checksum, date, version);
}
public static CloudFile loadFromGoogleDrive(String authToken) {
final HttpTransport transport = AndroidHttp.newCompatibleTransport();
final JsonFactory jsonFactory = new GsonFactory();
GoogleCredential credential = new GoogleCredential();
credential.setAccessToken(authToken);
Drive service = new Drive.Builder(transport, jsonFactory, credential)
.setApplicationName(Utils.getApplicationName())
.setJsonHttpRequestInitializer(new GoogleKeyInitializer(ClientCredentials.KEY))
.build();
List<File> files = retrieveAllFiles(service);
Log.i("CHEOK", "size is " + files.size());
for (File file : files) {
Log.i(TAG, "title = " + file.getTitle());
}
return null;
}
/**
* Retrieve a list of File resources.
*
* #param service Drive API service instance.
* #return List of File resources.
*/
private static List<File> retrieveAllFiles(Drive service) {
List<File> result = new ArrayList<File>();
Files.List request = null;
try {
request = service.files().list();
} catch (IOException e) {
Log.e(TAG, "", e);
return result;
}
do {
try {
FileList files = request.execute();
result.addAll(files.getItems());
request.setPageToken(files.getNextPageToken());
} catch (IOException e) {
Log.e(TAG, "", e);
request.setPageToken(null);
}
} while (request.getPageToken() != null && request.getPageToken().length() > 0);
Log.i("CHEOK", "yup!");
return result;
}
private static final String TAG = "CloudFile";
}
I always get 0 file returned from server, and there isn't any exception being thrown. Is there anything I had missed out?
You have to request access to the full Drive scope to list all files: https://developers.google.com/drive/scopes#requesting_full_drive_scope_for_an_app
If you use the default (and recommended) scope https://www.googleapis.com/auth/drive.file, you will only be able to see files created or opened by the app.