18.08.2015 – 4,7 km – 28:00
This week I created this iterator to read data from an csv file and return a list of beans. Maybe useful for other 🙂
import java.io.IOException; import java.io.Reader; import java.util.Arrays; import java.util.Iterator; import javax.annotation.Nullable; import org.supercsv.cellprocessor.ift.CellProcessor; import org.supercsv.io.CsvBeanReader; import org.supercsv.prefs.CsvPreference; import com.google.common.base.Function; import com.google.common.collect.AbstractIterator; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.io.Closeables; /** * @author Andreas Höhmann */ public class SuperCsvIterator<T> extends AbstractIterator<T> { public static <T> ImmutableList<T> toList(final CsvBeanReader beanReader, final Class<T> beanClass, final CellProcessor[] cellProcessors ) throws IOException { return new FluentIterable<T>() { @Override public Iterator<T> iterator() { return new CsvIterator<T>( beanClass, beanReader, cellProcessors ); } }.toList(); } public static <T> ImmutableList<T> toList(final Reader reader, final Class<T> beanClass, final CsvPreference csvPreference, final CellProcessor[] cellProcessors ) throws IOException { final CsvBeanReader beanReader = new CsvBeanReader( reader, CsvPreference.STANDARD_PREFERENCE ); try { return toList( beanReader, beanClass, cellProcessors ); } finally { Closeables.close( beanReader, false ); } } private static String[] trimHeaders( final String[] headers ) { return FluentIterable.from( Arrays.asList( headers ) ).transform( new Function<String, String>() { @Override @Nullable public String apply( @Nullable final String input ) { if ( input == null ) return null; return input.trim(); } } ).toArray( String.class ); } private final Class<T> beanClass; private final CsvBeanReader beanReader; private final CellProcessor[] cellProcessors; private final String[] headers; public SuperCsvIterator(final Class<T> beanClass, final CsvBeanReader beanReader, final CellProcessor[] cellProcessors ) throws IOException { this.beanClass = beanClass; this.beanReader = beanReader; this.cellProcessors = Arrays.copyOf( cellProcessors, cellProcessors.length ); this.headers = trimHeaders( beanReader.getHeader( true ) ); } @Override protected T computeNext() { try { final T result = beanReader.read( beanClass, headers, cellProcessors ); if ( result == null ) { return endOfData(); } return result; } catch ( final IOException e ) { throw new RuntimeException( "Can't compute next csv bean element", e ); } } }
public final class FooCsvBean { public static final String[] HEADER = new String[] { "Name", "Age", "Bar", }; public static final CellProcessor[] CELLPROCESSOR = new CellProcessor[] { null, // Name null, // Age null, // Bar }; private String name, age, bar; // Setter + Getter } final ImmutableList<FooCsvBean> foos = SuperCsvIterator.toList(new FileReader( fooCsvFile ), FooCsvBean.class, CsvPreference.STANDARD_PREFERENCE, FooCsvBean.CELLPROCESSOR );
Der „Leipziger Firmenlauf 2013“ ist nun auch langsam Geschichte. Doch als ich heute nochmals durch die Ergebnisse stöberte, kam ich auf die Idee die Daten doch einfach mal in eine „NoSQL“ Datenbank, in meinem Fall jetzt mal Neo4j, zu importieren.
Also gesagt getan, das Besorgen der Daten ist ein Kinderspiel. Die Homepage liefert alle gewünschten Datensätze per JSON, im Chrome die „CURL“ kopieren, auf Kommandozeile ausführen, in eine Datei umleiten … fertig 🙂
$ curl 'http://results.davengo.com/query' -H 'Origin: http://results.davengo.com' -H 'Accept-Encoding: gzip,deflate,sdch' -H 'Host: results.davengo.com' -H 'Accept-Language: en,de;q=0.8 ,en-US;q=0.6' -H 'User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36' -H 'Content-Type: application/json' -H 'Accept: */*' -H 'Referer: http://results.davengo.com/?event=50d426afb8ed1d2318282f54' -H 'Cookie: __utma=9661994.565730006.1373516179.1373516179.1373909845.2; __utmb=9661994.2.10.1373909845; __utmc=9661994; __utmz=9661994.1373909845.2.2.utmcsr=leipzig-firmenlauf.de|utmccn=(referral)|utmcmd=referral|utmcct=/ergebnisse2013.html; __utma=268406157.662336661.1373516307.1373516307.1373909929.2; __utmb=268406157.1.10.1373909929; __utmc=268406157; __utmz=268406157.1373516307.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); projekktorplayercontrolbar_volume=0.5' -H 'Connection: keep-alive' --data-binary '{"key":"50d426afb8ed1d2318282f54", "offset":0, "limit":100000, "where":{"categoryName":"Einzelwertung"}, "order": {"firma":"ASC"}, "category":"Einzelwertung"}' --compressed > results_2013.json
Jetzt kommt der „schwierige“ Part, JSON parsen und via Spring Data Neo4j in eine Neo4j Datenbank einfügen.
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.mymita.tools</groupId> <artifactId>firmenlauf2013</artifactId> <version>0.0.1-SNAPSHOT</version> <dependencies> <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-mapper-asl</artifactId> <version>1.9.12</version> </dependency> <dependency> <groupId>org.springframework.data</groupId> <artifactId>spring-data-neo4j</artifactId> <version>2.2.1.RELEASE</version> </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> <version>0.11.8</version> </dependency> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>14.0.1</version> </dependency> <dependency> <groupId>cglib</groupId> <artifactId>cglib</artifactId> <version>2.2.2</version> </dependency> <dependency> <groupId>javax.validation</groupId> <artifactId>validation-api</artifactId> <version>1.1.0.Final</version> </dependency> <dependency> <groupId>ch.qos.logback</groupId> <artifactId>logback-classic</artifactId> <version>1.0.13</version> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <version>3.1</version> <configuration> <source>1.7</source> <target>1.7</target> </configuration> </plugin> </plugins> </build> </project>
Der Importerservice
package com.mymita.firmenlauf2013.service; import java.io.IOException; import java.util.Map; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; import lombok.ToString; import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import org.springframework.data.neo4j.support.Neo4jTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import com.google.common.base.Objects; import com.google.common.collect.Maps; import com.mymita.firmenlauf2013.model.Company; import com.mymita.firmenlauf2013.model.Run; import com.mymita.firmenlauf2013.model.Runner; import com.mymita.firmenlauf2013.model.Team; @Service public class ImportService { @Getter @AllArgsConstructor @ToString public static class JsonRunner { public JsonRunner(Object[] data) { this((Integer) data[0], (Integer) data[1], (Integer) data[2], (Integer) data[3], (String) data[4], (String) data[5], (String) data[6], (String) data[7], (String) data[8], (String) data[9], (String) data[10]); } Integer platz; Integer platzMaennlich; Integer platzWeiblich; Integer nummer; String vorname; String nachname; String team; String zeit; String firma; String dummy; String key; } @Getter @Setter public static class JsonRunners { Object[][] items; } Logger logger = LoggerFactory.getLogger(getClass()); @Autowired private Neo4jTemplate template; private final Map<String, Company> companies = Maps.newHashMap(); private final Map<String, Team> teams = Maps.newHashMap(); private final Map<Integer, Runner> runners = Maps.newHashMap(); private final Map<String, Run> runs = Maps.newHashMap(); @Transactional public void importData() { try { JsonRunners jsonRunners = new ObjectMapper().readValue( new ClassPathResource("results.json").getInputStream(), JsonRunners.class); for (int i = 0; i < jsonRunners.items.length; i++) { JsonRunner jsonRunner = new JsonRunner(jsonRunners.items[i]); String teamName = jsonRunner.getTeam(); String companyName = jsonRunner.getFirma(); String time = jsonRunner.getZeit(); Company company = companies.get(companyName); if (company == null) { company = new Company(Objects.firstNonNull(companyName, "NONAME")); template.save(company); companies.put(company.getName(), company); logger.debug("Created company '{}'", company); } Team team = teams.get(teamName); if (team == null) { team = new Team(teamName).sponsoredBy(company); template.save(team); teams.put(team.getName(), team); logger.debug("Created team '{}'", team); } Run run = runs.get(time); if (run == null) { run = new Run(time); template.save(run); runs.put(run.getTime(), run); logger.debug("Created run '{}'", run); } Runner runner = new Runner(jsonRunner.getNummer(), jsonRunner.getVorname(), jsonRunner.getNachname()) .fightFor(team).finished(run); template.save(runner); runners.put(runner.getNumber(), runner); logger.debug("Created runner '{}'", runner); } for (Team t : teams.values()) { template.save(t); logger.debug("Updated team '{}'", t); } for (Run r : runs.values()) { template.save(r); logger.debug("Updated run '{}'", r); } } catch (IOException e) { e.printStackTrace(); } } }
Der Spring Context
<?xml version="1.0" encoding="UTF-8"?> <beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:neo4j="http://www.springframework.org/schema/data/neo4j" xmlns:tx="http://www.springframework.org/schema/tx" xmlns:context="http://www.springframework.org/schema/context" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd http://www.springframework.org/schema/data/neo4j http://www.springframework.org/schema/data/neo4j/spring-neo4j.xsd"> <neo4j:repositories base-package="com.mymita.firmenlauf2013" /> <neo4j:config graphDatabaseService="graphDatabaseService" /> <bean id="graphDatabaseService" class="org.neo4j.kernel.EmbeddedGraphDatabase" destroy-method="shutdown"> <constructor-arg value="target/firmenlauf2013.db" /> </bean> <context:annotation-config /> <context:component-scan base-package="com.mymita.firmenlauf2013" /> <tx:annotation-driven mode="proxy" transaction-manager="transactionManager" /> </beans>
Und ein kleines Hauptprogramm um alles zu starten
package com.mymita.firmenlauf2013.importer; import java.io.File; import java.io.IOException; import org.neo4j.kernel.impl.util.FileUtils; import org.springframework.context.support.ClassPathXmlApplicationContext; import com.mymita.firmenlauf2013.service.ImportService; public class Importer { public static void main(String[] args) throws IOException { FileUtils.deleteRecursively(new File("target/firmenlauf2013.db")); ClassPathXmlApplicationContext ctx = new ClassPathXmlApplicationContext( "classpath:application-context.xml"); try { new Importer().importData(ctx); } finally { ctx.close(); } } private void importData(ClassPathXmlApplicationContext ctx) { ctx.getBean(ImportService.class).importData(); } }
Die NodeEntites sehen folgendermassen aus …
package com.mymita.firmenlauf2013.model; import lombok.ToString; import org.springframework.data.neo4j.annotation.GraphId; @ToString public abstract class AbstractEntity { @GraphId private Long id; public Long getId() { return id; } @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (id == null || obj == null || !getClass().equals(obj.getClass())) { return false; } return id.equals(((AbstractEntity) obj).id); } @Override public int hashCode() { return id == null ? 0 : id.hashCode(); } }
Die Firma für die ein Team von Läufern startete
package com.mymita.firmenlauf2013.model; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.ToString; import org.springframework.data.annotation.TypeAlias; import org.springframework.data.neo4j.annotation.Indexed; import org.springframework.data.neo4j.annotation.NodeEntity; @Getter @NoArgsConstructor @ToString(callSuper=true) @NodeEntity @TypeAlias("Company") public class Company extends AbstractEntity { @Indexed(unique=true) private String name; public Company(String name) { this.name = name; } }
Das Team von Läufern
package com.mymita.firmenlauf2013.model; import java.util.Set; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.ToString; import org.springframework.data.annotation.TypeAlias; import org.springframework.data.neo4j.annotation.Indexed; import org.springframework.data.neo4j.annotation.NodeEntity; import org.springframework.data.neo4j.annotation.RelatedTo; import com.google.common.collect.Sets; @Getter @NoArgsConstructor @ToString(callSuper=true, exclude={"runners"}) @NodeEntity @TypeAlias("Team") public class Team extends AbstractEntity { @Indexed(unique=true) private String name; @RelatedTo(type="have") private Set<Runner> runners = Sets.newHashSet(); @RelatedTo(type="sponsored_by") private Company company; public Team(String name) { this.name = name; } public Team sponsoredBy(Company company) { this.company = company; return this; } public Team have(Runner runner) { runners.add(runner); return this; } }
Ein Läufer
package com.mymita.firmenlauf2013.model; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.ToString; import org.springframework.data.annotation.TypeAlias; import org.springframework.data.neo4j.annotation.Indexed; import org.springframework.data.neo4j.annotation.NodeEntity; import org.springframework.data.neo4j.annotation.RelatedTo; @Getter @NoArgsConstructor @ToString(callSuper=true, exclude={"run", "team"}) @NodeEntity @TypeAlias("Runner") public class Runner extends AbstractEntity{ @Indexed(numeric=true,unique=true) private Integer number; @Indexed private String firstName; @Indexed private String lastName; @RelatedTo(type="fight_for") private Team team; @RelatedTo(type="finished") private Run run; public Runner(Integer number, String firstName, String lastName) { this.number = number; this.firstName = firstName; this.lastName = lastName; } public Runner fightFor(Team team) { this.team=team; team.have(this); return this; } public Runner finished(Run run) { this.run=run; run.finishedBy(this); return this; } }
Und das Ergebnis eines Laufs
package com.mymita.firmenlauf2013.model; import java.util.Set; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.ToString; import org.springframework.data.annotation.TypeAlias; import org.springframework.data.neo4j.annotation.NodeEntity; import org.springframework.data.neo4j.annotation.RelatedTo; import com.google.common.collect.Sets; @Getter @NoArgsConstructor @ToString(callSuper=true, exclude={"runners"}) @NodeEntity @TypeAlias("Run") public class Run extends AbstractEntity{ private String time; @RelatedTo(type="finished_by") private Set<Runner> runners = Sets.newHashSet(); public Run(String time) { this.time = time; } public Run finishedBy(Runner runner) { runners.add(runner); return this; } }
Ergebnis der ganze Mühen ist ein Verzeichnis im target Ordner des Mavenprojekts, dessen Inhalt in einen Neo4j-Server (http://www.neo4j.org/download) kopiert wird. Ist der Server hochgefahren, kann man unter http://localhost:7474 das Ergebnis bestaunen 🙂
Eine kleine Abfrage bringt dann auch alle meine lieben Kollegen vom „Running suckz“ Team zum Vorschein
Currently I want run https://github.com/jamesward/hello-java-spring-neo4j
$ mvn -version
Apache Maven 3.1.0-alpha-1 (c726cdd3a9ad5c3a419e1171f8c1925e336ead18; 2013-06-01 15:03:28+0200)
Maven home: C:\Development\apache-maven-3.1.0-alpha-1
Java version: 1.7.0_09, vendor: Oracle Corporation
Java home: C:\Program Files\Java\jdk1.7.0_09\jre
Default locale: de_DE, platform encoding: Cp1252
OS name: „windows 7“, version: „6.1“, arch: „amd64“, family: „windows“
Unfortunately I got some strange error like „access denied“ for tinkerpop (I never heard about this maven repository until now) and slowwwww connection to memcache repo at googlecode (svn) … so here is a „workaround“ for that …
<?xml version="1.0" encoding="UTF-8"?> <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd"> <mirrors> <mirror> <id>tinkerpop</id> <name>tinkerpop</name> <url>http://repo.maven.apache.org/maven2</url> <mirrorOf>tinkerpop-repository</mirrorOf> </mirror> <mirror> <id>msm</id> <name>msm</name> <url>http://repo.maven.apache.org/maven2</url> <mirrorOf>msm</mirrorOf> </mirror> </mirrors> </settings>
I added support for Fluent API into https://github.com/ahoehma/vaadlets.
Now you can do this:
VaadletsBuilder.build( new Vaadlets().withRootComponent( new VerticalLayout() .withHeight("100px") .withWidth("100px") .withStyleName("foobar") .withComponents( new Button().withCaption("hustensaft") .withAlignment(Alignment.MIDDLE_CENTER) .withExpandRatio(1f))));
In a older post I describe my solution to handle VAADIN/GWT widgetset compilation with maven.
Today I will share some changes to use GWT 2.4 with my solution.
The problem: after GWT compilation a new directory gwt-unitCache exists under the VAADIN directory. I don’t need this 😉
I integrate this workaround in my profile:
<profile> <!-- Updates Vaadin widgetset definitions based on project dependencies --> <id>update-widgetset</id> <activation> <file> <missing>${basedir}/src/main/webapp/VAADIN/widgetsets/</missing> </file> </activation> <build> <plugins> <plugin> <groupId>com.vaadin</groupId> <artifactId>vaadin-maven-plugin</artifactId> <version>1.0.2</version> <executions> <execution> <phase>generate-resources</phase> <goals> <goal>update-widgetset</goal> </goals> </execution> </executions> </plugin> <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>gwt-maven-plugin</artifactId> <version>2.4.0</version> <configuration> <webappDirectory>${basedir}/src/main/webapp/VAADIN/widgetsets/</webappDirectory> <extraJvmArgs>-Xmx512M -Xss1024k </extraJvmArgs> <deploy>${project.build.directory}/gwt-tmp/</deploy> <soyc>false</soyc> <force>true</force> <strict>true</strict> <style>OBFUSCATED</style> <optimizationLevel>9</optimizationLevel> </configuration> <executions> <execution> <phase>generate-resources</phase> <goals> <goal>resources</goal> <goal>compile</goal> </goals> </execution> </executions> </plugin> <plugin> <!-- workaround for http://code.google.com/p/google-web-toolkit/issues/detail?id=6397 --> <artifactId>maven-clean-plugin</artifactId> <version>2.4.1</version> <configuration> <filesets> <fileset> <directory>${basedir}/src/main/webapp/VAADIN/</directory> <includes> <directory>gwt-unitCache/**</directory> </includes> <followSymlinks>false</followSymlinks> </fileset> </filesets> <excludeDefaultDirectories>true</excludeDefaultDirectories> </configuration> <executions> <execution> <id>default</id> <phase>generate-resources</phase> <goals> <goal>clean</goal> </goals> </execution> </executions> </plugin> </plugins> </build> </profile>
🙂
C:\Program Files\Microsoft SQL Server\80\Tools\Binn>osql -E sa -S [HOSTNAME]\SQLEXPRESS
1> sp_password @old=null, @new=’123geheim‘, @loginame = ’sa‘
2> go
Have a look at
And then you can monitor the JMX-exposed data with tools like JRDS http://jrds.fr/
Another little-bit offtopic goodie: http://hibernate-jcons.sourceforge.net/index.html
jmonkey have no maven support … here is my solution 🙂
1. clone jme3-thirdparty from git, mvn install … done
2. clone jme3-buildhelper, copy pom.xml into jmonkey-engine … done
3. start hacking your jmonkey game with maven build support 🙂
Hi Andreas!, great works!! What about updated version on jmonkey? I’ve looking for start new work like bloxel! Can you help me? Thank you!
Hi. I’m not up2date with the latest jmonkey version. Feel free to send me an pull-request 🙂 For bloxel-like (minecraft-clones) you can have a look at https://github.com/MovingBlocks/Terasology.
Antworten
Du musst angemeldet sein, um einen Kommentar abzugeben.