Lets play with neo4j


Der „Leipziger Firmenlauf 2013“ ist nun auch langsam Geschichte. Doch als ich heute nochmals durch die Ergebnisse stöberte, kam ich auf die Idee die Daten doch einfach mal in eine „NoSQL“ Datenbank, in meinem Fall jetzt mal Neo4j, zu importieren.

Also gesagt getan, das Besorgen der Daten ist ein Kinderspiel. Die Homepage liefert alle gewünschten Datensätze per JSON, im Chrome die „CURL“ kopieren, auf Kommandozeile ausführen, in eine Datei umleiten … fertig🙂

$ curl 'http://results.davengo.com/query' -H 'Origin: http://results.davengo.com' -H 'Accept-Encoding: gzip,deflate,sdch' -H 'Host: results.davengo.com' -H 'Accept-Language: en,de;q=0.8
,en-US;q=0.6' -H 'User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36' -H 'Content-Type: application/json' -H 
'Accept: */*' -H 'Referer: http://results.davengo.com/?event=50d426afb8ed1d2318282f54' -H 'Cookie: __utma=9661994.565730006.1373516179.1373516179.1373909845.2; __utmb=9661994.2.10.1373909845; 
__utmc=9661994; __utmz=9661994.1373909845.2.2.utmcsr=leipzig-firmenlauf.de|utmccn=(referral)|utmcmd=referral|utmcct=/ergebnisse2013.html; 
__utma=268406157.662336661.1373516307.1373516307.1373909929.2; __utmb=268406157.1.10.1373909929; __utmc=268406157; __utmz=268406157.1373516307.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); 
projekktorplayercontrolbar_volume=0.5' -H 'Connection: keep-alive' --data-binary '{"key":"50d426afb8ed1d2318282f54", "offset":0, "limit":100000, "where":{"categoryName":"Einzelwertung"}, "order":
{"firma":"ASC"}, "category":"Einzelwertung"}' --compressed > results_2013.json

Jetzt kommt der „schwierige“ Part, JSON parsen und via Spring Data Neo4j in eine Neo4j Datenbank einfügen.

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>com.mymita.tools</groupId>
	<artifactId>firmenlauf2013</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<dependencies>
		<dependency>
			<groupId>org.codehaus.jackson</groupId>
			<artifactId>jackson-mapper-asl</artifactId>
			<version>1.9.12</version>
		</dependency>
		<dependency>
			<groupId>org.springframework.data</groupId>
			<artifactId>spring-data-neo4j</artifactId>
			<version>2.2.1.RELEASE</version>
		</dependency>
		<dependency>
			<groupId>org.projectlombok</groupId>
			<artifactId>lombok</artifactId>
			<version>0.11.8</version>
		</dependency>
		<dependency>
			<groupId>com.google.guava</groupId>
			<artifactId>guava</artifactId>
			<version>14.0.1</version>
		</dependency>
		<dependency>
			<groupId>cglib</groupId>
			<artifactId>cglib</artifactId>
			<version>2.2.2</version>
		</dependency>
		<dependency>
			<groupId>javax.validation</groupId>
			<artifactId>validation-api</artifactId>
			<version>1.1.0.Final</version>
		</dependency>
		<dependency>
			<groupId>ch.qos.logback</groupId>
			<artifactId>logback-classic</artifactId>
			<version>1.0.13</version>
		</dependency>
	</dependencies>
	<build>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>3.1</version>
				<configuration>
					<source>1.7</source>
					<target>1.7</target>
				</configuration>
			</plugin>
		</plugins>
	</build>
</project>

Der Importerservice

package com.mymita.firmenlauf2013.service;

import java.io.IOException;
import java.util.Map;

import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;

import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.data.neo4j.support.Neo4jTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import com.google.common.base.Objects;
import com.google.common.collect.Maps;
import com.mymita.firmenlauf2013.model.Company;
import com.mymita.firmenlauf2013.model.Run;
import com.mymita.firmenlauf2013.model.Runner;
import com.mymita.firmenlauf2013.model.Team;

@Service
public class ImportService {

	@Getter
	@AllArgsConstructor
	@ToString
	public static class JsonRunner {

		public JsonRunner(Object[] data) {
			this((Integer) data[0], (Integer) data[1], (Integer) data[2],
					(Integer) data[3], (String) data[4], (String) data[5],
					(String) data[6], (String) data[7], (String) data[8],
					(String) data[9], (String) data[10]);
		}

		Integer platz;
		Integer platzMaennlich;
		Integer platzWeiblich;
		Integer nummer;
		String vorname;
		String nachname;
		String team;
		String zeit;
		String firma;
		String dummy;
		String key;
	}

	@Getter
	@Setter
	public static class JsonRunners {
		Object[][] items;
	}

	Logger logger = LoggerFactory.getLogger(getClass());

	@Autowired
	private Neo4jTemplate template;

	private final Map<String, Company> companies = Maps.newHashMap();
	private final Map<String, Team> teams = Maps.newHashMap();
	private final Map<Integer, Runner> runners = Maps.newHashMap();
	private final Map<String, Run> runs = Maps.newHashMap();

	@Transactional
	public void importData() {
		try {
			JsonRunners jsonRunners = new ObjectMapper().readValue(
					new ClassPathResource("results.json").getInputStream(),
					JsonRunners.class);
			for (int i = 0; i < jsonRunners.items.length; i++) {
				JsonRunner jsonRunner = new JsonRunner(jsonRunners.items[i]);
				String teamName = jsonRunner.getTeam();
				String companyName = jsonRunner.getFirma();
				String time = jsonRunner.getZeit();
				Company company = companies.get(companyName);
				if (company == null) {
					company = new Company(Objects.firstNonNull(companyName,
							"NONAME"));
					template.save(company);
					companies.put(company.getName(), company);
					logger.debug("Created company '{}'", company);
				}
				Team team = teams.get(teamName);
				if (team == null) {
					team = new Team(teamName).sponsoredBy(company);
					template.save(team);
					teams.put(team.getName(), team);
					logger.debug("Created team '{}'", team);
				}
				Run run = runs.get(time);
				if (run == null) {
					run = new Run(time);
					template.save(run);
					runs.put(run.getTime(), run);
					logger.debug("Created run '{}'", run);
				}
				Runner runner = new Runner(jsonRunner.getNummer(),
						jsonRunner.getVorname(), jsonRunner.getNachname())
						.fightFor(team).finished(run);
				template.save(runner);
				runners.put(runner.getNumber(), runner);
				logger.debug("Created runner '{}'", runner);
			}
			for (Team t : teams.values()) {
				template.save(t);
				logger.debug("Updated team '{}'", t);
			}
			for (Run r : runs.values()) {
				template.save(r);
				logger.debug("Updated run '{}'", r);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}

Der Spring Context

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:neo4j="http://www.springframework.org/schema/data/neo4j"
	xmlns:tx="http://www.springframework.org/schema/tx" xmlns:context="http://www.springframework.org/schema/context"
	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
		                http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx.xsd
		                http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
		                http://www.springframework.org/schema/data/neo4j http://www.springframework.org/schema/data/neo4j/spring-neo4j.xsd">
	<neo4j:repositories base-package="com.mymita.firmenlauf2013" />
	<neo4j:config graphDatabaseService="graphDatabaseService" />
	<bean id="graphDatabaseService" class="org.neo4j.kernel.EmbeddedGraphDatabase"
		destroy-method="shutdown">
		<constructor-arg value="target/firmenlauf2013.db" />
	</bean>
	<context:annotation-config />
	<context:component-scan base-package="com.mymita.firmenlauf2013" />
	<tx:annotation-driven mode="proxy"
		transaction-manager="transactionManager" />
</beans>

Und ein kleines Hauptprogramm um alles zu starten

package com.mymita.firmenlauf2013.importer;

import java.io.File;
import java.io.IOException;

import org.neo4j.kernel.impl.util.FileUtils;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import com.mymita.firmenlauf2013.service.ImportService;

public class Importer {

	public static void main(String[] args) throws IOException {
		FileUtils.deleteRecursively(new File("target/firmenlauf2013.db"));
		ClassPathXmlApplicationContext ctx = new ClassPathXmlApplicationContext(
				"classpath:application-context.xml");
		try {
			new Importer().importData(ctx);
		} finally {
			ctx.close();
		}
	}

	private void importData(ClassPathXmlApplicationContext ctx) {
        ctx.getBean(ImportService.class).importData();
    }

}

Die NodeEntites sehen folgendermassen aus …

package com.mymita.firmenlauf2013.model;

import lombok.ToString;
import org.springframework.data.neo4j.annotation.GraphId;

@ToString
public abstract class AbstractEntity {

	@GraphId
	private Long id;

	public Long getId() {
		return id;
	}

	@Override
	public boolean equals(Object obj) {
		if (this == obj) {
			return true;
		}
		if (id == null || obj == null || !getClass().equals(obj.getClass())) {
			return false;
		}
		return id.equals(((AbstractEntity) obj).id);
	}

	@Override
	public int hashCode() {
		return id == null ? 0 : id.hashCode();
	}
}

Die Firma für die ein Team von Läufern startete

package com.mymita.firmenlauf2013.model;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;

import org.springframework.data.annotation.TypeAlias;
import org.springframework.data.neo4j.annotation.Indexed;
import org.springframework.data.neo4j.annotation.NodeEntity;

@Getter
@NoArgsConstructor
@ToString(callSuper=true)
@NodeEntity
@TypeAlias("Company")
public class Company extends AbstractEntity {

    @Indexed(unique=true)
    private String name;

    public Company(String name) {
		this.name = name;
	}

}

Das Team von Läufern

package com.mymita.firmenlauf2013.model;
import java.util.Set;

import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;

import org.springframework.data.annotation.TypeAlias;
import org.springframework.data.neo4j.annotation.Indexed;
import org.springframework.data.neo4j.annotation.NodeEntity;
import org.springframework.data.neo4j.annotation.RelatedTo;

import com.google.common.collect.Sets;

@Getter
@NoArgsConstructor
@ToString(callSuper=true, exclude={"runners"})
@NodeEntity
@TypeAlias("Team")
public class Team extends AbstractEntity {

    @Indexed(unique=true)
    private String name;
    @RelatedTo(type="have")
    private Set<Runner> runners = Sets.newHashSet();
    @RelatedTo(type="sponsored_by")
    private Company company;

    public Team(String name) {
		this.name = name;
	}

    public Team sponsoredBy(Company company) {
    	this.company = company;
    	return this;
    }

    public Team have(Runner runner) {
    	runners.add(runner);
    	return this;
    }

}

Ein Läufer

package com.mymita.firmenlauf2013.model;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;

import org.springframework.data.annotation.TypeAlias;
import org.springframework.data.neo4j.annotation.Indexed;
import org.springframework.data.neo4j.annotation.NodeEntity;
import org.springframework.data.neo4j.annotation.RelatedTo;

@Getter
@NoArgsConstructor
@ToString(callSuper=true, exclude={"run", "team"})
@NodeEntity
@TypeAlias("Runner")
public class Runner extends AbstractEntity{

    @Indexed(numeric=true,unique=true)
    private Integer number;
    @Indexed
    private String firstName;
    @Indexed
    private String lastName;
    @RelatedTo(type="fight_for")
    private Team team;
    @RelatedTo(type="finished")
    private Run run;

    public Runner(Integer number, String firstName, String lastName) {
		this.number = number;
		this.firstName = firstName;
		this.lastName = lastName;
    }

    public Runner fightFor(Team team) {
    	this.team=team;
    	team.have(this);
    	return this;
    }

    public Runner finished(Run run) {
    	this.run=run;
    	run.finishedBy(this);
    	return this;
    }

}

Und das Ergebnis eines Laufs

package com.mymita.firmenlauf2013.model;
import java.util.Set;

import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;

import org.springframework.data.annotation.TypeAlias;
import org.springframework.data.neo4j.annotation.NodeEntity;
import org.springframework.data.neo4j.annotation.RelatedTo;

import com.google.common.collect.Sets;

@Getter
@NoArgsConstructor
@ToString(callSuper=true, exclude={"runners"})
@NodeEntity
@TypeAlias("Run")
public class Run extends AbstractEntity{

	private String time;

	@RelatedTo(type="finished_by")
	private Set<Runner> runners = Sets.newHashSet();

    public Run(String time) {
		this.time = time;
    }

	public Run finishedBy(Runner runner) {
		runners.add(runner);
		return this;
	}
}

Ergebnis der ganze Mühen ist ein Verzeichnis im target Ordner des Mavenprojekts, dessen Inhalt in einen Neo4j-Server (http://www.neo4j.org/download) kopiert wird. Ist der Server hochgefahren, kann man unter http://localhost:7474 das Ergebnis bestaunen🙂

Eine kleine Abfrage bringt dann auch alle meine lieben Kollegen vom „Running suckz“ Team zum Vorschein

neo4j_firmenlauf2013