diff --git a/run_orientdb.sh b/run_orientdb.sh index 9a1e960..774253f 100755 --- a/run_orientdb.sh +++ b/run_orientdb.sh @@ -1,8 +1,11 @@ +docker stop orientdb +docker rm orientdb + docker run -d -p 2424:2424 -p 2480:2480 \ -v orientdb_config:/orientdb/config \ -v orient_database:/orientdb/databases \ -v orient_backup:/orientdb/backup \ -e ORIENTDB_ROOT_PASSWORD=$1 \ --name orientdb \ wikitolearn/orientdb:latest diff --git a/src/main/java/org/wikitolearn/dao/PageDAO.java b/src/main/java/org/wikitolearn/dao/PageDAO.java index 8e00dc4..1263404 100644 --- a/src/main/java/org/wikitolearn/dao/PageDAO.java +++ b/src/main/java/org/wikitolearn/dao/PageDAO.java @@ -1,88 +1,92 @@ /** * */ package org.wikitolearn.dao; import com.orientechnologies.orient.core.metadata.schema.OClass; import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; + +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import com.tinkerpop.blueprints.impls.orient.OrientVertexType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import org.wikitolearn.models.Page; import org.wikitolearn.utils.DbConnection; /** * * This class will handles the operations on DB handling the Pages * @author alessandro * */ @Repository public class PageDAO { private static final Logger LOG = LoggerFactory.getLogger(PageDAO.class); @Autowired private DbConnection connection; /** * This method is used to create the class on the DB. * It creates an unique index on pageid to avoid duplicated. */ public void createDBClass() { LOG.info("Creating DB classes for PageDAO..."); OrientGraphNoTx graph = connection.getDbGraphNT(); try{ OrientVertexType vertex = graph.createVertexType("Page"); vertex.createProperty("pageid", OType.INTEGER).setMandatory(true); vertex.createIndex("pageid", OClass.INDEX_TYPE.UNIQUE, "pageid"); } catch( Exception e ) { LOG.error("Something went wrong during class creation. Operation will be rollbacked.", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } } /** * Insert all the given pages in the database as vertexes. * If there are duplicates all the insertion is rolled back. * @param pages List The pages to be inserted * @return boolean True if insertion was committed, false otherwise */ public Boolean insertPages(List pages){ OrientGraph graph = connection.getGraph(); LOG.info("Starting to insert pages..."); try{ for(Page p : pages){ - // Implicitly begins the transaction and create the class too - Vertex pageNode = graph.addVertex("class:Page", - "pageid", p.getPageid()); - pageNode.setProperty( "title", p.getTitle()); - pageNode.setProperty("pageRank", p.getPageRank()); + Map props = new HashMap<>(); + props.put("pageid", p.getPageid()); + props.put( "title", p.getTitle()); + props.put("pageRank", p.getPageRank()); + + Vertex pageNode = graph.addVertex("class:Page", props); LOG.info("Page inserted " + pageNode.toString()); } graph.commit(); LOG.info("Pages insertion committed"); return true; } catch (ORecordDuplicatedException or) { LOG.error("Some of the pages are duplicates. Operation will be rollbacked.", or.getMessage()); graph.rollback(); } catch( Exception e ) { LOG.error("Something went wrong during page insertion. Operation will be rollbacked.", e.getMessage()); graph.rollback(); }finally { graph.shutdown(); } return false; } } diff --git a/src/main/java/org/wikitolearn/dao/RevisionDAO.java b/src/main/java/org/wikitolearn/dao/RevisionDAO.java index 7b51824..e53cd65 100644 --- a/src/main/java/org/wikitolearn/dao/RevisionDAO.java +++ b/src/main/java/org/wikitolearn/dao/RevisionDAO.java @@ -1,138 +1,142 @@ package org.wikitolearn.dao; import com.orientechnologies.orient.core.metadata.schema.OClass; import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import com.tinkerpop.blueprints.impls.orient.OrientVertexType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import org.wikitolearn.models.Revision; import org.wikitolearn.models.User; import org.wikitolearn.utils.DbConnection; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; /** * This class will handle the Revision data processing on the OrientDB. * Created by valsdav on 14/03/17. */ @Repository public class RevisionDAO { private static final Logger LOG = LoggerFactory.getLogger(RevisionDAO.class); @Autowired private DbConnection connection; /** * This method is used to create the classes on the DB. * Moreover it creates a unique index on the userid property to avoid duplication. */ public void createDBClass() { LOG.info("Creating DB classes for RevisionDAO..."); OrientGraphNoTx graph = connection.getDbGraphNT(); try{ //Vertex type for the revision OrientVertexType vertex = graph.createVertexType("Revision"); vertex.createProperty("revid", OType.INTEGER).setMandatory(true); vertex.createIndex("revid", OClass.INDEX_TYPE.UNIQUE, "revid"); //Edge type for the created edge from User to Revision graph.createEdgeType("Author"); //Edge type to connect revision to parent revision graph.createEdgeType("ParentRevision"); //Edge type to connect last revision to page vertex graph.createEdgeType("LastRevision"); //Edge type to connect the first revision of a page graph.createEdgeType("FirstRevision"); } catch( Exception e ) { LOG.error("Something went wrong during class creation. Operation will be rollbacked.", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } } /** * This method will insert the revisions of one page, creating the link ParentRevision between them and * the link FirstRevision and LastRevision with the Page vertex. Moreover it connects the Users to * the revisions they have created. * This method must be used only for the firt INIT import, NOT for incremental inserction. * @param pageid * @param revs * @return */ public Boolean insertRevisions(int pageid, List revs){ OrientGraph graph = connection.getGraph(); LOG.info("Starting to insert revisions..."); HashMap revsNodes = new HashMap(); Vertex firstRev = null; Vertex lastRev = null; try{ for(Revision rev : revs){ - Vertex revNode = graph.addVertex("class:Revision", - "revid", rev.getRevid()); - revNode.setProperty("length", rev.getLength()); - revNode.setProperty("changeCoefficient", rev.getChangeCoefficient()); - revNode.setProperty( "currentMeanVote", rev.getCurrentMeanVote()); - revNode.setProperty( "currentVotesReliability", rev.getCurrentVotesReliability()); - revNode.setProperty( "currentNormalizedVotesReliability", rev.getCurrentNormalisesVotesReliability()); - revNode.setProperty( "totalMeanVote", rev.getTotalMeanVote()); - revNode.setProperty( "totalVotesReliability", rev.getTotalVotesReliability()); - revNode.setProperty( "totalNormalizedVotesReliability", rev.getTotalNormalisesVotesReliability()); + Map props = new HashMap<>(); + props.put("revid", rev.getRevid()); + props.put("length", rev.getLength()); + props.put("changeCoefficient", rev.getChangeCoefficient()); + props.put( "currentMeanVote", rev.getCurrentMeanVote()); + props.put( "currentVotesReliability", rev.getCurrentVotesReliability()); + props.put( "currentNormalizedVotesReliability", rev.getCurrentNormalisesVotesReliability()); + props.put( "totalMeanVote", rev.getTotalMeanVote()); + props.put( "totalVotesReliability", rev.getTotalVotesReliability()); + props.put( "totalNormalizedVotesReliability", rev.getTotalNormalisesVotesReliability()); + props.put("validated", rev.isValidated()); + + Vertex revNode = graph.addVertex("class:Revision", props); LOG.info("Revision inserted " + revNode.toString()); revsNodes.put(Integer.toString(rev.getRevid()), revNode); if (rev.getParentid() == 0){ firstRev = revNode; } if (lastRev==null || rev.getRevid() > (int) lastRev.getProperty("revid")){ lastRev = revNode; } //connecting the creator of the revisions Vertex userCreator = null; try{ userCreator = graph.getVertices("User.userid", rev.getUserid()).iterator().next(); } catch (NoSuchElementException e){ //if the user is not found we link it to the Anonimous user. userCreator = graph.getVertices("User.userid", "0" ).iterator().next(); } graph.addEdge("class:Author", userCreator, revNode, "Author"); } //now we have to create the the links between revisions for (Revision r : revs){ if (r.getParentid() != 0){ graph.addEdge("class:ParentRevision", revsNodes.get(Integer.toString(r.getRevid())), revsNodes.get(Integer.toString(r.getParentid())), "ParentRevision"); } } //now let's create the LastRevision and FirstRevision edges Vertex page = graph.getVertices("Page.pageid", pageid).iterator().next(); graph.addEdge("class:LastRevision", page, lastRev, "LastRevision"); graph.addEdge("class:FirstRevision", page, firstRev, "FirstRevision"); graph.commit(); LOG.info(String.format("Revisions of page %s insertion committed", pageid)); return true; } catch (ORecordDuplicatedException or) { LOG.error("Some of the pages are duplicates. Operation will be rollbacked.", or.getMessage()); graph.rollback(); } catch( Exception e ) { LOG.error("Something went wrong during user insertion. Operation will be rollbacked.", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } return false; } } diff --git a/src/main/java/org/wikitolearn/dao/UserDAO.java b/src/main/java/org/wikitolearn/dao/UserDAO.java index 57d5e32..84faae1 100644 --- a/src/main/java/org/wikitolearn/dao/UserDAO.java +++ b/src/main/java/org/wikitolearn/dao/UserDAO.java @@ -1,89 +1,92 @@ /** * */ package org.wikitolearn.dao; import com.orientechnologies.orient.core.metadata.schema.OClass; import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import com.tinkerpop.blueprints.impls.orient.OrientVertexType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import org.wikitolearn.models.Page; import org.wikitolearn.models.User; import org.wikitolearn.utils.DbConnection; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * @author alessandro * */ @Repository public class UserDAO { private static final Logger LOG = LoggerFactory.getLogger(UserDAO.class); @Autowired private DbConnection connection; /** * This method is used to create the class on the DB. * Moreover it creates a unique index on the userid property to avoid duplication. */ public void createDBClass() { LOG.info("Creating DB classes for RevisionDAO..."); OrientGraphNoTx graph = connection.getDbGraphNT(); try{ OrientVertexType vertex = graph.createVertexType("User"); vertex.createProperty("userid", OType.INTEGER).setMandatory(true); vertex.createIndex("userid", OClass.INDEX_TYPE.UNIQUE, "userid"); } catch( Exception e ) { LOG.error("Something went wrong during class creation. Operation will be rollbacked.", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } } /** * Insert all the given users in the database as vertexes. * If there are duplicates all the insertion is rolled back. * @param users List The pages to be inserted * @return boolean True if insertion was committed, false otherwise */ public Boolean insertUsers(List users){ OrientGraph graph = connection.getGraph(); LOG.info("Starting to insert users..."); try{ for(User p : users){ - // Implicitly begins the transaction and create the class too - Vertex userNode = graph.addVertex("class:User", - "userid", p.getUserid()); - userNode.setProperty( "username", p.getUsername()); - userNode.setProperty("votesReliability", p.getVotesReliability()); - userNode.setProperty("contributesReliability", p.getContributesReliability()); - userNode.setProperty("totalReliability", p.getTotalReliability()); + Map props = new HashMap<>(); + props.put("userid", p.getUserid()); + props.put( "username", p.getUsername()); + props.put("votesReliability", p.getVotesReliability()); + props.put("contributesReliability", p.getContributesReliability()); + props.put("totalReliability", p.getTotalReliability()); + + Vertex userNode = graph.addVertex("class:User", props); LOG.info("User inserted " + userNode.toString()); } graph.commit(); LOG.info("Users insertion committed"); return true; } catch (ORecordDuplicatedException or) { LOG.error("Some of the pages are duplicates. Operation will be rollbacked.", or.getMessage()); graph.rollback(); } catch( Exception e ) { LOG.error("Something went wrong during user insertion. Operation will be rollbacked.", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } return false; } } diff --git a/src/main/java/org/wikitolearn/models/Revision.java b/src/main/java/org/wikitolearn/models/Revision.java index ba4c0e5..9afbe24 100644 --- a/src/main/java/org/wikitolearn/models/Revision.java +++ b/src/main/java/org/wikitolearn/models/Revision.java @@ -1,130 +1,140 @@ package org.wikitolearn.models; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; /** * This class handles the data of the Revision of a page. * Created by valsdav on 14/03/17. */ @JsonIgnoreProperties(ignoreUnknown = true) public class Revision { private int revid; private int userid; private int parentid; private long length; private double changeCoefficient; private double currentMeanVote; private double currentVotesReliability; private double currentNormalisesVotesReliability; private double totalMeanVote; private double totalVotesReliability; private double totalNormalisesVotesReliability; + private boolean validated; public Revision() { } public Revision(int revid, int userid, int parentid, long length, double changeCoefficient, double currentMeanVote, double currentVotesReliability, double currentNormalisesVotesReliability, double totalMeanVote, - double totalVotesReliability, double totalNormalisesVotesReliability) { + double totalVotesReliability, double totalNormalisesVotesReliability, boolean validated) { this.revid = revid; this.userid = userid; this.parentid = parentid; this.length = length; this.changeCoefficient = changeCoefficient; this.currentMeanVote = currentMeanVote; this.currentVotesReliability = currentVotesReliability; this.currentNormalisesVotesReliability = currentNormalisesVotesReliability; this.totalMeanVote = totalMeanVote; this.totalVotesReliability = totalVotesReliability; this.totalNormalisesVotesReliability = totalNormalisesVotesReliability; + this.validated = validated; } public int getRevid() { return revid; } public void setRevid(int revid) { this.revid = revid; } public int getUserid() { return userid; } public void setUserid(int userid) { this.userid = userid; } public int getParentid() { return parentid; } public void setParentid(int parentid) { this.parentid = parentid; } public long getLength() { return length; } public void setLength(long length) { this.length = length; } public double getChangeCoefficient() { return changeCoefficient; } public void setChangeCoefficient(double changeCoefficient) { this.changeCoefficient = changeCoefficient; } public double getCurrentMeanVote() { return currentMeanVote; } public void setCurrentMeanVote(double currentMeanVote) { this.currentMeanVote = currentMeanVote; } public double getCurrentVotesReliability() { return currentVotesReliability; } public void setCurrentVotesReliability(double currentVotesReliability) { this.currentVotesReliability = currentVotesReliability; } public double getCurrentNormalisesVotesReliability() { return currentNormalisesVotesReliability; } public void setCurrentNormalisesVotesReliability(double currentNormalisesVotesReliability) { this.currentNormalisesVotesReliability = currentNormalisesVotesReliability; } public double getTotalMeanVote() { return totalMeanVote; } public void setTotalMeanVote(double totalMeanVote) { this.totalMeanVote = totalMeanVote; } public double getTotalVotesReliability() { return totalVotesReliability; } public void setTotalVotesReliability(double totalVotesReliability) { this.totalVotesReliability = totalVotesReliability; } public double getTotalNormalisesVotesReliability() { return totalNormalisesVotesReliability; } public void setTotalNormalisesVotesReliability(double totalNormalisesVotesReliability) { this.totalNormalisesVotesReliability = totalNormalisesVotesReliability; } + + public boolean isValidated() { + return validated; + } + + public void setValidated(boolean validated) { + this.validated = validated; + } }