diff --git a/src/main/java/org/wikitolearn/controllers/InitializerController.java b/src/main/java/org/wikitolearn/controllers/InitializerController.java index de773ba..c6e817e 100644 --- a/src/main/java/org/wikitolearn/controllers/InitializerController.java +++ b/src/main/java/org/wikitolearn/controllers/InitializerController.java @@ -1,140 +1,142 @@ /** * */ package org.wikitolearn.controllers; import java.util.List; import com.orientechnologies.orient.core.Orient; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; import org.wikitolearn.controllers.mediawikiClient.PageMediaWikiController; import org.wikitolearn.controllers.mediawikiClient.RevisionMediaWikiController; import org.wikitolearn.controllers.mediawikiClient.UserMediaWikiController; import org.wikitolearn.dao.PageDAO; import org.wikitolearn.dao.RevisionDAO; import org.wikitolearn.dao.UserDAO; import org.wikitolearn.models.Page; import org.wikitolearn.models.Revision; import org.wikitolearn.models.User; import org.wikitolearn.utils.DbConnection; /** * @author alessandro * */ @RestController public class InitializerController { private static final Logger LOG = LoggerFactory.getLogger(InitializerController.class); @Autowired private PageMediaWikiController pageController; @Autowired private UserMediaWikiController userController; @Autowired private RevisionMediaWikiController revsController; @Autowired private PageDAO pageDao; @Autowired private UserDAO userDao; @Autowired private RevisionDAO revisionDAO; @Autowired private DbConnection dbConnection; @RequestMapping(value = "/init", method = RequestMethod.GET, produces = "application/json") public String initialize(){ //Inizitializing the DB schema, only the first time initializeDbClasses(); boolean resultPages = addAllPages(); boolean resultUsers = addAllUsers(); boolean resultRevs = addAllRevisions(); return "Pages: " + Boolean.toString(resultPages) + "\nUsers: "+ Boolean.toString(resultUsers) + "\nRevisions: "+ Boolean.toString(resultRevs); } /** * This methods initializes all the DAO Classes, creating the right types on the DB. */ private void initializeDbClasses(){ LOG.info("Create DB Classes..."); pageDao.createDBClass(); userDao.createDBClass(); revisionDAO.createDBClass(); LOG.info("Create DB Classes...DONE"); } /** * This methods inserts all the pages inside the DB quering the mediawiki api. */ private boolean addAllPages(){ List pages = pageController.getAllPages("https://en.wikitolearn.org/api.php"); LOG.info("Fetched all the pages"); boolean insertionResultPages = pageDao.insertPages(pages); if(insertionResultPages){ LOG.info("Inserted pages"); }else{ LOG.error("Something went wrong during pages insertion"); } return insertionResultPages; } /** * This methods inserts all the users inside the DB quering the mediawiki api. */ private boolean addAllUsers(){ List users = userController.getAllUsers("https://en.wikitolearn.org/api.php"); + //adding the Anonimous user + users.add(new User("Anonimous", 0, 0, 0, 0)); LOG.info("Fetched all the users"); boolean insertionResultUsers = userDao.insertUsers(users); if(insertionResultUsers){ LOG.info("Inserted users"); }else{ LOG.error("Something went wrong during users insertion"); } return insertionResultUsers; } /** * This method inserts all the revisions for every page, creating the connections between them * and between the users that have written them. * @return */ private boolean addAllRevisions(){ OrientGraph graph = dbConnection.getGraph(); try { for (Vertex page : graph.getVerticesOfClass("Page")) { int pageid = page.getProperty("pageid"); LOG.info("Processing page: " + pageid); List revs = revsController.getAllRevisionForPage("https://en.wikitolearn.org/api.php", pageid); boolean revInsertionResult = revisionDAO.insertRevisions(pageid, revs); if(!revInsertionResult){ LOG.error("Something was wrong during the insertion of the revisions of page "+ pageid); } } return true; } catch (Exception e){ LOG.error("Something bad has appended getting revisions", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } return false; } } diff --git a/src/main/java/org/wikitolearn/dao/RevisionDAO.java b/src/main/java/org/wikitolearn/dao/RevisionDAO.java index 23ccc53..7b51824 100644 --- a/src/main/java/org/wikitolearn/dao/RevisionDAO.java +++ b/src/main/java/org/wikitolearn/dao/RevisionDAO.java @@ -1,139 +1,138 @@ package org.wikitolearn.dao; import com.orientechnologies.orient.core.metadata.schema.OClass; import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import com.tinkerpop.blueprints.impls.orient.OrientVertexType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import org.wikitolearn.models.Revision; import org.wikitolearn.models.User; import org.wikitolearn.utils.DbConnection; import java.util.HashMap; import java.util.List; import java.util.NoSuchElementException; /** * This class will handle the Revision data processing on the OrientDB. * Created by valsdav on 14/03/17. */ @Repository public class RevisionDAO { private static final Logger LOG = LoggerFactory.getLogger(RevisionDAO.class); @Autowired private DbConnection connection; /** * This method is used to create the classes on the DB. * Moreover it creates a unique index on the userid property to avoid duplication. */ public void createDBClass() { LOG.info("Creating DB classes for RevisionDAO..."); OrientGraphNoTx graph = connection.getDbGraphNT(); try{ //Vertex type for the revision OrientVertexType vertex = graph.createVertexType("Revision"); vertex.createProperty("revid", OType.INTEGER).setMandatory(true); vertex.createIndex("revid", OClass.INDEX_TYPE.UNIQUE, "revid"); //Edge type for the created edge from User to Revision - graph.createEdgeType("CreatedBy"); + graph.createEdgeType("Author"); //Edge type to connect revision to parent revision graph.createEdgeType("ParentRevision"); //Edge type to connect last revision to page vertex graph.createEdgeType("LastRevision"); //Edge type to connect the first revision of a page graph.createEdgeType("FirstRevision"); } catch( Exception e ) { LOG.error("Something went wrong during class creation. Operation will be rollbacked.", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } } /** * This method will insert the revisions of one page, creating the link ParentRevision between them and * the link FirstRevision and LastRevision with the Page vertex. Moreover it connects the Users to * the revisions they have created. * This method must be used only for the firt INIT import, NOT for incremental inserction. * @param pageid * @param revs * @return */ public Boolean insertRevisions(int pageid, List revs){ OrientGraph graph = connection.getGraph(); LOG.info("Starting to insert revisions..."); HashMap revsNodes = new HashMap(); Vertex firstRev = null; Vertex lastRev = null; try{ for(Revision rev : revs){ Vertex revNode = graph.addVertex("class:Revision", "revid", rev.getRevid()); revNode.setProperty("length", rev.getLength()); revNode.setProperty("changeCoefficient", rev.getChangeCoefficient()); revNode.setProperty( "currentMeanVote", rev.getCurrentMeanVote()); revNode.setProperty( "currentVotesReliability", rev.getCurrentVotesReliability()); revNode.setProperty( "currentNormalizedVotesReliability", rev.getCurrentNormalisesVotesReliability()); revNode.setProperty( "totalMeanVote", rev.getTotalMeanVote()); revNode.setProperty( "totalVotesReliability", rev.getTotalVotesReliability()); revNode.setProperty( "totalNormalizedVotesReliability", rev.getTotalNormalisesVotesReliability()); LOG.info("Revision inserted " + revNode.toString()); revsNodes.put(Integer.toString(rev.getRevid()), revNode); if (rev.getParentid() == 0){ firstRev = revNode; } if (lastRev==null || rev.getRevid() > (int) lastRev.getProperty("revid")){ lastRev = revNode; } //connecting the creator of the revisions Vertex userCreator = null; try{ userCreator = graph.getVertices("User.userid", rev.getUserid()).iterator().next(); } catch (NoSuchElementException e){ - LOG.error("User creator of the revision not found!", e.getMessage()); - return false; + //if the user is not found we link it to the Anonimous user. + userCreator = graph.getVertices("User.userid", "0" ).iterator().next(); } - graph.addEdge("class:CreatedBy", revNode, userCreator, "CreatedBy"); - + graph.addEdge("class:Author", userCreator, revNode, "Author"); } //now we have to create the the links between revisions for (Revision r : revs){ if (r.getParentid() != 0){ graph.addEdge("class:ParentRevision", revsNodes.get(Integer.toString(r.getRevid())), revsNodes.get(Integer.toString(r.getParentid())), "ParentRevision"); } } //now let's create the LastRevision and FirstRevision edges Vertex page = graph.getVertices("Page.pageid", pageid).iterator().next(); graph.addEdge("class:LastRevision", page, lastRev, "LastRevision"); graph.addEdge("class:FirstRevision", page, firstRev, "FirstRevision"); graph.commit(); LOG.info(String.format("Revisions of page %s insertion committed", pageid)); return true; } catch (ORecordDuplicatedException or) { LOG.error("Some of the pages are duplicates. Operation will be rollbacked.", or.getMessage()); graph.rollback(); } catch( Exception e ) { LOG.error("Something went wrong during user insertion. Operation will be rollbacked.", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } return false; } }