diff --git a/src/main/java/org/wikitolearn/controllers/InitializerController.java b/src/main/java/org/wikitolearn/controllers/InitializerController.java index 8a2758f..36bf2aa 100644 --- a/src/main/java/org/wikitolearn/controllers/InitializerController.java +++ b/src/main/java/org/wikitolearn/controllers/InitializerController.java @@ -1,160 +1,162 @@ /** * */ package org.wikitolearn.controllers; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import com.orientechnologies.orient.core.sql.OCommandSQL; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.wikitolearn.dao.PageDAO; import org.wikitolearn.dao.RevisionDAO; import org.wikitolearn.dao.UserDAO; import org.wikitolearn.models.Page; import org.wikitolearn.models.Revision; import org.wikitolearn.models.User; import org.wikitolearn.services.mediawiki.PageMediaWikiController; import org.wikitolearn.services.mediawiki.RevisionMediaWikiController; import org.wikitolearn.services.mediawiki.UserMediaWikiController; import org.wikitolearn.utils.DbConnection; /** * @author alessandro * */ @RestController public class InitializerController { private static final Logger LOG = LoggerFactory.getLogger(InitializerController.class); @Autowired private PageMediaWikiController pageController; @Autowired private UserMediaWikiController userController; @Autowired private RevisionMediaWikiController revsController; @Autowired private PageDAO pageDao; @Autowired private UserDAO userDao; @Autowired private RevisionDAO revisionDAO; @Autowired private DbConnection dbConnection; private boolean initializedDB = false; @RequestMapping(value = "/init", method = RequestMethod.GET, produces = "application/json") public boolean initialize(@RequestParam("lang") String lang){ String apiUrl = "https://" + lang + ".wikitolearn.org/api.php"; // Initializing the DB schema, only the first time if (! initializedDB){ initializeDbClasses(); initializedDB = true; } CompletableFuture parallelInsertions = CompletableFuture.allOf(addAllPages(lang, apiUrl), addAllUsers(apiUrl)) .thenCompose(s -> addAllRevisions(lang, apiUrl)); try { return parallelInsertions.get(); } catch (InterruptedException | ExecutionException e) { LOG.error("Something went wrong during database initialization.", e.getMessage()); return false; } } /** * This methods initializes all the DAO Classes, creating the right types on the DB. */ private void initializeDbClasses(){ LOG.info("Creating DB lasses..."); pageDao.createDBClass(); userDao.createDBClass(); revisionDAO.createDBClass(); LOG.info("Completed creation of DB classes."); } /** * This methods inserts all the pages inside the DB querying the MediaWiki API. * * @return CompletableFuture */ @Async private CompletableFuture addAllPages( String lang, String apiUrl ){ List pages = pageController.getAllPages(apiUrl); LOG.info("Fetched all the pages"); boolean insertionResultPages = pageDao.insertPages(pages, lang); if(insertionResultPages){ LOG.info("Inserted pages"); }else{ LOG.error("Something went wrong during pages insertion"); } return CompletableFuture.completedFuture(insertionResultPages); } /** * This methods inserts all the users inside the DB querying the MediaWiki API. * * @return CompletableFuture */ @Async private CompletableFuture addAllUsers(String apiUrl){ List users = userController.getAllUsers(apiUrl); // Adding the Anonymous user users.add(new User("Anonimous", 0, 0, 0, 0)); LOG.info("Fetched all the users"); boolean insertionResultUsers = userDao.insertUsers(users); if(insertionResultUsers){ LOG.info("Inserted users"); }else{ LOG.error("Something went wrong during users insertion"); } return CompletableFuture.completedFuture(insertionResultUsers); } /** * This method inserts all the revisions for every page, creating the connections between them * and between the users that have written them. * * @return CompletableFuture */ private CompletableFuture addAllRevisions(String lang, String apiUrl){ OrientGraph graph = dbConnection.getGraph(); boolean revInsertionResult = false; try { - for (Vertex page : graph.getVertices("Page", new String[] {"lang"},new Object[] {lang})) { + for (Vertex page : (Iterable) graph.command(new OCommandSQL( + "SELECT FROM cluster:Pages_"+ lang)).execute()) { int pageid = page.getProperty("pageid"); LOG.info("Processing page: " + pageid); List revs = revsController.getAllRevisionForPage(apiUrl, pageid); revInsertionResult = revisionDAO.insertRevisions(pageid, revs); if(!revInsertionResult){ LOG.error("Something was wrong during the insertion of the revisions of page "+ pageid); } } } catch (Exception e){ LOG.error("Something bad has appended getting revisions", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } return CompletableFuture.completedFuture(revInsertionResult); } } diff --git a/src/main/java/org/wikitolearn/dao/PageDAO.java b/src/main/java/org/wikitolearn/dao/PageDAO.java index 0b57806..3d57b1a 100644 --- a/src/main/java/org/wikitolearn/dao/PageDAO.java +++ b/src/main/java/org/wikitolearn/dao/PageDAO.java @@ -1,94 +1,101 @@ /** * */ package org.wikitolearn.dao; import com.orientechnologies.orient.core.metadata.schema.OClass; import com.orientechnologies.orient.core.metadata.schema.OType; +import com.orientechnologies.orient.core.sql.OCommandSQL; import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; import java.util.HashMap; import java.util.List; import java.util.Map; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import com.tinkerpop.blueprints.impls.orient.OrientVertex; import com.tinkerpop.blueprints.impls.orient.OrientVertexType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import org.wikitolearn.models.Page; import org.wikitolearn.utils.DbConnection; /** * * This class will handles the operations on DB handling the Pages * @author alessandro * */ @Repository public class PageDAO { private static final Logger LOG = LoggerFactory.getLogger(PageDAO.class); @Autowired private DbConnection connection; /** * This method is used to create the class on the DB. * It creates an unique index on pageid to avoid duplicated. */ public void createDBClass() { LOG.info("Creating DB classes for PageDAO..."); OrientGraphNoTx graph = connection.getDbGraphNT(); try{ - OrientVertexType vertex = graph.createVertexType("Page"); + OrientVertexType vertex = graph.createVertexType("Page",1); vertex.createProperty("pageid", OType.INTEGER).setMandatory(true); - vertex.createIndex("pageid", OClass.INDEX_TYPE.UNIQUE, "pageid"); + vertex.createProperty("lang", OType.STRING).setMandatory(true); + vertex.createIndex("page_lang", OClass.INDEX_TYPE.UNIQUE, "pageid", "lang"); + //now we want to add clusters + graph.command(new OCommandSQL("ALTER CLASS Page ADDCLUSTER Pages_it")).execute(); + graph.command(new OCommandSQL("ALTER CLASS Page ADDCLUSTER Pages_en")).execute(); + //adding the clusters to the class Page + //graph.getRawGraph().getMetadata().getSchema().reload(); } catch( Exception e ) { LOG.error("Something went wrong during class creation. Operation will be rollbacked.", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } } /** * Insert all the given pages in the database as vertexes. * If there are duplicates all the insertion is rolled back. * @param pages List The pages to be inserted * @return boolean True if insertion was committed, false otherwise */ public Boolean insertPages(List pages, String lang){ OrientGraph graph = connection.getGraph(); LOG.info("Starting to insert pages..."); try{ for(Page p : pages){ Map props = new HashMap<>(); props.put("pageid", p.getPageid()); props.put( "title", p.getTitle()); props.put("lang", lang); props.put("pageRank", p.getPageRank()); - OrientVertex pageNode = graph.addVertex("class:Page", props); + OrientVertex pageNode = graph.addVertex("class:Page,cluster:Pages_"+lang, props); LOG.info("Page inserted " + pageNode.toString()); } graph.commit(); LOG.info("Pages insertion committed"); return true; } catch (ORecordDuplicatedException or) { LOG.error("Some of the pages are duplicates. Operation will be rollbacked.", or.getMessage()); graph.rollback(); } catch( Exception e ) { LOG.error("Something went wrong during page insertion. Operation will be rollbacked.", e.getMessage()); graph.rollback(); }finally { graph.shutdown(); } return false; } } diff --git a/src/main/java/org/wikitolearn/dao/UserDAO.java b/src/main/java/org/wikitolearn/dao/UserDAO.java index 9027cfd..aec01d1 100644 --- a/src/main/java/org/wikitolearn/dao/UserDAO.java +++ b/src/main/java/org/wikitolearn/dao/UserDAO.java @@ -1,93 +1,93 @@ /** * */ package org.wikitolearn.dao; import com.orientechnologies.orient.core.metadata.schema.OClass; import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import com.tinkerpop.blueprints.impls.orient.OrientVertexType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import org.wikitolearn.models.User; import org.wikitolearn.utils.DbConnection; import java.util.HashMap; import java.util.List; import java.util.Map; /** * @author alessandro * */ @Repository public class UserDAO { private static final Logger LOG = LoggerFactory.getLogger(UserDAO.class); @Autowired private DbConnection connection; /** * This method is used to create the class on the DB. * Moreover it creates a unique index on the userid property to avoid duplication. */ public void createDBClass() { LOG.info("Creating DB classes for RevisionDAO..."); OrientGraphNoTx graph = connection.getDbGraphNT(); try{ OrientVertexType vertex = graph.createVertexType("User"); vertex.createProperty("userid", OType.INTEGER).setMandatory(true); vertex.createIndex("userid", OClass.INDEX_TYPE.UNIQUE, "userid"); } catch( Exception e ) { LOG.error("Something went wrong during class creation. Operation will be rollbacked.", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } } /** * Insert all the given users in the database as vertexes. * If there are duplicates all the insertion is rolled back. * @param users List The pages to be inserted * @return boolean True if insertion was committed, false otherwise */ public Boolean insertUsers(List users){ OrientGraph graph = connection.getGraph(); LOG.info("Starting to insert users..."); try{ for(User p : users) { try { Map props = new HashMap<>(); props.put("userid", p.getUserid()); props.put("username", p.getUsername()); props.put("votesReliability", p.getVotesReliability()); props.put("contributesReliability", p.getContributesReliability()); props.put("totalReliability", p.getTotalReliability()); Vertex userNode = graph.addVertex("class:User", props); graph.commit(); LOG.info("User inserted " + userNode.toString()); } catch (ORecordDuplicatedException or) { - LOG.error("Some of the pages are duplicates. Operation will be rollbacked.", or.getMessage()); + LOG.error("The user is already in the DB. Operation will be rollbacked.", or.getMessage()); graph.rollback(); } } LOG.info("Users insertion ended"); graph.shutdown(); return true; } catch( Exception e ) { LOG.error("Something went wrong during user insertion. Operation will be rollbacked.", e.getMessage()); graph.rollback(); graph.shutdown(); } return false; } }