diff --git a/src/main/java/org/wikitolearn/controllers/InitializerController.java b/src/main/java/org/wikitolearn/controllers/InitializerController.java index 36bf2aa..b4e932c 100644 --- a/src/main/java/org/wikitolearn/controllers/InitializerController.java +++ b/src/main/java/org/wikitolearn/controllers/InitializerController.java @@ -1,162 +1,153 @@ /** * */ package org.wikitolearn.controllers; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import com.orientechnologies.orient.core.sql.OCommandSQL; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.wikitolearn.dao.PageDAO; import org.wikitolearn.dao.RevisionDAO; import org.wikitolearn.dao.UserDAO; import org.wikitolearn.models.Page; import org.wikitolearn.models.Revision; import org.wikitolearn.models.User; import org.wikitolearn.services.mediawiki.PageMediaWikiController; import org.wikitolearn.services.mediawiki.RevisionMediaWikiController; import org.wikitolearn.services.mediawiki.UserMediaWikiController; import org.wikitolearn.utils.DbConnection; /** * @author alessandro * */ @RestController public class InitializerController { private static final Logger LOG = LoggerFactory.getLogger(InitializerController.class); @Autowired private PageMediaWikiController pageController; @Autowired private UserMediaWikiController userController; @Autowired private RevisionMediaWikiController revsController; @Autowired private PageDAO pageDao; @Autowired private UserDAO userDao; @Autowired private RevisionDAO revisionDAO; @Autowired private DbConnection dbConnection; private boolean initializedDB = false; @RequestMapping(value = "/init", method = RequestMethod.GET, produces = "application/json") public boolean initialize(@RequestParam("lang") String lang){ String apiUrl = "https://" + lang + ".wikitolearn.org/api.php"; // Initializing the DB schema, only the first time if (! initializedDB){ initializeDbClasses(); initializedDB = true; } CompletableFuture parallelInsertions = CompletableFuture.allOf(addAllPages(lang, apiUrl), addAllUsers(apiUrl)) .thenCompose(s -> addAllRevisions(lang, apiUrl)); try { return parallelInsertions.get(); } catch (InterruptedException | ExecutionException e) { LOG.error("Something went wrong during database initialization.", e.getMessage()); return false; } } /** * This methods initializes all the DAO Classes, creating the right types on the DB. */ private void initializeDbClasses(){ LOG.info("Creating DB lasses..."); pageDao.createDBClass(); userDao.createDBClass(); revisionDAO.createDBClass(); LOG.info("Completed creation of DB classes."); } /** * This methods inserts all the pages inside the DB querying the MediaWiki API. * * @return CompletableFuture */ @Async private CompletableFuture addAllPages( String lang, String apiUrl ){ List pages = pageController.getAllPages(apiUrl); LOG.info("Fetched all the pages"); boolean insertionResultPages = pageDao.insertPages(pages, lang); if(insertionResultPages){ LOG.info("Inserted pages"); }else{ LOG.error("Something went wrong during pages insertion"); } return CompletableFuture.completedFuture(insertionResultPages); } /** * This methods inserts all the users inside the DB querying the MediaWiki API. * * @return CompletableFuture */ @Async private CompletableFuture addAllUsers(String apiUrl){ List users = userController.getAllUsers(apiUrl); // Adding the Anonymous user users.add(new User("Anonimous", 0, 0, 0, 0)); LOG.info("Fetched all the users"); boolean insertionResultUsers = userDao.insertUsers(users); if(insertionResultUsers){ LOG.info("Inserted users"); }else{ LOG.error("Something went wrong during users insertion"); } return CompletableFuture.completedFuture(insertionResultUsers); } /** * This method inserts all the revisions for every page, creating the connections between them * and between the users that have written them. * * @return CompletableFuture */ private CompletableFuture addAllRevisions(String lang, String apiUrl){ - OrientGraph graph = dbConnection.getGraph(); boolean revInsertionResult = false; - try { - for (Vertex page : (Iterable) graph.command(new OCommandSQL( - "SELECT FROM cluster:Pages_"+ lang)).execute()) { - int pageid = page.getProperty("pageid"); - LOG.info("Processing page: " + pageid); - List revs = revsController.getAllRevisionForPage(apiUrl, pageid); - revInsertionResult = revisionDAO.insertRevisions(pageid, revs); - if(!revInsertionResult){ - LOG.error("Something was wrong during the insertion of the revisions of page "+ pageid); - } + for (Vertex page : pageDao.getPagesIteratorFromCluster(lang)) { + int pageid = page.getProperty("pageid"); + LOG.info("Processing page: " + pageid); + List revs = revsController.getAllRevisionForPage(apiUrl, pageid); + revInsertionResult = revisionDAO.insertRevisions(pageid, revs); + if(!revInsertionResult){ + LOG.error("Something was wrong during the insertion of the revisions of page "+ pageid); } - } catch (Exception e){ - LOG.error("Something bad has appended getting revisions", e.getMessage()); - graph.rollback(); - } finally { - graph.shutdown(); } return CompletableFuture.completedFuture(revInsertionResult); } } diff --git a/src/main/java/org/wikitolearn/dao/PageDAO.java b/src/main/java/org/wikitolearn/dao/PageDAO.java index 06c3b33..9b9ca93 100644 --- a/src/main/java/org/wikitolearn/dao/PageDAO.java +++ b/src/main/java/org/wikitolearn/dao/PageDAO.java @@ -1,100 +1,120 @@ /** * */ package org.wikitolearn.dao; import com.orientechnologies.orient.core.metadata.schema.OClass; import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.sql.OCommandSQL; import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; import com.tinkerpop.blueprints.impls.orient.OrientGraph; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import com.tinkerpop.blueprints.impls.orient.OrientVertex; import com.tinkerpop.blueprints.impls.orient.OrientVertexType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import org.wikitolearn.models.Page; import org.wikitolearn.utils.DbConnection; /** * * This class will handles the operations on DB handling the Pages * @author alessandro * */ @Repository public class PageDAO { private static final Logger LOG = LoggerFactory.getLogger(PageDAO.class); @Autowired private DbConnection connection; /** * This method is used to create the class on the DB. * It creates an unique index on pageid to avoid duplicated. */ public void createDBClass() { LOG.info("Creating DB classes for PageDAO..."); OrientGraphNoTx graph = connection.getDbGraphNT(); try{ OrientVertexType vertex = graph.createVertexType("Page",1); vertex.createProperty("pageid", OType.INTEGER).setMandatory(true); vertex.createProperty("lang", OType.STRING).setMandatory(true); vertex.createIndex("page_lang", OClass.INDEX_TYPE.UNIQUE, "pageid", "lang"); //now we want to add clusters graph.command(new OCommandSQL("ALTER CLASS Page ADDCLUSTER Pages_it")).execute(); graph.command(new OCommandSQL("ALTER CLASS Page ADDCLUSTER Pages_en")).execute(); //adding the clusters to the class Page //graph.getRawGraph().getMetadata().getSchema().reload(); } catch( Exception e ) { LOG.error("Something went wrong during class creation. Operation will be rollbacked.", e.getMessage()); graph.rollback(); } finally { graph.shutdown(); } } /** * Insert all the given pages in the database as vertexes. * If there are duplicates all the insertion is rolled back. * @param pages List The pages to be inserted * @return boolean True if insertion was committed, false otherwise */ public Boolean insertPages(List pages, String lang){ OrientGraph graph = connection.getGraph(); LOG.info("Starting to insert pages..."); try{ for(Page p : pages){ Map props = new HashMap<>(); props.put("pageid", p.getPageid()); props.put( "title", p.getTitle()); props.put("lang", lang); props.put("pageRank", p.getPageRank()); OrientVertex pageNode = graph.addVertex("class:Page,cluster:Pages_"+lang, props); LOG.info("Page inserted " + pageNode.toString()); } graph.commit(); LOG.info("Pages insertion committed"); return true; } catch (ORecordDuplicatedException or) { LOG.error("Some of the pages are duplicates. Operation will be rollbacked.", or.getMessage()); graph.rollback(); } catch( Exception e ) { LOG.error("Something went wrong during page insertion. Operation will be rollbacked.", e.getMessage()); graph.rollback(); }finally { graph.shutdown(); } return false; } - + + /** + * This methods returns an Iterable over all the pages belonging to a certain cluster, + * so coming from the same lang domain. + * @param lang lang of the cluster + * @return Iterable with all the pages of the cluster + */ + public Iterable getPagesIteratorFromCluster(String lang){ + OrientGraph graph = connection.getGraph(); + Iterable result = null; + try { + result = (Iterable) graph.command(new OCommandSQL( + "SELECT FROM cluster:Pages_"+ lang)).execute(); + } catch (Exception e){ + LOG.error("Something went wrong during quering for pages.", e.getMessage()); + } finally{ + graph.shutdown(); + } + return result; + } }