diff --git a/src/main/java/org/wikitolearn/controllers/InitializerController.java b/src/main/java/org/wikitolearn/controllers/InitializerController.java index f53bb4a..6871c3b 100644 --- a/src/main/java/org/wikitolearn/controllers/InitializerController.java +++ b/src/main/java/org/wikitolearn/controllers/InitializerController.java @@ -1,83 +1,87 @@ /** * */ package org.wikitolearn.controllers; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.wikitolearn.dao.MetadataDAO; import org.wikitolearn.dao.PageDAO; import org.wikitolearn.dao.RevisionDAO; import org.wikitolearn.dao.UserDAO; import org.wikitolearn.services.PageService; import org.wikitolearn.services.RevisionService; import org.wikitolearn.services.UserService; /** * * @author aletundo, valsdav * */ @RestController public class InitializerController { private static final Logger LOG = LoggerFactory.getLogger(InitializerController.class); @Autowired private PageService pageService; @Autowired private UserService userService; @Autowired private RevisionService revisionService; + @Autowired + private MetadataDAO metadataDAO; @Autowired private PageDAO pageDao; @Autowired private UserDAO userDao; @Autowired private RevisionDAO revisionDAO; private boolean initializedDB = false; @RequestMapping(value = "/init", method = RequestMethod.GET, produces = "application/json") public boolean initialize(@RequestParam("lang") String lang){ String apiUrl = "https://" + lang + ".wikitolearn.org/api.php"; // Initializing the DB schema, only the first time if (! initializedDB){ initializeDbClasses(); initializedDB = true; } CompletableFuture parallelInsertions = CompletableFuture.allOf(pageService.addAllPages(lang, apiUrl), userService.addAllUsers(apiUrl)) .thenCompose(s -> revisionService.addAllRevisions(lang, apiUrl)); try { return parallelInsertions.get(); } catch (InterruptedException | ExecutionException e) { LOG.error("Something went wrong during database initialization. {}", e.getMessage()); return false; } } /** * This methods initializes all the DAO Classes, creating the right types on the DB. * @return void */ private void initializeDbClasses(){ LOG.info("Creating Database classes..."); + metadataDAO.createDatabaseClass(); pageDao.createDatabaseClass(); userDao.createDatabaseClass(); revisionDAO.createDatabaseClass(); LOG.info("Completed creation of database classes."); } } diff --git a/src/main/java/org/wikitolearn/dao/MetadataDAO.java b/src/main/java/org/wikitolearn/dao/MetadataDAO.java new file mode 100644 index 0000000..04bea59 --- /dev/null +++ b/src/main/java/org/wikitolearn/dao/MetadataDAO.java @@ -0,0 +1,52 @@ +package org.wikitolearn.dao; + +import com.orientechnologies.orient.core.metadata.schema.OClass; +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.orientechnologies.orient.core.sql.OCommandSQL; +import com.sun.org.apache.xpath.internal.operations.Bool; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertex; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; +import org.springframework.stereotype.Repository; + +import java.util.Date; + +/** + * This class represents Metadata nodes in the DB. There is a unique Metadata node, + * as a entrypoint, and a chain of Process nodes, saving some useful information at every + * process in the rating engine. For example we can save the number of fetched pages or + * saved user votes. + * Created by valsdav on 21/03/17. + */ +@Repository +public class MetadataDAO extends GenericDAO { + + /** + * This method creates the classes Metadata and Process in the DB. + * The Metadata node is the entrypoint for the chain of Processes, via the + * edges LastProcess and PreviousProcess + */ + @Override + public void createDatabaseClass() { + LOG.info("Creating DB classes for MetadataDAO..."); + OrientGraphNoTx graph = connection.getGraphNT(); + try{ + graph.createVertexType("Metadata",1); + OrientVertexType processVertex = graph.createVertexType("Process",1); + processVertex.createProperty("timestamp", OType.DATETIME).setMandatory(true); + graph.createEdgeType("LastProcess"); + graph.createEdgeType("PreviousProcess"); + + //We want also to create the singleton node for Metadata. + OrientVertex metadata_main = graph.addVertex("class:Metadata"); + metadata_main.setProperty("creation_date", new Date()); + } catch( Exception e ) { + LOG.error("Something went wrong during class creation. {}.", e.getMessage()); + } finally { + graph.shutdown(); + } + } + +} + + diff --git a/src/main/java/org/wikitolearn/dao/RevisionDAO.java b/src/main/java/org/wikitolearn/dao/RevisionDAO.java index 0182659..b82fd55 100644 --- a/src/main/java/org/wikitolearn/dao/RevisionDAO.java +++ b/src/main/java/org/wikitolearn/dao/RevisionDAO.java @@ -1,156 +1,156 @@ package org.wikitolearn.dao; import com.orientechnologies.orient.core.metadata.schema.OClass; import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.sql.OCommandSQL; import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.impls.orient.OrientGraph; import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; import com.tinkerpop.blueprints.impls.orient.OrientVertex; import com.tinkerpop.blueprints.impls.orient.OrientVertexType; import org.springframework.stereotype.Repository; import org.wikitolearn.models.Revision; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; /** * * @author aletundo, valsdav * */ @Repository public class RevisionDAO extends GenericDAO{ /** * This method is used to create the classes on the DB. * Moreover it creates a unique index on the userid property to avoid duplication. * @return void */ @Override public void createDatabaseClass() { LOG.info("Creating DB classes for RevisionDAO..."); OrientGraphNoTx graph = connection.getGraphNT(); try{ // Vertex type for the revision - OrientVertexType vertex = graph.createVertexType("Revision"); + OrientVertexType vertex = graph.createVertexType("Revision", 1); vertex.createProperty("revid", OType.INTEGER).setMandatory(true); vertex.createProperty("lang", OType.STRING).setMandatory(true); vertex.createIndex("revid", OClass.INDEX_TYPE.UNIQUE, "revid", "lang"); // Creating clusters for Revision class graph.command(new OCommandSQL("ALTER CLASS Revision ADDCLUSTER Revs_it")).execute(); graph.command(new OCommandSQL("ALTER CLASS Revision ADDCLUSTER Revs_en")).execute(); // Edge type for the created edge from User to Revision graph.createEdgeType("Author"); // Edge type to connect revision to parent revision graph.createEdgeType("ParentRevision"); // Edge type to connect last revision to page vertex graph.createEdgeType("LastRevision"); // Edge type to connect the first revision of a page graph.createEdgeType("FirstRevision"); } catch( Exception e ) { LOG.error("Something went wrong during class creation. {}.", e.getMessage()); } finally { graph.shutdown(); } } /** * This method will insert the revisions of one page, creating the link ParentRevision between them and * the link FirstRevision and LastRevision with the Page vertex. Moreover it connects the Users to * the revisions they have created. * This method must be used only for the first INIT import, NOT for incremental insertion. * @param pageId * @param revs * @return */ public Boolean insertRevisions(int pageId, List revs, String lang){ OrientGraphNoTx graph = connection.getGraphNT(); LOG.info("Starting to insert revisions..."); HashMap revsNodes = new HashMap(); Vertex firstRev = null; Vertex lastRev = null; try{ for(Revision rev : revs){ Map props = new HashMap<>(); props.put("revid", rev.getRevid()); props.put("lang", lang); props.put("length", rev.getLength()); props.put("changeCoefficient", rev.getChangeCoefficient()); props.put( "currentMeanVote", rev.getCurrentMeanVote()); props.put( "currentVotesReliability", rev.getCurrentVotesReliability()); props.put( "currentNormalizedVotesReliability", rev.getCurrentNormalisesVotesReliability()); props.put( "totalMeanVote", rev.getTotalMeanVote()); props.put( "totalVotesReliability", rev.getTotalVotesReliability()); props.put( "totalNormalizedVotesReliability", rev.getTotalNormalisesVotesReliability()); props.put("validated", rev.isValidated()); Vertex revNode = graph.addVertex("class:Revision,cluster:Revs_"+lang, props); //LOG.info("Revision inserted {}.", revNode.toString()); revsNodes.put(Integer.toString(rev.getRevid()), revNode); if (rev.getParentid() == 0){ firstRev = revNode; } if (lastRev==null || rev.getRevid() > (int) lastRev.getProperty("revid")){ lastRev = revNode; } // Connecting the creator of the revisions Vertex userCreator = null; try{ userCreator = graph.getVertices("User.userid", rev.getUserid()).iterator().next(); } catch (NoSuchElementException e){ //if the user is not found we link it to the Anonymous user. userCreator = graph.getVertices("User.userid", "0" ).iterator().next(); } graph.addEdge("class:Author", userCreator, revNode, "Author"); } // Now we have to create the the links between revisions for (Revision r : revs){ if (r.getParentid() != 0){ graph.addEdge("class:ParentRevision", revsNodes.get(Integer.toString(r.getRevid())), revsNodes.get(Integer.toString(r.getParentid())), "ParentRevision"); } } // Now let's create the LastRevision and FirstRevision edges Vertex page = graph.getVertices("Page.pageid", pageId).iterator().next(); graph.addEdge("class:LastRevision", page, lastRev, "LastRevision"); graph.addEdge("class:FirstRevision", page, firstRev, "FirstRevision"); LOG.info("Revisions of page {} insertion committed", pageId); return true; } catch (ORecordDuplicatedException or) { LOG.error("Some of the pages are duplicates. {}", or.getMessage()); } catch( Exception e ) { LOG.error("Something went wrong during user insertion. {}", e.getMessage()); } finally { graph.shutdown(); } return false; } /** * This methods returns an Iterable over all the Revisions belonging to a certain cluster, * so coming from the same language domain. * @param lang String The language of the cluster * @return result Iterable with all the revisions of the cluster */ public Iterable getRevisionsIteratorFromCluster(OrientGraph graph, String lang){ Iterable result = null; try { result = (Iterable) graph.command(new OCommandSQL( "SELECT FROM cluster:Revs_"+ lang)).execute(); } catch (Exception e){ LOG.error("Something went wrong during quering for revisions. {}", e.getMessage()); } return result; } }