diff --git a/src/main/java/org/wikitolearn/wikirating/repository/CourseLevelThreeRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/CourseLevelThreeRepository.java index 0af4500..1e43d49 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/CourseLevelThreeRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/CourseLevelThreeRepository.java @@ -1,14 +1,24 @@ /** * */ package org.wikitolearn.wikirating.repository; +import org.springframework.data.neo4j.annotation.Query; +import org.springframework.data.repository.query.Param; import org.wikitolearn.wikirating.model.graph.CourseLevelThree; /** * @author aletundo * */ public interface CourseLevelThreeRepository extends PageRepository { + /** + * Update the LAST_REVISION relationship deleting the existing edge and + * creating a new one. + */ + @Query("MATCH (p:CourseLevelThree {langPageId: {id}})-[lr:LAST_REVISION]->(r1:Revision)<-[pp:PREVIOUS_REVISION]-(r2:Revision) " + + "DELETE lr CREATE (p)-[:LAST_REVISION]->(r2)") + void updateLastRevision(@Param("id") String langPageId); + } diff --git a/src/main/java/org/wikitolearn/wikirating/repository/MetadataRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/MetadataRepository.java index 9c4fdfa..5314130 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/MetadataRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/MetadataRepository.java @@ -1,27 +1,27 @@ package org.wikitolearn.wikirating.repository; import org.springframework.data.neo4j.annotation.Query; import org.springframework.data.neo4j.repository.GraphRepository; import org.wikitolearn.wikirating.model.graph.Metadata; import org.wikitolearn.wikirating.util.enums.MetadataType; /** * @author aletundo * @author valsdav */ public interface MetadataRepository extends GraphRepository { /** * Get the Metadata node of the specified type * * @param type * @return the Metadata node of the specified type */ Metadata getMetadataByType(MetadataType type); /** - * Update the LAST_PROCESS relationship deleting the existing edge and + * Update the LATEST_PROCESS relationship deleting the existing edge and * creating a new one. */ @Query("MATCH (m:Metadata)-[lp:LATEST_PROCESS]->(p1:Process)<-[pp:PREVIOUS_PROCESS]-(p2:Process) DELETE lp CREATE (m)-[:LATEST_PROCESS]->(p2)") void updateLatestProcess(); } diff --git a/src/main/java/org/wikitolearn/wikirating/service/PageService.java b/src/main/java/org/wikitolearn/wikirating/service/PageService.java index 588728e..a4bf3ea 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/PageService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/PageService.java @@ -1,397 +1,402 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.GetPagesUpdateInfoException; import org.wikitolearn.wikirating.exception.PageNotFoundException; import org.wikitolearn.wikirating.exception.UpdatePagesAndRevisionsException; import org.wikitolearn.wikirating.model.CourseTree; import org.wikitolearn.wikirating.model.UpdateInfo; import org.wikitolearn.wikirating.model.graph.*; import org.wikitolearn.wikirating.repository.CourseLevelThreeRepository; import org.wikitolearn.wikirating.repository.CourseLevelTwoRepository; import org.wikitolearn.wikirating.repository.CourseRootRepository; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.service.mediawiki.PageMediaWikiService; import org.wikitolearn.wikirating.service.mediawiki.UpdateMediaWikiService; import org.wikitolearn.wikirating.util.enums.CourseLevel; /** * * @author aletundo, valsdav * */ @Service public class PageService { private static final Logger LOG = LoggerFactory.getLogger(PageService.class); @Autowired private PageMediaWikiService pageMediaWikiService; @Autowired private RevisionService revisionService; @Autowired private UpdateMediaWikiService updateMediaWikiService; @Autowired private UserService userService; @Autowired private PageRepository pageRepository; @Autowired private CourseRootRepository courseRootRepository; @Autowired private CourseLevelTwoRepository courseLevelTwoRepository; @Autowired private CourseLevelThreeRepository courseLevelThreeRepository; @Value("${mediawiki.namespace}") private String namespace; /** * This methods inserts all the pages inside the DB querying the MediaWiki API. * @param lang String * @param apiUrl String The MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initPages( String lang, String apiUrl ){ List pages = pageMediaWikiService.getAll(apiUrl); pages.forEach(page -> { page.setLang(lang); page.setLangPageId(lang + "_" +page.getPageId()); //Now we check the level of the page to set the right // additional laber for the Course Structure. CourseLevel levelLabel = getPageLevelFromTitle(page.getTitle()); if (levelLabel != CourseLevel.UNCATEGORIZED){ page.addLabel(levelLabel.name()); } }); pageRepository.save(pages); LOG.info("Inserted all {} pages", lang); return CompletableFuture.completedFuture(true); } /** * * @param lang * @param apiUrl * @param start * @param end * @return * @throws UpdatePagesAndRevisionsException */ @Async public CompletableFuture updatePages(String lang, String apiUrl, Date start, Date end) throws UpdatePagesAndRevisionsException{ try{ List updates = updateMediaWikiService.getPagesUpdateInfo(apiUrl, namespace, start, end); for(UpdateInfo update : updates){ if(!namespace.equals(update.getNs())) continue; switch (update.getType()) { case "new": // Add the new page with the right Course level Page newPage = addCoursePage(update.getPageLevelFromTitle(),update.getPageid(), update.getTitle(), lang); if (update.getPageLevelFromTitle() == CourseLevel.CourseLevelThree) { // Create the new revision. The change coefficient for the new revision is set to 0 by default. Revision newRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); // Add the first revision to the page ((CourseLevelThree) newPage).initFirstRevision(newRev); + // It's necessary to save the page again + courseLevelThreeRepository.save((CourseLevelThree) newPage); userService.setAuthorship(newRev); } break; case "edit": // Act only on CourseLevelThree pages if (update.getPageLevelFromTitle() == CourseLevel.CourseLevelThree){ // Create a new revision Revision updateRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); // Then add it to the page addRevisionToPage(lang + "_" + update.getPageid(), updateRev); // Then calculate the changeCoefficient revisionService.setChangeCoefficient(apiUrl, updateRev); // Finally set the authorship userService.setAuthorship(updateRev); } break; case "move": // We have to change the label in case the Course level is changed // Move the page to the new title movePage(update.getTitle(), update.getNewTitle(), lang); break; case "delete": // Delete the page and all its revisions deletePage(update.getTitle(), lang); break; default: break; } } }catch(GetPagesUpdateInfoException | PageNotFoundException e){ LOG.error("An error occurred while updating pages and revisions: {}", e.getMessage()); throw new UpdatePagesAndRevisionsException(); } return CompletableFuture.completedFuture(true); } /** * Create a new generic Page entity. * @param pageid * @param title * @param lang * @return the added page */ public Page addPage(int pageid, String title, String lang){ Page page = new Page(pageid, title, lang, lang + "_" + pageid); pageRepository.save(page); return page; } /** * Add a page entity distinguishing between the different Course levels. * @param level * @param pageid * @param title * @param lang * @return */ public Page addCoursePage(CourseLevel level, int pageid, String title, String lang){ switch(level){ case CourseRoot: CourseRoot pageRoot = new CourseRoot(pageid, title, lang, lang + "_" + pageid); courseRootRepository.save(pageRoot); return pageRoot; case CourseLevelTwo: CourseLevelTwo pageTwo = new CourseLevelTwo(pageid, title, lang, lang + "_" + pageid); courseLevelTwoRepository.save(pageTwo); return pageTwo; case CourseLevelThree: CourseLevelThree pageThree = new CourseLevelThree(pageid, title, lang, lang + "_" + pageid); courseLevelThreeRepository.save(pageThree); return pageThree; default: return addPage(pageid, title, lang); } } /** * Create a new CourseLevelThree page. It requires the firstRevision of the Page in order * to create the initial relationships. * @param pageid * @param title * @param lang * @param firstRevision * @return the added page */ public CourseLevelThree addCourseLevelThreePage(int pageid, String title, String lang, Revision firstRevision){ CourseLevelThree page = new CourseLevelThree(pageid, title, lang, lang + "_" + pageid, firstRevision); courseLevelThreeRepository.save(page); return page; } /** * Get the page with the given pageId and language * @param pageId the id of the page * @param lang the language of the page * @return the requested page * @throws PageNotFoundException */ public Page getPage(int pageId, String lang) throws PageNotFoundException{ Page page = pageRepository.findByLangPageId(lang + "_" + pageId); if(page == null){ LOG.error("Page with pageId {} and lang {} not found.", pageId, lang); throw new PageNotFoundException(); } return page; } /** * Get the page with the given langePageId * @param langPageId the langPageId of the page * @return the requested page * @throws PageNotFoundException */ public Page getPage(String langPageId) throws PageNotFoundException{ Page page = pageRepository.findByLangPageId(langPageId); if(page == null){ LOG.error("Page with langPageId: {} not found.", langPageId); throw new PageNotFoundException(); } return page; } /** * Add a new revision to a CourseLevelThree page. It links the page to the new revision via * LAST_REVISION link. Moreover it create the PREVIOUS_REVISION link. * @param langPageId * @param rev */ public void addRevisionToPage(String langPageId, Revision rev) throws PageNotFoundException{ CourseLevelThree page = courseLevelThreeRepository.findByLangPageId(langPageId); if(page == null){ throw new PageNotFoundException(); } + //TODO Check if the page has no previous revisions // Add PREVIOUS_REVISION relationship rev.setPreviousRevision(page.getLastRevision()); - page.setLastRevision(rev); + //page.setLastRevision(rev); //Maybe this is not necessary // The changes on the revision will be automatically persisted courseLevelThreeRepository.save(page); + //Update the LAST_REVISION edge with a query + courseLevelThreeRepository.updateLastRevision(page.getLangPageId()); } /** * Change title of a page. The method is prefixed by move to follow MediaWiki naming. * The method checks also the new page title to set the right Course level label * in case of changes. * @param oldTitle the old title of the page * @param newTitle the new title to set * @param lang the language of the page * @return the updated page * @throws PageNotFoundException */ public Page movePage(String oldTitle, String newTitle, String lang) throws PageNotFoundException{ Page page = pageRepository.findByTitleAndLang(oldTitle, lang); if(page == null){ throw new PageNotFoundException(); } page.setTitle(newTitle); // Check if the Course level has changed if (getPageLevelFromTitle(oldTitle) != getPageLevelFromTitle(newTitle)){ page.removeLabel(getPageLevelFromTitle(oldTitle).name()); page.addLabel(getPageLevelFromTitle(newTitle).name()); } pageRepository.save(page); return page; } /** * Delete a page from the graph given its title and domain language. * @param title the title of the page * @param lang the language of the page * @throws PageNotFoundException */ public void deletePage(String title, String lang) throws PageNotFoundException{ Page page = pageRepository.findByTitleAndLang(title, lang); if(page == null){ throw new PageNotFoundException(); } // Delete the revisions of the page if it's CourseLevelThree if (page.hasLabel("CourseLevelThree")){ revisionService.deleteRevisionsOfPage(page.getLangPageId()); } // Delete finally the page itself pageRepository.delete(page); } /*** * Get the level of a Page in the Course Structure * anaylizing the number of slashes in the title. * @param title * @return null if the page has more that 2 slashes. */ public static CourseLevel getPageLevelFromTitle(String title){ int nslash = StringUtils.countMatches(title, "/"); switch (nslash){ case 0: return CourseLevel.CourseRoot; case 1: return CourseLevel.CourseLevelTwo; case 2: return CourseLevel.CourseLevelThree; default: //The page remains generic. return CourseLevel.UNCATEGORIZED; } } /** * Get the pages labeled by :CourseRoot label * @return the list of course root pages */ public List getCourseRootPages(String lang){ return courseRootRepository.findByLang(lang); } /** * Get the page labeled only by :Page label * @return the list of the uncategorized pages */ public List getUncategorizedPages(String lang){ return pageRepository.findAllUncategorizedPages(lang); } /** * * @param lang the language of the domain * @param apiUrl the MediaWiki API url * @return */ public CompletableFuture updateCourseStructure(String lang, String apiUrl) { List courseRootPages = getCourseRootPages(lang); applyCourseStructure(lang, apiUrl, courseRootPages); return CompletableFuture.completedFuture(true); } /** * @param lang the language of the domain * @param apiUrl the MediaWiki API url * @param courseRootPages */ private void applyCourseStructure(String lang, String apiUrl, List courseRootPages) { for (CourseRoot pageRoot : courseRootPages) { // Get course tree and prepare relationship set CourseTree tree = pageMediaWikiService.getCourseTree(apiUrl, pageRoot.getTitle()); Set levelsTwo = (pageRoot.getLevelsTwo() == null) ? new HashSet<>() : pageRoot.getLevelsTwo(); int index = 0; for (String levelTwo : tree.getLevelsTwo()) { String levelTwoTitle = (tree.getRoot() + "/" + levelTwo).trim(); CourseLevelTwo levelTwoPage = courseLevelTwoRepository.findByTitleAndLang(levelTwoTitle, lang); // Skip malformed page if (levelTwoPage == null) continue; // Add levelstwo to the set to be saved if(!levelsTwo.contains(levelTwoPage)){ levelsTwo.add(levelTwoPage); } Set levelsThree = (levelTwoPage.getLevelsThree() == null) ? new HashSet<>() : levelTwoPage.getLevelsThree(); // Add levels three to the set to be saved for (String levelThree : tree.getLevelsTree().get(index)) { String levelThreeTitle = (levelTwoTitle + "/" + levelThree).trim(); CourseLevelThree levelThreePage = courseLevelThreeRepository.findByTitleAndLang(levelThreeTitle, lang); // Skip malformed page if (levelThreePage == null) continue; if(!levelsThree.contains(levelThreePage)){ levelsThree.add(levelThreePage); } } // Set LEVEL_THREE relationships levelTwoPage.setLevelsThree(levelsThree); courseLevelThreeRepository.save(levelsThree); index++; } // Set LEVEL_TWO relationships and CourseRoot label pageRoot.setLevelsTwo(levelsTwo); courseLevelTwoRepository.save(levelsTwo); courseRootRepository.save(pageRoot); } } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java index 9c288be..192dc31 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java @@ -1,212 +1,214 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.Date; import java.util.List; import java.util.ListIterator; import java.util.Set; import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.RevisionNotFoundException; import org.wikitolearn.wikirating.model.graph.CourseLevelThree; import org.wikitolearn.wikirating.model.graph.Revision; import org.wikitolearn.wikirating.repository.CourseLevelThreeRepository; import org.wikitolearn.wikirating.repository.RevisionRepository; import org.wikitolearn.wikirating.service.mediawiki.RevisionMediaWikiService; import static java.lang.Math.exp; /** * * @author aletundo, valsdav * */ @Service public class RevisionService { private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); @Autowired private RevisionMediaWikiService revisionMediaWikiService; @Autowired private RevisionRepository revisionRepository; @Autowired private CourseLevelThreeRepository courseLevelThreeRepository; /** * Initialize the revisions for the first time querying the MediaWiki API. * This method adds the revisions and sets the FIRST_REVISION, * LAST_REVISION and PREVIOUS_REVISION relationships. * Only the revisions of CourseLevelThree pages are fetched. * @param lang the domain language * @param apiUrl the MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initRevisions(String lang, String apiUrl) { List pages = courseLevelThreeRepository.findByLang(lang); for(CourseLevelThree page : pages){ List revisions = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, page.getPageId()); // Set the first and the last revisions for the current page page.setFirstRevision(revisions.get(0)); page.setLastRevision(revisions.get(revisions.size() - 1)); // Set the last validated revision to the first one. page.setLastValidatedRevision(revisions.get(0)); + // Create the PreviousRevision links ListIterator it = revisions.listIterator(); while(it.hasNext()){ Revision rev = it.next(); rev.setLangRevId(lang + "_" + rev.getRevId()); rev.setLang(lang); if (it.previousIndex() != 0) { rev.setPreviousRevision(revisions.get(it.previousIndex() - 1)); } } // Saving all the revisions node and the page node revisionRepository.save(revisions); courseLevelThreeRepository.save(page); LOG.info("Inserted revisions for page {}", page.getLangPageId()); } return CompletableFuture.completedFuture(true); } @Async public CompletableFuture calculateChangeCoefficientAllRevisions(String lang, String apiUrl){ LOG.info("Calculating changeCoefficient for all the revisions..."); Set revisions = revisionRepository.findByLang(lang); for (Revision rev : revisions){ double changeCoefficient = calculateChangeCoefficient(apiUrl, rev); rev.setChangeCoefficient(changeCoefficient); } revisionRepository.save(revisions); LOG.info("ChangeCoefficient calculated for all revisions."); return CompletableFuture.completedFuture(true); } /** - * Add a new Revision to the graph + * Add a new Revision to the graph. This method DOESN'T link the + * revision to a Page. * @param revid * @param lang * @param userid * @param parentid * @param length * @param timestamp * @return */ public Revision addRevision(int revid, String lang, int userid, int parentid, int length, Date timestamp){ Revision rev = new Revision(revid, lang, userid, parentid, length, timestamp); revisionRepository.save(rev); return rev; } /** * Delete a revision given its langPageId * @param langPageId the langPageId of the revision */ public void deleteRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } revisionRepository.delete(revisions); } public Set getRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } return revisions; } /** * Get the requested revision * @param langRevId the langRevId of the revision * @return the revision * @throws RevisionNotFoundException */ public Revision getRevision(String langRevId) throws RevisionNotFoundException{ Revision revision = revisionRepository.findByLangRevId(langRevId); if(revision == null){ LOG.error("Revision {} not found", langRevId); throw new RevisionNotFoundException(); } return revision; } /** * Update the given revision * @param revision * @return the updated revision */ public Revision updateRevision(Revision revision){ revisionRepository.save(revision); return revision; } /** * Calculate and set the changeCoefficient of a Revision. * This method also persist the Revision changes. * @param apiUrl * @param revision */ public void setChangeCoefficient(String apiUrl, Revision revision){ double cc = calculateChangeCoefficient(apiUrl, revision); revision.setChangeCoefficient(cc); revisionRepository.save(revision); } /** * Calculate the changeCoefficient for a given Revision querying mediawiki * and returns the number. It doesn't store it in the Revision. * @param revision * @return */ public double calculateChangeCoefficient(String apiUrl, Revision revision){ double previousLength = 0.0; double changeCoefficient = 0.0; // Get the previous Revision Revision previousRevision = revision.getPreviousRevision(); if (previousRevision == null){ previousLength = 1.0; changeCoefficient = 0.0; LOG.info("Change coefficient of revision {} (first-rev): {}", revision.getLangRevId(), changeCoefficient); } else{ double prevL = (double) previousRevision.getLength(); // Suppose the mean line length of 120 characters and that the length is in bytes. // We want a "lenght" in n° of lines if (prevL == 0){ previousLength = 1.0; }else { previousLength = (prevL < 120) ? 1 : prevL / 120; } // Query mediawiki for diff text String diffText = revisionMediaWikiService.getDiffPreviousRevision(apiUrl, previousRevision.getRevId(), revision.getRevId()); int addedLines = StringUtils.countMatches(diffText, "diff-addedline"); int deletedLines = StringUtils.countMatches(diffText, "diff-deletedline"); //int inlineChanges = StringUtils.countMatches(diffText, "diffchange-inline"); // Finally calculation of change Coefficient double t = ((1.2 * deletedLines + addedLines) ) / previousLength; - changeCoefficient = 1 / exp(0.7 * t); + changeCoefficient = 1 / exp(0.5 * t); LOG.info("Change coefficient of revision {} (+{}-{}/{}): {}", revision.getLangRevId(), addedLines, deletedLines, previousLength, changeCoefficient); } return changeCoefficient; } }