diff --git a/src/main/java/org/wikitolearn/wikirating/controller/PageController.java b/src/main/java/org/wikitolearn/wikirating/controller/PageController.java index 2e9e108..f20f9c0 100644 --- a/src/main/java/org/wikitolearn/wikirating/controller/PageController.java +++ b/src/main/java/org/wikitolearn/wikirating/controller/PageController.java @@ -1,159 +1,157 @@ /** * */ package org.wikitolearn.wikirating.controller; import java.time.Instant; import java.util.List; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.wikitolearn.wikirating.exception.PageNotFoundException; import org.wikitolearn.wikirating.exception.RevisionNotFoundException; import org.wikitolearn.wikirating.model.api.ApiResponseSuccess; -import org.wikitolearn.wikirating.model.graph.Page; -import org.wikitolearn.wikirating.model.graph.Revision; -import org.wikitolearn.wikirating.model.graph.TemporaryVote; -import org.wikitolearn.wikirating.model.graph.Vote; +import org.wikitolearn.wikirating.model.graph.*; import org.wikitolearn.wikirating.service.PageService; import org.wikitolearn.wikirating.service.RevisionService; import org.wikitolearn.wikirating.service.VoteService; /** * RESTController for pages resources. It handles all the requests to {lang}/pages * @author aletundo */ @RestController @RequestMapping("{lang}/pages/") public class PageController { private static final Logger LOG = LoggerFactory.getLogger(PageController.class); @Autowired private PageService pageService; @Autowired private RevisionService revisionService; @Autowired private VoteService voteService; /** * Handle GET requests on {lang}/pages/{pageId} URI. The last revision of the page is returned. * * @param pageId * int The id of the page * @return the last revision of the requested page * @throws PageNotFoundException */ @RequestMapping(value = "{pageId}", method = RequestMethod.GET, produces = "application/json") @ResponseStatus(HttpStatus.OK) @ResponseBody public ApiResponseSuccess getLastRevisionByPageId(@PathVariable("lang") String lang, @PathVariable("pageId") int pageId) throws PageNotFoundException{ try{ - Page page = pageService.getPage(pageId, lang); + //TODO Add check for page label and send exception in case of Course structure pages + CourseLevelThree page = (CourseLevelThree) pageService.getPage(pageId, lang); ApiResponseSuccess body = new ApiResponseSuccess(page.getLastRevision(), Instant.now().toEpochMilli()); return body; }catch(PageNotFoundException e){ LOG.error("Impossible to get the last revision of {}_{}: {}",lang, pageId, e.getMessage()); throw e; } } /** * Handle GET requests on {lang}/pages/{pageId}/revisions URI. All the revision for the requested page are returned. * * @param pageId * int The id of the page * @return a list with all revisions of the requested page */ @RequestMapping(value = "{pageId}/revisions", method = RequestMethod.GET, produces = "application/json") @ResponseStatus(HttpStatus.OK) @ResponseBody public ApiResponseSuccess getAllPageRevisions(@PathVariable("lang") String lang, @PathVariable("pageId") int pageId) { String langPageId = lang + "_" + pageId; try{ Set revisions = revisionService.getRevisionsOfPage(langPageId); ApiResponseSuccess body = new ApiResponseSuccess(revisions, Instant.now().toEpochMilli()); return body; }catch(RevisionNotFoundException e){ LOG.error("Impossible to get revisions of page {}", langPageId); throw e; } } /** * Handle GET requests on {lang}/pages/{pageId}/revisions/{revId} URI. The requested page revision is returned. * * @param pageId * int The id of the page * @param revId * int The id of the revision * @return the requested revision of the page */ @RequestMapping(value = "{pageId}/revisions/{revId}", method = RequestMethod.GET, produces = "application/json") @ResponseStatus(HttpStatus.OK) @ResponseBody public ApiResponseSuccess getRevisionById(@PathVariable("lang") String lang, @PathVariable("pageId") int pageId, @PathVariable("revId") int revId) { String langRevId = lang + "_" + revId; try{ Revision revision = revisionService.getRevision(langRevId); ApiResponseSuccess body = new ApiResponseSuccess(revision, Instant.now().toEpochMilli()); return body; }catch(RevisionNotFoundException e){ String langPageId = lang + "_" + pageId; LOG.error("Impossible to get revision {} of page {}", langRevId, langPageId); throw e; } } /** * Handle GET requests on {lang}/pages/{pageId}/revisions/{revId}/votes URI. All * the votes for the requested page revision are returned. * * @param pageId * int The id of the page * @param revId * int The id of the revision * @return all the votes of the requested revision */ @RequestMapping(value = "{pageId}/revisions/{revId}/votes", method = RequestMethod.GET, produces = "application/json") @ResponseStatus(HttpStatus.OK) @ResponseBody public ApiResponseSuccess showVotes(@PathVariable("lang") String lang, @PathVariable("pageId") int pageId, @PathVariable("revId") int revId) { // TODO: Work in progress String langRevId = lang + "_" + revId; List votes = voteService.getAllVotesOfRevision(langRevId); ApiResponseSuccess body = new ApiResponseSuccess(votes, Instant.now().toEpochMilli()); return body; } /** * Handle POST requests on {lang}/pages/{pageId}/revisions/{revId}/votes URI. It * expects vote and userId as request parameters. * * @param pageId * int The id of the page * @param revId * int The id of the revision * @param vote * double The value of the vote * @param userId * int The id of the user who submit the request * @return response Vote */ @RequestMapping(value = "{pageId}/revisions/{revId}/votes", method = RequestMethod.POST, produces = "application/json") @ResponseStatus(HttpStatus.CREATED) @ResponseBody public TemporaryVote addVote(@PathVariable("lang") String lang, @PathVariable("pageId") int pageId, @PathVariable("revId") int revId, @RequestParam("vote") double vote, @RequestParam("userId") int userId) { //TODO return null; } } diff --git a/src/main/java/org/wikitolearn/wikirating/model/UpdateInfo.java b/src/main/java/org/wikitolearn/wikirating/model/UpdateInfo.java index 8627080..32f1a13 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/UpdateInfo.java +++ b/src/main/java/org/wikitolearn/wikirating/model/UpdateInfo.java @@ -1,132 +1,140 @@ package org.wikitolearn.wikirating.model; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.wikitolearn.wikirating.service.PageService; +import org.wikitolearn.wikirating.util.enums.CourseLevel; //import org.wikitolearn.wikirating.util.enums.UpdateType; import java.util.Date; /** * This class represents a RecentChange entry during the fetch Process. * It stores the information fetched from the API before the update of the DB. * Created by valsdav on 30/03/17. */ @JsonIgnoreProperties(ignoreUnknown = true) public class UpdateInfo { private String type; private String title; private String newTitle; private int pageid; private int revid; private int old_revid; private String user; private int userid; private int oldlen; private int newlen; private String ns; private Date timestamp; public UpdateInfo(){} public String getType() { return type; } public void setType(String type) { this.type = type; } public int getPageid() { return pageid; } public void setPageid(int pageid) { this.pageid = pageid; } public int getRevid() { return revid; } public void setRevid(int revid) { this.revid = revid; } public int getOld_revid() { return old_revid; } public void setOld_revid(int old_revid) { this.old_revid = old_revid; } public int getUserid() { return userid; } public void setUserid(int userid) { this.userid = userid; } public int getOldlen() { return oldlen; } public void setOldlen(int oldlen) { this.oldlen = oldlen; } public int getNewlen() { return newlen; } public void setNewlen(int newlen) { this.newlen = newlen; } public Date getTimestamp() { return timestamp; } public void setTimestamp(Date timestamp) { this.timestamp = timestamp; } public String getTitle() { return title; } public void setTitle(String title) { this.title = title; } public String getUser() { return user; } public void setUser(String user) { this.user = user; } public String getNewTitle() { return newTitle; } public void setNewTitle(String newTitle) { this.newTitle = newTitle; } /** * @return the ns */ public String getNs() { return ns; } /** * @param ns the ns to set */ public void setNs(String ns) { this.ns = ns; } + + public CourseLevel getPageLevelFromTitle(){ return PageService.getPageLevelFromTitle(title);} + + public CourseLevel getPageLevelFromNewTitle(){ return PageService.getPageLevelFromTitle(title);} + } + diff --git a/src/main/java/org/wikitolearn/wikirating/model/graph/CourseLevelThree.java b/src/main/java/org/wikitolearn/wikirating/model/graph/CourseLevelThree.java index 7cfe512..6991246 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/graph/CourseLevelThree.java +++ b/src/main/java/org/wikitolearn/wikirating/model/graph/CourseLevelThree.java @@ -1,64 +1,87 @@ /** * */ package org.wikitolearn.wikirating.model.graph; import org.neo4j.ogm.annotation.NodeEntity; import org.neo4j.ogm.annotation.Relationship; /** * @author aletundo * */ @NodeEntity( label = "CourseLevelThree") public class CourseLevelThree extends Page { @Relationship(type = "LAST_REVISION", direction = Relationship.OUTGOING) private Revision lastRevision; @Relationship(type = "FIRST_REVISION", direction = Relationship.OUTGOING) private Revision firstRevision; @Relationship(type = "LAST_VALIDATED_REVISION", direction = Relationship.OUTGOING) private Revision lastValidatedRevision; - + + public CourseLevelThree(){} + + public CourseLevelThree(int pageId, String title, String lang, String langPageId){ + super(pageId, title, lang, langPageId); + } + + public CourseLevelThree(int pageId, String title, String lang, String langPageId, Revision firstRevision){ + super(pageId, title, lang, langPageId); + setFirstRevision(firstRevision); + setLastRevision(firstRevision); + setLastValidatedRevision(firstRevision); + } + + /** + * Add the first revision the the Page setting the right links. + * @param firstRevision + */ + public void initFirstRevision(Revision firstRevision){ + setFirstRevision(firstRevision); + setLastRevision(firstRevision); + setLastValidatedRevision(firstRevision); + } + /** * @return the lastRevision */ public Revision getLastRevision() { return lastRevision; } /** * @param lastRevision the lastRevision to set */ public void setLastRevision(Revision lastRevision) { this.lastRevision = lastRevision; } /** * @return the firstRevision */ public Revision getFirstRevision() { return firstRevision; } /** * @param firstRevision the firstRevision to set */ public void setFirstRevision(Revision firstRevision) { this.firstRevision = firstRevision; } /** * @return the lastValidatedRevision */ public Revision getLastValidatedRevision() { return lastValidatedRevision; } /** * @param lastValidatedRevision the lastValidatedRevision to set */ public void setLastValidatedRevision(Revision lastValidatedRevision) { this.lastValidatedRevision = lastValidatedRevision; } } diff --git a/src/main/java/org/wikitolearn/wikirating/model/graph/CourseLevelTwo.java b/src/main/java/org/wikitolearn/wikirating/model/graph/CourseLevelTwo.java index 446655e..90e9035 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/graph/CourseLevelTwo.java +++ b/src/main/java/org/wikitolearn/wikirating/model/graph/CourseLevelTwo.java @@ -1,46 +1,53 @@ /** * */ package org.wikitolearn.wikirating.model.graph; import java.util.Set; import org.neo4j.ogm.annotation.NodeEntity; import org.neo4j.ogm.annotation.Relationship; /** * @author aletundo * */ @NodeEntity( label = "CourseLevelTwo") public class CourseLevelTwo extends Page { @Relationship(type = "LEVEL_THREE", direction = Relationship.OUTGOING) private Set levelsThree; @Relationship(type = "FIRST_CALCULATION", direction = Relationship.OUTGOING) private History firstCalculation; @Relationship(type = "LAST_CALCULATION", direction = Relationship.OUTGOING) private History lastCalculation; - + + public CourseLevelTwo() {} + + public CourseLevelTwo(int pageId, String title, String lang, String langPageId){ + super(pageId, title, lang, langPageId); + } + + /** * @return the levelsThree */ public Set getLevelsThree() { return levelsThree; } /** * @param levelsThree the levelsThree to set */ public void setLevelsThree(Set levelsThree) { this.levelsThree = levelsThree; } /** * @param levelThree the levelThree to add */ public void addLevelThree(CourseLevelThree levelThree){ this.levelsThree.add(levelThree); } } diff --git a/src/main/java/org/wikitolearn/wikirating/model/graph/CourseRoot.java b/src/main/java/org/wikitolearn/wikirating/model/graph/CourseRoot.java index 39e7732..6215e6a 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/graph/CourseRoot.java +++ b/src/main/java/org/wikitolearn/wikirating/model/graph/CourseRoot.java @@ -1,46 +1,52 @@ /** * */ package org.wikitolearn.wikirating.model.graph; import java.util.Set; import org.neo4j.ogm.annotation.NodeEntity; import org.neo4j.ogm.annotation.Relationship; /** * @author aletundo * */ @NodeEntity( label = "CourseRoot") public class CourseRoot extends Page { @Relationship(type = "LEVEL_TWO", direction = Relationship.OUTGOING) private Set levelsTwo; @Relationship(type = "FIRST_CALCULATION", direction = Relationship.OUTGOING) private History firstCalculation; @Relationship(type = "LAST_CALCULATION", direction = Relationship.OUTGOING) private History lastCalculation; + + public CourseRoot(){} + + public CourseRoot(int pageId, String title, String lang, String langPageId){ + super(pageId, title, lang, langPageId); + } /** * @return the levelsTwo */ public Set getLevelsTwo() { return levelsTwo; } /** * @param levelsTwo the levelsTwo to set */ public void setLevelsTwo(Set levelsTwo) { this.levelsTwo = levelsTwo; } /** * @param levelTwo the levelTwo to add */ public void addLevelTwo(CourseLevelTwo levelTwo){ this.levelsTwo.add(levelTwo); } } diff --git a/src/main/java/org/wikitolearn/wikirating/model/graph/Page.java b/src/main/java/org/wikitolearn/wikirating/model/graph/Page.java index 6ea51e4..df9e227 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/graph/Page.java +++ b/src/main/java/org/wikitolearn/wikirating/model/graph/Page.java @@ -1,198 +1,191 @@ /** * */ package org.wikitolearn.wikirating.model.graph; import java.util.HashSet; import java.util.Set; import org.neo4j.ogm.annotation.GraphId; import org.neo4j.ogm.annotation.Index; import org.neo4j.ogm.annotation.Labels; import org.neo4j.ogm.annotation.NodeEntity; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; /** * @author aletundo * */ @NodeEntity( label = "Page") public class Page { @GraphId @JsonIgnore private Long graphId; @JsonProperty("pageId") private int pageId; @Index private String title; @Index private String lang; @Index(unique = true, primary = true) @JsonProperty(access = Access.WRITE_ONLY) private String langPageId; - private double pageRank; @Labels private Set labels = new HashSet<>(); /** * */ public Page() {} /** * @param pageId * @param title * @param lang */ public Page(int pageId, String title, String lang, String langPageId) { this.pageId = pageId; this.title = title; this.lang = lang; this.langPageId = langPageId; } /** * @return the graphId */ public Long getGraphId() { return graphId; } /** * @param graphId the graphId to set */ public void setGraphId(Long graphId) { this.graphId = graphId; } /** * @return the pageId */ public int getPageId() { return pageId; } /** * @param pageId the pageId to set */ public void setPageid(int pageId) { this.pageId = pageId; } /** * @return the title */ public String getTitle() { return title; } /** * @param title the title to set */ public void setTitle(String title) { this.title = title; } /** * @return the lang */ public String getLang() { return lang; } /** * @param lang the lang to set */ public void setLang(String lang) { this.lang = lang; } /** * @return the langPageId */ public String getLangPageId() { return langPageId; } /** * @param langPageId the langPageId to set */ public void setLangPageId(String langPageId) { this.langPageId = langPageId; } - /** - * @return the pageRank - */ - public double getPageRank() { - return pageRank; - } - - /** - * @param pageRank the pageRank to set - */ - public void setPageRank(double pageRank) { - this.pageRank = pageRank; - } - /** * @return the labels */ public Set getLabels() { return labels; } /** * @param labels the labels to set */ public void setLabels(Set labels) { this.labels = labels; } + + public boolean hasLabel(String label){ + return labels.contains(label); + } /** * @param label the label to set */ public void addLabel(String label) { this.labels.add(label); } + public void removeLabel(String label){ + this.labels.remove(label); + } + /* (non-Javadoc) * @see java.lang.Object#toString() */ @Override public String toString() { return "Page [graphId=" + graphId + ", pageId=" + pageId + ", title=" + title + ", lang=" + lang - + ", langPageId=" + langPageId + ", pageRank=" + pageRank + ", labels=" + labels + "]"; + + ", langPageId=" + langPageId + ", labels=" + labels + "]"; } /* (non-Javadoc) * @see java.lang.Object#equals(java.lang.Object) */ @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (!(obj instanceof Page)) { return false; } Page other = (Page) obj; if (langPageId == null) { if (other.langPageId != null) { return false; } } else if (!langPageId.equals(other.langPageId)) { return false; } return true; } } \ No newline at end of file diff --git a/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java index 16ffe28..e5334fc 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java @@ -1,70 +1,54 @@ /** * */ package org.wikitolearn.wikirating.repository; import java.util.List; import org.springframework.data.neo4j.annotation.Query; import org.springframework.data.neo4j.repository.GraphRepository; import org.springframework.data.repository.query.Param; import org.wikitolearn.wikirating.model.graph.Page; /** * @author aletundo * */ public interface PageRepository extends GraphRepository { /** * * @param title * @param lang * @return */ - Page findByTitleAndLang(String title, String lang); + T findByTitleAndLang(String title, String lang); /** * * @param langPageId * @return */ - Page findByLangPageId(String langPageId); + T findByLangPageId(String langPageId); /** * * @param lang * @return */ - @Query("MATCH (p:Page {lang:{0}}) RETURN p") - List findAllByLang(String lang); + List findByLang(String lang); - /** - * - * @return - */ - @Query("MATCH (p:CourseRoot) RETURN p") - List findAllCourseRootPages(); - - /** - * - * @param lang - * @return - */ - @Query("MATCH (p:CourseRoot) WHERE p.lang = {lang} RETURN p") - List findAllCourseRootPages(@Param("lang") String lang); - - /** + /** * * @return */ @Query("MATCH (p:Page) WHERE NOT p:CourseRoot AND NOT p:CourseLevelTwo AND NOT p:CourseLevelThree RETURN p") - List findAllUncategorizedPages(); + List findAllUncategorizedPages(); /** * * @param lang * @return */ @Query("MATCH (p:Page) WHERE p.lang = {lang} AND NOT p:CourseRoot AND NOT p:CourseLevelTwo AND NOT p:CourseLevelThree RETURN p") - List findAllUncategorizedPages(@Param("lang") String lang); + List findAllUncategorizedPages(@Param("lang") String lang); } diff --git a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java index f97e241..e67db64 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java @@ -1,56 +1,56 @@ /** * */ package org.wikitolearn.wikirating.repository; import java.util.Set; import org.springframework.data.neo4j.annotation.Query; import org.springframework.data.neo4j.repository.GraphRepository; import org.wikitolearn.wikirating.model.graph.Revision; /** * @author aletundo * */ public interface RevisionRepository extends GraphRepository { /** * * @param langRevId * @return */ Revision findByLangRevId(String langRevId); /** * * @param lang * @return */ Set findByLang(String lang); /** * * @param userId * @return */ Set findByUserId(int userId); /** - * This query returns all the Revisions of a Page. + * This query returns all the Revisions of a Page (CourseLevelThree). * The direction -> of the link is important to traverse * only the chain of Revisions of the page without reaching other nodes. * @param langPageId * @return */ - @Query("MATCH (p:Page {langPageId:{0}})-[:LAST_REVISION|PREVIOUS_REVISION*]->(r:Revision) RETURN r") + @Query("MATCH (p:CourseLevelThree {langPageId:{0}})-[:LAST_REVISION|PREVIOUS_REVISION*]->(r:Revision) RETURN r") Set findAllRevisionOfPage(String langPageId); /** * This query return the previous revision of a Revision identified * by langRevId. * @param langRevId langRevId of the requested revision * @return the previous revision */ @Query("MATCH (r:Revision {langRevId:{0}})-[:PREVIOUS_REVISION]->(a:Revision) RETURN a") Revision findPreviousRevision(String langRevId); } diff --git a/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java b/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java index 553da37..02ba1ce 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java @@ -1,255 +1,256 @@ package org.wikitolearn.wikirating.service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.*; import org.wikitolearn.wikirating.model.graph.Process; import org.wikitolearn.wikirating.util.enums.ProcessStatus; import org.wikitolearn.wikirating.util.enums.ProcessType; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; /** * @author aletundo * @author valsdav */ @Service public class MaintenanceService { private static final Logger LOG = LoggerFactory.getLogger(MaintenanceService.class); @Autowired private UserService userService; @Autowired private PageService pageService; @Autowired private RevisionService revisionService; @Autowired private MetadataService metadataService; @Autowired private ProcessService processService; @Autowired private VoteService voteService; @Value("#{'${mediawiki.langs}'.split(',')}") private List langs; @Value("${mediawiki.protocol}") private String protocol; @Value("${mediawiki.api.url}") private String apiUrl; @Value("${mediawiki.namespace}") private String namespace; /** * Initialize the graph for the first time using parallel threads for each domain language * @return true if initialization succeed */ public boolean initializeGraph() { // Initialize Metadata service metadataService.initMetadata(); // Start a new Process Process initProcess = processService.addProcess(ProcessType.INIT); metadataService.addFirstProcess(initProcess); CompletableFuture initFuture = CompletableFuture // Users and pages .allOf(buildUsersAndPagesFuturesList().toArray(new CompletableFuture[langs.size() + 1])) + // CourseStructure + .thenCompose(result -> CompletableFuture + .allOf(buildInitCourseStructureFuturesList().toArray(new CompletableFuture[langs.size()]))) + // Revisions .thenCompose(result -> CompletableFuture - // Revisions .allOf(buildRevisionsFuturesList().toArray(new CompletableFuture[langs.size()]))) - .thenCompose(result -> CompletableFuture - // Change Coefficients - .allOf(buildChangeCoefficientFuturesList().toArray(new CompletableFuture[langs.size()]))) + // Change Coefficients .thenCompose(result -> CompletableFuture - // CourseStructure - .allOf(buildInitCourseStructureFuturesList().toArray(new CompletableFuture[langs.size()]))) + .allOf(buildChangeCoefficientFuturesList().toArray(new CompletableFuture[langs.size()]))) + // Users authorship .thenCompose(result -> userService.initAuthorship()); try { boolean result = initFuture.get(); // Save the result of the process if (result){ processService.closeCurrentProcess(ProcessStatus.DONE); }else{ processService.closeCurrentProcess(ProcessStatus.EXCEPTION); } return result; } catch (InterruptedException | ExecutionException e) { LOG.error("Something went wrong. {}", e.getMessage()); return false; } } /** * Entry point for the scheduled graph updated * @return true if the update succeed */ @Scheduled(cron = "${maintenance.update.cron}") public void updateGraph() { Process currentFetchProcess; Date startTimestampCurrentFetch, startTimestampLatestFetch; // Get start timestamp of the latest FETCH Process before opening a new process startTimestampLatestFetch = (processService.getLastProcessStartDateByType(ProcessType.FETCH) != null) ? processService.getLastProcessStartDateByType(ProcessType.FETCH) : processService.getLastProcessStartDateByType(ProcessType.INIT); // Create a new FETCH process try { currentFetchProcess = processService.addProcess(ProcessType.FETCH); metadataService.updateLatestProcess(); startTimestampCurrentFetch = currentFetchProcess.getStartOfProcess(); } catch (PreviousProcessOngoingException e){ LOG.error("Cannot start Update process because the previous process is still ONGOING." + "The update will be aborted."); return; } try { CompletableFuture updateFuture = CompletableFuture .allOf(userService.updateUsers(protocol + langs.get(0) + "." + apiUrl, startTimestampLatestFetch, startTimestampCurrentFetch)) .thenCompose(result -> CompletableFuture .allOf(buildUpdatePagesFuturesList(startTimestampLatestFetch, startTimestampCurrentFetch) .toArray(new CompletableFuture[langs.size()]))) .thenCompose(result -> CompletableFuture .allOf(buildUpdateCourseStructureFuturesList().toArray(new CompletableFuture[langs.size()]))) .thenCompose(result -> voteService.validateTemporaryVotes(startTimestampCurrentFetch)); boolean result = updateFuture.get(); // Save the result of the process, closing the current one if (result) { processService.closeCurrentProcess(ProcessStatus.DONE); } else { processService.closeCurrentProcess(ProcessStatus.ERROR); } } catch (TemporaryVoteValidationException | UpdateUsersException | UpdatePagesAndRevisionsException | InterruptedException | ExecutionException e) { processService.closeCurrentProcess(ProcessStatus.EXCEPTION); LOG.error("An error occurred during a scheduled graph update procedure"); throw new UpdateGraphException(); } } /** * Build a list of CompletableFuture. * * @return a list of CompletableFuture */ private List> buildInitCourseStructureFuturesList() { List> parallelInitCourseStructureFutures = new ArrayList<>(); // Add course structure for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; - parallelInitCourseStructureFutures.add(pageService.initCourseStructure(lang, url)); + parallelInitCourseStructureFutures.add(pageService.updateCourseStructure(lang, url)); } return parallelInitCourseStructureFutures; } /** * Build a list of CompletableFuture. * * @return a list of CompletableFuture */ private List> buildUpdateCourseStructureFuturesList() { List> parallelUpdateCourseStructureFutures = new ArrayList<>(); // Add course structure for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelUpdateCourseStructureFutures.add(pageService.updateCourseStructure(lang, url)); } return parallelUpdateCourseStructureFutures; } /** * * @param start * @param end * @return */ private List> buildUpdatePagesFuturesList(Date start, Date end) { List> futures = new ArrayList<>(); // Add update pages for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; futures.add(pageService.updatePages(lang, url, start, end)); } return futures; } /** * Build a list of CompletableFuture. The elements are the fetches of pages' * revisions from each domain language. * * @return a list of CompletableFuture */ private List> buildRevisionsFuturesList() { List> parallelRevisionsFutures = new ArrayList<>(); // Add revisions fetch for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelRevisionsFutures.add(revisionService.initRevisions(lang, url)); } return parallelRevisionsFutures; } private List> buildChangeCoefficientFuturesList(){ List> parallelCCFutures = new ArrayList<>(); // Add revisions fetch for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelCCFutures.add(revisionService.calculateChangeCoefficientAllRevisions(lang, url)); } return parallelCCFutures; } /** * Build a list of CompletableFuture. The first one is the fetch of the * users from the first domain in mediawiki.langs list. The rest of the * elements are the fetches of the pages for each language. This * implementation assumes that the users are shared among domains. * * @return a list of CompletableFuture */ private List> buildUsersAndPagesFuturesList() { List> usersAndPagesInsertion = new ArrayList<>(); // Add users fetch as fist operation usersAndPagesInsertion.add(userService.initUsers(protocol + langs.get(0) + "." + apiUrl)); // Add pages fetch for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; usersAndPagesInsertion.add(pageService.initPages(lang, url)); } return usersAndPagesInsertion; } /*@SuppressWarnings("unchecked") private List> buildFuturesList(Object obj, String methodPrefix) { List> futures = new ArrayList<>(); for (String lang : langs) { String url = protocol + lang + "." + apiUrl; Method[] methods = obj.getClass().getMethods(); for (int i = 0; i < methods.length; i++) { try { if (methods[i].getName().startsWith(methodPrefix) && methods[i].getParameterCount() == 1) { futures.add((CompletableFuture) methods[i].invoke(obj, url)); } else if (methods[i].getName().startsWith(methodPrefix) && methods[i].getParameterCount() == 2) { futures.add((CompletableFuture) methods[i].invoke(obj, lang, url)); } } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } return futures; }*/ } diff --git a/src/main/java/org/wikitolearn/wikirating/service/PageService.java b/src/main/java/org/wikitolearn/wikirating/service/PageService.java index 92100c4..588728e 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/PageService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/PageService.java @@ -1,333 +1,397 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.function.Predicate; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.GetPagesUpdateInfoException; import org.wikitolearn.wikirating.exception.PageNotFoundException; import org.wikitolearn.wikirating.exception.UpdatePagesAndRevisionsException; import org.wikitolearn.wikirating.model.CourseTree; import org.wikitolearn.wikirating.model.UpdateInfo; -import org.wikitolearn.wikirating.model.graph.Page; -import org.wikitolearn.wikirating.model.graph.Revision; +import org.wikitolearn.wikirating.model.graph.*; +import org.wikitolearn.wikirating.repository.CourseLevelThreeRepository; +import org.wikitolearn.wikirating.repository.CourseLevelTwoRepository; +import org.wikitolearn.wikirating.repository.CourseRootRepository; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.service.mediawiki.PageMediaWikiService; import org.wikitolearn.wikirating.service.mediawiki.UpdateMediaWikiService; +import org.wikitolearn.wikirating.util.enums.CourseLevel; /** * * @author aletundo, valsdav * */ @Service public class PageService { private static final Logger LOG = LoggerFactory.getLogger(PageService.class); @Autowired private PageMediaWikiService pageMediaWikiService; @Autowired private RevisionService revisionService; - @Autowired private PageRepository pageRepository; @Autowired private UpdateMediaWikiService updateMediaWikiService; @Autowired private UserService userService; + + @Autowired private PageRepository pageRepository; + @Autowired private CourseRootRepository courseRootRepository; + @Autowired private CourseLevelTwoRepository courseLevelTwoRepository; + @Autowired private CourseLevelThreeRepository courseLevelThreeRepository; + @Value("${mediawiki.namespace}") private String namespace; /** * This methods inserts all the pages inside the DB querying the MediaWiki API. * @param lang String * @param apiUrl String The MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initPages( String lang, String apiUrl ){ List pages = pageMediaWikiService.getAll(apiUrl); pages.forEach(page -> { page.setLang(lang); page.setLangPageId(lang + "_" +page.getPageId()); + //Now we check the level of the page to set the right + // additional laber for the Course Structure. + CourseLevel levelLabel = getPageLevelFromTitle(page.getTitle()); + if (levelLabel != CourseLevel.UNCATEGORIZED){ + page.addLabel(levelLabel.name()); + } }); pageRepository.save(pages); LOG.info("Inserted all {} pages", lang); return CompletableFuture.completedFuture(true); } - + + /** * * @param lang * @param apiUrl * @param start * @param end * @return * @throws UpdatePagesAndRevisionsException */ @Async public CompletableFuture updatePages(String lang, String apiUrl, Date start, Date end) throws UpdatePagesAndRevisionsException{ try{ List updates = updateMediaWikiService.getPagesUpdateInfo(apiUrl, namespace, start, end); for(UpdateInfo update : updates){ if(!namespace.equals(update.getNs())) continue; switch (update.getType()) { - case "new": - // Create the new page. The change coefficient for the new revision is set to 0 by default. - Revision newRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), - update.getOld_revid(), update.getNewlen(), update.getTimestamp()); - // Then create a new Page and link it with the revision - addPage(update.getPageid(), update.getTitle(), lang, newRev); - userService.setAuthorship(newRev); + case "new": + // Add the new page with the right Course level + Page newPage = addCoursePage(update.getPageLevelFromTitle(),update.getPageid(), update.getTitle(), lang); + if (update.getPageLevelFromTitle() == CourseLevel.CourseLevelThree) { + // Create the new revision. The change coefficient for the new revision is set to 0 by default. + Revision newRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), + update.getOld_revid(), update.getNewlen(), update.getTimestamp()); + // Add the first revision to the page + ((CourseLevelThree) newPage).initFirstRevision(newRev); + userService.setAuthorship(newRev); + } break; case "edit": - // Create a new revision - Revision updateRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), - update.getOld_revid(), update.getNewlen(), update.getTimestamp()); - // Then add it to the page - addRevisionToPage(lang + "_" + update.getPageid(), updateRev); - // Then calculate the changeCoefficient - revisionService.setChangeCoefficient(apiUrl, updateRev); - // Finally set the authorship - userService.setAuthorship(updateRev); + // Act only on CourseLevelThree pages + if (update.getPageLevelFromTitle() == CourseLevel.CourseLevelThree){ + // Create a new revision + Revision updateRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), + update.getOld_revid(), update.getNewlen(), update.getTimestamp()); + // Then add it to the page + addRevisionToPage(lang + "_" + update.getPageid(), updateRev); + // Then calculate the changeCoefficient + revisionService.setChangeCoefficient(apiUrl, updateRev); + // Finally set the authorship + userService.setAuthorship(updateRev); + } break; case "move": - // Move the page to the new title + // We have to change the label in case the Course level is changed + // Move the page to the new title movePage(update.getTitle(), update.getNewTitle(), lang); break; case "delete": // Delete the page and all its revisions deletePage(update.getTitle(), lang); break; default: break; } } }catch(GetPagesUpdateInfoException | PageNotFoundException e){ LOG.error("An error occurred while updating pages and revisions: {}", e.getMessage()); throw new UpdatePagesAndRevisionsException(); } return CompletableFuture.completedFuture(true); } + + /** + * Create a new generic Page entity. + * @param pageid + * @param title + * @param lang + * @return the added page + */ + public Page addPage(int pageid, String title, String lang){ + Page page = new Page(pageid, title, lang, lang + "_" + pageid); + pageRepository.save(page); + return page; + } + + /** + * Add a page entity distinguishing between the different Course levels. + * @param level + * @param pageid + * @param title + * @param lang + * @return + */ + public Page addCoursePage(CourseLevel level, int pageid, String title, String lang){ + switch(level){ + case CourseRoot: + CourseRoot pageRoot = new CourseRoot(pageid, title, lang, lang + "_" + pageid); + courseRootRepository.save(pageRoot); + return pageRoot; + case CourseLevelTwo: + CourseLevelTwo pageTwo = new CourseLevelTwo(pageid, title, lang, lang + "_" + pageid); + courseLevelTwoRepository.save(pageTwo); + return pageTwo; + case CourseLevelThree: + CourseLevelThree pageThree = new CourseLevelThree(pageid, title, lang, lang + "_" + pageid); + courseLevelThreeRepository.save(pageThree); + return pageThree; + default: + return addPage(pageid, title, lang); + } + } + /** - * Create a new Page. It requires the firstRevision of the Page in order - * to create the LAST_REVISION and FIRST_REVISION relationships. + * Create a new CourseLevelThree page. It requires the firstRevision of the Page in order + * to create the initial relationships. * @param pageid * @param title * @param lang * @param firstRevision * @return the added page */ - public Page addPage(int pageid, String title, String lang, Revision firstRevision){ - Page page = new Page(pageid, title, lang, lang + "_" + pageid); - // Add the links from the page to the first and last revision - page.setFistRevision(firstRevision); - page.setLastRevision(firstRevision); - pageRepository.save(page); - + public CourseLevelThree addCourseLevelThreePage(int pageid, String title, String lang, Revision firstRevision){ + CourseLevelThree page = new CourseLevelThree(pageid, title, lang, lang + "_" + pageid, firstRevision); + courseLevelThreeRepository.save(page); return page; } /** * Get the page with the given pageId and language * @param pageId the id of the page * @param lang the language of the page * @return the requested page * @throws PageNotFoundException */ public Page getPage(int pageId, String lang) throws PageNotFoundException{ Page page = pageRepository.findByLangPageId(lang + "_" + pageId); if(page == null){ LOG.error("Page with pageId {} and lang {} not found.", pageId, lang); throw new PageNotFoundException(); } return page; } /** * Get the page with the given langePageId * @param langPageId the langPageId of the page * @return the requested page * @throws PageNotFoundException */ public Page getPage(String langPageId) throws PageNotFoundException{ Page page = pageRepository.findByLangPageId(langPageId); if(page == null){ LOG.error("Page with langPageId: {} not found.", langPageId); throw new PageNotFoundException(); } return page; } /** - * Add a new revision to a page. It links the Page to the new revision via + * Add a new revision to a CourseLevelThree page. It links the page to the new revision via * LAST_REVISION link. Moreover it create the PREVIOUS_REVISION link. * @param langPageId * @param rev */ public void addRevisionToPage(String langPageId, Revision rev) throws PageNotFoundException{ - Page page = pageRepository.findByLangPageId(langPageId); + CourseLevelThree page = courseLevelThreeRepository.findByLangPageId(langPageId); if(page == null){ throw new PageNotFoundException(); } // Add PREVIOUS_REVISION relationship rev.setPreviousRevision(page.getLastRevision()); page.setLastRevision(rev); // The changes on the revision will be automatically persisted - pageRepository.save(page); + courseLevelThreeRepository.save(page); } /** * Change title of a page. The method is prefixed by move to follow MediaWiki naming. + * The method checks also the new page title to set the right Course level label + * in case of changes. * @param oldTitle the old title of the page * @param newTitle the new title to set * @param lang the language of the page * @return the updated page * @throws PageNotFoundException */ public Page movePage(String oldTitle, String newTitle, String lang) throws PageNotFoundException{ Page page = pageRepository.findByTitleAndLang(oldTitle, lang); if(page == null){ throw new PageNotFoundException(); } page.setTitle(newTitle); + // Check if the Course level has changed + if (getPageLevelFromTitle(oldTitle) != getPageLevelFromTitle(newTitle)){ + page.removeLabel(getPageLevelFromTitle(oldTitle).name()); + page.addLabel(getPageLevelFromTitle(newTitle).name()); + } pageRepository.save(page); return page; } - + /** * Delete a page from the graph given its title and domain language. * @param title the title of the page * @param lang the language of the page * @throws PageNotFoundException */ public void deletePage(String title, String lang) throws PageNotFoundException{ Page page = pageRepository.findByTitleAndLang(title, lang); if(page == null){ throw new PageNotFoundException(); } - // Delete the revisions of the page - revisionService.deleteRevisionsOfPage(page.getLangPageId()); + // Delete the revisions of the page if it's CourseLevelThree + if (page.hasLabel("CourseLevelThree")){ + revisionService.deleteRevisionsOfPage(page.getLangPageId()); + } // Delete finally the page itself pageRepository.delete(page); } + + /*** + * Get the level of a Page in the Course Structure + * anaylizing the number of slashes in the title. + * @param title + * @return null if the page has more that 2 slashes. + */ + public static CourseLevel getPageLevelFromTitle(String title){ + int nslash = StringUtils.countMatches(title, "/"); + switch (nslash){ + case 0: + return CourseLevel.CourseRoot; + case 1: + return CourseLevel.CourseLevelTwo; + case 2: + return CourseLevel.CourseLevelThree; + default: + //The page remains generic. + return CourseLevel.UNCATEGORIZED; + } + } /** * Get the pages labeled by :CourseRoot label * @return the list of course root pages */ - public List getCourseRootPages(String lang){ - return pageRepository.findAllCourseRootPages(lang); + public List getCourseRootPages(String lang){ + return courseRootRepository.findByLang(lang); } /** * Get the page labeled only by :Page label * @return the list of the uncategorized pages */ public List getUncategorizedPages(String lang){ return pageRepository.findAllUncategorizedPages(lang); } - /** - * Initialize for the first time the course structure using labels and relationships - * @param lang the language of the domain - * @param apiUrl the MediaWiki API url - */ - @Async - public CompletableFuture initCourseStructure(String lang, String apiUrl){ - List pages = pageRepository.findAllByLang(lang); - - // Remove all pages that are not course root pages - Predicate pagePredicate = page -> page.getTitle().contains("/"); - pages.removeIf(pagePredicate); - applyCourseStructure(lang, apiUrl, pages); - - return CompletableFuture.completedFuture(true); - } - + /** * * @param lang the language of the domain * @param apiUrl the MediaWiki API url * @return */ public CompletableFuture updateCourseStructure(String lang, String apiUrl) { - List courseRootPages = getCourseRootPages(lang); - List uncategorizedPages = getUncategorizedPages(lang); - - // Remove all uncategorized pages that are not course root pages - Predicate pagePredicate = page -> page.getTitle().contains("/"); - uncategorizedPages.removeIf(pagePredicate); - - courseRootPages.addAll(uncategorizedPages); - - applyCourseStructure(lang, apiUrl, courseRootPages); + List courseRootPages = getCourseRootPages(lang); + applyCourseStructure(lang, apiUrl, courseRootPages); return CompletableFuture.completedFuture(true); } /** * @param lang the language of the domain * @param apiUrl the MediaWiki API url * @param courseRootPages */ - private void applyCourseStructure(String lang, String apiUrl, List courseRootPages) { - for (Page p : courseRootPages) { + private void applyCourseStructure(String lang, String apiUrl, List courseRootPages) { + for (CourseRoot pageRoot : courseRootPages) { // Get course tree and prepare relationship set - CourseTree tree = pageMediaWikiService.getCourseTree(apiUrl, p.getTitle()); - Set levelsTwo = (p.getLevelsTwo() == null) ? new HashSet<>() : p.getLevelsTwo(); + CourseTree tree = pageMediaWikiService.getCourseTree(apiUrl, pageRoot.getTitle()); + Set levelsTwo = (pageRoot.getLevelsTwo() == null) ? new HashSet<>() : + pageRoot.getLevelsTwo(); int index = 0; for (String levelTwo : tree.getLevelsTwo()) { String levelTwoTitle = (tree.getRoot() + "/" + levelTwo).trim(); - Page levelTwoPage = pageRepository.findByTitleAndLang(levelTwoTitle, lang); + CourseLevelTwo levelTwoPage = courseLevelTwoRepository.findByTitleAndLang(levelTwoTitle, lang); // Skip malformed page if (levelTwoPage == null) continue; - // Add CourseLevelTwo label and add levels two to the set to be saved + // Add levelstwo to the set to be saved if(!levelsTwo.contains(levelTwoPage)){ - levelTwoPage.addLabel("CourseLevelTwo"); levelsTwo.add(levelTwoPage); } - Set levelsThree = (levelTwoPage.getLevelsThree() == null) ? new HashSet<>() + Set levelsThree = (levelTwoPage.getLevelsThree() == null) ? new HashSet<>() : levelTwoPage.getLevelsThree(); - // Add CourseLevelThree labels and add levels three to the set to be saved + // Add levels three to the set to be saved for (String levelThree : tree.getLevelsTree().get(index)) { String levelThreeTitle = (levelTwoTitle + "/" + levelThree).trim(); - Page levelThreePage = pageRepository.findByTitleAndLang(levelThreeTitle, lang); + CourseLevelThree levelThreePage = courseLevelThreeRepository.findByTitleAndLang(levelThreeTitle, lang); // Skip malformed page if (levelThreePage == null) continue; if(!levelsThree.contains(levelThreePage)){ - levelThreePage.addLabel("CourseLevelThree"); levelsThree.add(levelThreePage); } } // Set LEVEL_THREE relationships levelTwoPage.setLevelsThree(levelsThree); - pageRepository.save(levelsThree); + courseLevelThreeRepository.save(levelsThree); index++; } // Set LEVEL_TWO relationships and CourseRoot label - p.addLabel("CourseRoot"); - p.setLevelsTwo(levelsTwo); - pageRepository.save(levelsTwo); - pageRepository.save(p); + pageRoot.setLevelsTwo(levelsTwo); + courseLevelTwoRepository.save(levelsTwo); + courseRootRepository.save(pageRoot); } } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java index 19fb367..9c288be 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java @@ -1,209 +1,212 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.Date; import java.util.List; import java.util.ListIterator; import java.util.Set; import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.RevisionNotFoundException; -import org.wikitolearn.wikirating.model.graph.Page; +import org.wikitolearn.wikirating.model.graph.CourseLevelThree; import org.wikitolearn.wikirating.model.graph.Revision; -import org.wikitolearn.wikirating.repository.PageRepository; +import org.wikitolearn.wikirating.repository.CourseLevelThreeRepository; import org.wikitolearn.wikirating.repository.RevisionRepository; import org.wikitolearn.wikirating.service.mediawiki.RevisionMediaWikiService; import static java.lang.Math.exp; /** * * @author aletundo, valsdav * */ @Service public class RevisionService { private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); @Autowired private RevisionMediaWikiService revisionMediaWikiService; @Autowired private RevisionRepository revisionRepository; @Autowired - private PageRepository pageRepository; + private CourseLevelThreeRepository courseLevelThreeRepository; /** * Initialize the revisions for the first time querying the MediaWiki API. * This method adds the revisions and sets the FIRST_REVISION, * LAST_REVISION and PREVIOUS_REVISION relationships. + * Only the revisions of CourseLevelThree pages are fetched. * @param lang the domain language * @param apiUrl the MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initRevisions(String lang, String apiUrl) { - List pages = pageRepository.findAllByLang(lang); - for(Page page : pages){ + List pages = courseLevelThreeRepository.findByLang(lang); + for(CourseLevelThree page : pages){ List revisions = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, page.getPageId()); // Set the first and the last revisions for the current page - page.setFistRevision(revisions.get(0)); + page.setFirstRevision(revisions.get(0)); page.setLastRevision(revisions.get(revisions.size() - 1)); + // Set the last validated revision to the first one. + page.setLastValidatedRevision(revisions.get(0)); ListIterator it = revisions.listIterator(); while(it.hasNext()){ Revision rev = it.next(); rev.setLangRevId(lang + "_" + rev.getRevId()); rev.setLang(lang); if (it.previousIndex() != 0) { rev.setPreviousRevision(revisions.get(it.previousIndex() - 1)); } } // Saving all the revisions node and the page node revisionRepository.save(revisions); - pageRepository.save(page); + courseLevelThreeRepository.save(page); LOG.info("Inserted revisions for page {}", page.getLangPageId()); } return CompletableFuture.completedFuture(true); } @Async public CompletableFuture calculateChangeCoefficientAllRevisions(String lang, String apiUrl){ LOG.info("Calculating changeCoefficient for all the revisions..."); Set revisions = revisionRepository.findByLang(lang); for (Revision rev : revisions){ double changeCoefficient = calculateChangeCoefficient(apiUrl, rev); rev.setChangeCoefficient(changeCoefficient); } revisionRepository.save(revisions); LOG.info("ChangeCoefficient calculated for all revisions."); return CompletableFuture.completedFuture(true); } /** * Add a new Revision to the graph * @param revid * @param lang * @param userid * @param parentid * @param length * @param timestamp * @return */ public Revision addRevision(int revid, String lang, int userid, int parentid, int length, Date timestamp){ Revision rev = new Revision(revid, lang, userid, parentid, length, timestamp); revisionRepository.save(rev); return rev; } /** * Delete a revision given its langPageId * @param langPageId the langPageId of the revision */ public void deleteRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } revisionRepository.delete(revisions); } public Set getRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } return revisions; } /** * Get the requested revision * @param langRevId the langRevId of the revision * @return the revision * @throws RevisionNotFoundException */ public Revision getRevision(String langRevId) throws RevisionNotFoundException{ Revision revision = revisionRepository.findByLangRevId(langRevId); if(revision == null){ LOG.error("Revision {} not found", langRevId); throw new RevisionNotFoundException(); } return revision; } /** * Update the given revision * @param revision * @return the updated revision */ public Revision updateRevision(Revision revision){ revisionRepository.save(revision); return revision; } /** * Calculate and set the changeCoefficient of a Revision. * This method also persist the Revision changes. * @param apiUrl * @param revision */ public void setChangeCoefficient(String apiUrl, Revision revision){ double cc = calculateChangeCoefficient(apiUrl, revision); revision.setChangeCoefficient(cc); revisionRepository.save(revision); } /** * Calculate the changeCoefficient for a given Revision querying mediawiki * and returns the number. It doesn't store it in the Revision. * @param revision * @return */ public double calculateChangeCoefficient(String apiUrl, Revision revision){ double previousLength = 0.0; double changeCoefficient = 0.0; // Get the previous Revision Revision previousRevision = revision.getPreviousRevision(); if (previousRevision == null){ previousLength = 1.0; changeCoefficient = 0.0; LOG.info("Change coefficient of revision {} (first-rev): {}", revision.getLangRevId(), changeCoefficient); } else{ double prevL = (double) previousRevision.getLength(); // Suppose the mean line length of 120 characters and that the length is in bytes. // We want a "lenght" in n° of lines if (prevL == 0){ previousLength = 1.0; }else { previousLength = (prevL < 120) ? 1 : prevL / 120; } // Query mediawiki for diff text String diffText = revisionMediaWikiService.getDiffPreviousRevision(apiUrl, previousRevision.getRevId(), revision.getRevId()); int addedLines = StringUtils.countMatches(diffText, "diff-addedline"); int deletedLines = StringUtils.countMatches(diffText, "diff-deletedline"); //int inlineChanges = StringUtils.countMatches(diffText, "diffchange-inline"); // Finally calculation of change Coefficient double t = ((1.2 * deletedLines + addedLines) ) / previousLength; changeCoefficient = 1 / exp(0.7 * t); LOG.info("Change coefficient of revision {} (+{}-{}/{}): {}", revision.getLangRevId(), addedLines, deletedLines, previousLength, changeCoefficient); } return changeCoefficient; } } diff --git a/src/main/java/org/wikitolearn/wikirating/util/enums/CourseLevel.java b/src/main/java/org/wikitolearn/wikirating/util/enums/CourseLevel.java new file mode 100644 index 0000000..5c1fdd0 --- /dev/null +++ b/src/main/java/org/wikitolearn/wikirating/util/enums/CourseLevel.java @@ -0,0 +1,9 @@ +package org.wikitolearn.wikirating.util.enums; + +/** + * Possible levels for Course pages. + * Created by valsdav on 05/05/17. + */ +public enum CourseLevel { + CourseRoot, CourseLevelTwo, CourseLevelThree, UNCATEGORIZED; +} diff --git a/src/test/java/org/wikitolearn/wikirating/service/PageServiceTest.java b/src/test/java/org/wikitolearn/wikirating/service/PageServiceTest.java index 5ee92e8..045a9f7 100644 --- a/src/test/java/org/wikitolearn/wikirating/service/PageServiceTest.java +++ b/src/test/java/org/wikitolearn/wikirating/service/PageServiceTest.java @@ -1,146 +1,131 @@ /** * */ package org.wikitolearn.wikirating.service; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.ArrayList; import java.util.List; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.wikitolearn.wikirating.exception.PageNotFoundException; +import org.wikitolearn.wikirating.model.graph.CourseRoot; import org.wikitolearn.wikirating.model.graph.Page; import org.wikitolearn.wikirating.model.graph.Revision; import org.wikitolearn.wikirating.repository.PageRepository; /** * @author aletundo * */ @RunWith(SpringJUnit4ClassRunner.class) public class PageServiceTest { @Mock - private PageRepository pageRepository; + private PageRepository pageRepository; @InjectMocks private PageService pageService; @Before public void setup(){ MockitoAnnotations.initMocks(this); } @Test public void testGetPageByLangPageId(){ Page page = new Page(1, "Title", "en", "en_1"); when(pageRepository.findByLangPageId("en_1")).thenReturn(page); Page result = pageService.getPage("en_1"); assertEquals(1, result.getPageId()); assertEquals("Title", result.getTitle()); assertEquals("en", result.getLang()); assertEquals("en_1", result.getLangPageId()); } @Test public void testGetPageByPageIdAndLang(){ Page page = new Page(1, "Title", "en", "en_1"); when(pageRepository.findByLangPageId("en_1")).thenReturn(page); Page result = pageService.getPage(1, "en"); assertEquals(1, result.getPageId()); assertEquals("Title", result.getTitle()); assertEquals("en", result.getLang()); assertEquals("en_1", result.getLangPageId()); } @Test(expected = PageNotFoundException.class) public void testGetPageByPageIdAndLangNotFound(){ when(pageRepository.findByLangPageId("en_1")).thenReturn(null); pageService.getPage(1, "en"); } @Test(expected = PageNotFoundException.class) public void testGetPageByLangPageIdNotFound(){ when(pageRepository.findByLangPageId("en_1")).thenReturn(null); pageService.getPage("en_1"); } @Test(expected = PageNotFoundException.class) public void testDeletePageNotFound(){ when(pageRepository.findByTitleAndLang("Title", "en")).thenReturn(null); pageService.deletePage("Title", "en"); } @Test public void testDeletePage(){ Page page = new Page(1, "Title", "en", "en_1"); when(pageRepository.findByTitleAndLang("Title", "en")).thenReturn(page); pageService.deletePage("Title", "en"); verify(pageRepository, times(1)).delete(page); } @Test(expected = PageNotFoundException.class) public void testMovePageNotFound(){ when(pageRepository.findByLangPageId("en_1")).thenReturn(null); pageService.movePage("Title", "NewTitle", "en"); } @Test public void testMovePage(){ Page page = new Page(1, "Title", "en", "en_1"); when(pageRepository.findByTitleAndLang("Title", "en")).thenReturn(page); Page result = pageService.movePage("Title", "NewTitle", "en"); assertEquals("NewTitle", result.getTitle()); } @Test public void testAddPage(){ Page page = new Page(1, "Title", "en", "en_1"); when(pageRepository.save(page)).thenReturn(page); - Page result = pageService.addPage(1, "Title", "en", new Revision()); + Page result = pageService.addPage(1, "Title", "en"); assertEquals(1, result.getPageId()); assertEquals("Title", result.getTitle()); assertEquals("en", result.getLang()); assertEquals("en_1", result.getLangPageId()); } @Test public void testGetUncategorizedPages(){ List uncategorizedPages = new ArrayList(); uncategorizedPages.add(new Page(1, "Title", "en", "en_1")); uncategorizedPages.add(new Page(2, "Title2", "en", "en_2")); uncategorizedPages.add(new Page(3, "Title3", "en", "en_4")); when(pageRepository.findAllUncategorizedPages("en")).thenReturn(uncategorizedPages); List result = pageService.getUncategorizedPages("en"); assertEquals(3, result.size()); } - @Test - public void testGetCourseRootPages(){ - List uncategorizedPages = new ArrayList(); - Page p1 = new Page(1, "Title", "en", "en_1"); - p1.addLabel("CourseRoot"); - Page p2 = new Page(2, "Title2", "en", "en_2"); - p2.addLabel("CourseRoot"); - Page p3 = new Page(3, "Title3", "en", "en_3"); - p3.addLabel("CourseRoot"); - uncategorizedPages.add(p1); - uncategorizedPages.add(p2); - uncategorizedPages.add(p3); - when(pageRepository.findAllCourseRootPages("en")).thenReturn(uncategorizedPages); - List result = pageService.getCourseRootPages("en"); - assertEquals(3, result.size()); - } }